From 4477bc99118178a9e06b65691518af02cdc52e96 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 15:04:04 -0700 Subject: [PATCH 01/50] feat(runtime): isolate opt-in dissemination on merged lifecycle baseline --- .github/workflows/dissemination-api.yml | 59 + .../src/content/docs/host/monitoring/index.md | 3 +- .../docs/host/monitoring/metrics-catalog.md | 25 +- .../host/monitoring/runtime-dissemination.md | 96 + .../src/content/docs/implementation/index.md | 3 +- .../implementation/runtime-dissemination.md | 150 ++ docs/site/src/content/docs/toc.yml | 4 + .../IDs/SiloAddress.cs | 9 + .../Options/ClusterMembershipOptions.cs | 17 + .../Options/DisseminationOptions.cs | 195 ++ src/Orleans.Core/OrleansContracts.txt | 4 + src/Orleans.Core/Runtime/Constants.cs | 2 + .../IDisseminationSystemTarget.cs | 283 +++ .../Options/DeploymentLoadPublisherOptions.cs | 13 + .../Options/DisseminationOptionsValidator.cs | 110 ++ ...entLoadStatisticsDisseminationNamespace.cs | 128 ++ .../Dissemination/DisseminationApplyResult.cs | 9 + .../DisseminationBroadcastQueue.cs | 1595 +++++++++++++++++ .../Dissemination/DisseminationEvents.cs | 192 ++ .../Dissemination/DisseminationInstruments.cs | 235 +++ .../Dissemination/DisseminationMembership.cs | 130 ++ .../DisseminationMembershipSnapshot.cs | 144 ++ .../DisseminationNamespaceNames.cs | 7 + .../Dissemination/DisseminationProtocol.cs | 1340 ++++++++++++++ .../Dissemination/DisseminationSendGate.cs | 151 ++ .../DisseminationSystemTarget.cs | 145 ++ .../Dissemination/IDisseminationNamespace.cs | 96 + .../Dissemination/IDisseminationService.cs | 13 + .../MembershipDisseminationNamespace.cs | 482 +++++ .../Dissemination/WakeTimer.cs | 215 +++ .../Hosting/DefaultSiloServices.cs | 14 + .../MembershipService/MembershipGossiper.cs | 49 +- src/Orleans.Runtime/OrleansContracts.txt | 2 + .../Placement/DeploymentLoadPublisher.cs | 154 +- .../Runtime/DeploymentLoadPublisherTests.cs | 221 ++- 35 files changed, 6265 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/dissemination-api.yml create mode 100644 docs/site/src/content/docs/host/monitoring/runtime-dissemination.md create mode 100644 docs/site/src/content/docs/implementation/runtime-dissemination.md create mode 100644 src/Orleans.Core/Configuration/Options/DisseminationOptions.cs create mode 100644 src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs create mode 100644 src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs create mode 100644 src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs create mode 100644 src/Orleans.Runtime/Dissemination/DisseminationApplyResult.cs create mode 100644 src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs create mode 100644 src/Orleans.Runtime/Dissemination/DisseminationEvents.cs create mode 100644 src/Orleans.Runtime/Dissemination/DisseminationInstruments.cs create mode 100644 src/Orleans.Runtime/Dissemination/DisseminationMembership.cs create mode 100644 src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs create mode 100644 src/Orleans.Runtime/Dissemination/DisseminationNamespaceNames.cs create mode 100644 src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs create mode 100644 src/Orleans.Runtime/Dissemination/DisseminationSendGate.cs create mode 100644 src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs create mode 100644 src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs create mode 100644 src/Orleans.Runtime/Dissemination/IDisseminationService.cs create mode 100644 src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs create mode 100644 src/Orleans.Runtime/Dissemination/WakeTimer.cs diff --git a/.github/workflows/dissemination-api.yml b/.github/workflows/dissemination-api.yml new file mode 100644 index 00000000000..a7efecf93b9 --- /dev/null +++ b/.github/workflows/dissemination-api.yml @@ -0,0 +1,59 @@ +name: Dissemination API generation + +on: + workflow_dispatch: + pull_request: + branches: [main] + paths: + - .github/workflows/dissemination-api.yml + - src/Orleans.Core.Abstractions/IDs/SiloAddress.cs + - src/Orleans.Core/Configuration/Options/DisseminationOptions.cs + - src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs + - src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs + +permissions: + contents: read + +concurrency: + group: dissemination-api-${{ github.ref }} + cancel-in-progress: true + +jobs: + generate-api: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 + with: + global-json-file: global.json + - name: Generate affected public API surfaces + shell: bash + run: | + set -euo pipefail + export CI=false + for name in Orleans.Core.Abstractions Orleans.Core Orleans.Runtime; do + project="src/$name/$name.csproj" + dotnet restore "$project" --configfile .github/NuGet.GenAPI.Config -p:GenerateOrleansApiSource=true + dotnet build "$project" --framework net8.0 --configuration Release --no-incremental --no-restore -p:GenerateOrleansApiSource=true '/t:Build;GenAPIGenerateReferenceAssemblySource' "-bl:genapi-$name.binlog" + done + - name: Upload exact generated surfaces + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-generated-api + path: | + src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs + src/api/Orleans.Core/Orleans.Core.cs + src/api/Orleans.Runtime/Orleans.Runtime.cs + if-no-files-found: error + retention-days: 7 + - name: Upload generation diagnostics + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-api-binlogs + path: genapi-*.binlog + retention-days: 7 diff --git a/docs/site/src/content/docs/host/monitoring/index.md b/docs/site/src/content/docs/host/monitoring/index.md index e60e9c0e51b..268616b89aa 100644 --- a/docs/site/src/content/docs/host/monitoring/index.md +++ b/docs/site/src/content/docs/host/monitoring/index.md @@ -1,7 +1,7 @@ --- title: Orleans observability description: Configure OpenTelemetry logging, metrics, and tracing for Orleans. -ms.date: 08/02/2026 +ms.date: 09/04/2026 ms.topic: overview --- @@ -79,6 +79,7 @@ The example's 10% head-sampling ratio is a starting point, not a universal produ ## Next steps - [Monitor Orleans metrics](metrics.md) +- [Monitor runtime dissemination](runtime-dissemination.md) - [Interpret Orleans signals](signals.md) - [Troubleshoot Orleans incidents](troubleshooting.md) - [Secure and operate the Orleans Dashboard](../../dashboard/index.md) diff --git a/docs/site/src/content/docs/host/monitoring/metrics-catalog.md b/docs/site/src/content/docs/host/monitoring/metrics-catalog.md index 6fc6dc0abd5..e729b784ee7 100644 --- a/docs/site/src/content/docs/host/monitoring/metrics-catalog.md +++ b/docs/site/src/content/docs/host/monitoring/metrics-catalog.md @@ -1,7 +1,7 @@ --- title: Orleans metrics catalog description: Complete reference for metrics emitted by the Microsoft.Orleans meter. -ms.date: 08/18/2026 +ms.date: 09/04/2026 ms.topic: reference --- @@ -65,6 +65,29 @@ Request latency covers the interval until the caller's callback completes, inclu `Destination` and `silo` contain silo addresses and incarnations. They are useful during incident diagnosis and can create new time series during restarts. +## Runtime dissemination + +| Instrument | Type | Unit | Attributes | Description | +|---|---|---|---|---| +| `orleans-dissemination-anti-entropy-digests` | C | `digests` | `direction` | Digest entries sent in successful outbound exchanges or received in inbound exchanges. | +| `orleans-dissemination-anti-entropy-exchanges` | C | `operations` | `direction`, `truncated` | Successful anti-entropy requests and responses. `truncated=true` indicates that response budgets left candidates for a later exchange. | +| `orleans-dissemination-anti-entropy-failures` | C | `operations` | `reason` | Outbound anti-entropy peer operations which ended in `timeout` or `error`. | +| `orleans-dissemination-anti-entropy-values` | C | `values` | `direction` | Repair values returned by successful anti-entropy exchanges. | +| `orleans-dissemination-broadcast-received` | C | `messages` | `namespace`, `kind` | Broadcast batches received, counted once for each namespace represented in the batch. | +| `orleans-dissemination-broadcast-scheduled` | C | `schedules` | `reason` | Per-peer pump schedules split into `immediate`, `coalesce`, `retry`, and `priority`. | +| `orleans-dissemination-broadcast-send-failures` | C | `attempts` | `reason` | Broadcast send attempts which ended in `timeout` or `error`. | +| `orleans-dissemination-broadcast-sent` | C | `messages` | `namespace`, `kind` | Successfully completed broadcast sends, counted once for each namespace represented in the batch. | +| `orleans-dissemination-bytes-sent` | C | `bytes` | `namespace`, `kind` | Serialized dissemination payload bytes in successful broadcasts. | +| `orleans-dissemination-payload-dropped` | C | `values` | `namespace`, `reason` | Values rejected by a payload guard. | +| `orleans-dissemination-pump-failures` | C | `failures` | `status` | Unexpected peer-pump failures split into `recovered` and `permanent`. | +| `orleans-dissemination-publications` | C | `operations` | `namespace`, `result`, `reason` | Publication attempts classified as `accepted` or `rejected`. Rejection reasons identify disabled namespaces, unavailable membership or values, invalid versions or repairs, and over-budget repairs. | +| `orleans-dissemination-queue-admission-rejected` | C | `keys` | `namespace`, `reason` | New distinct keys rejected after a namespace reached its per-peer pending-key limit. | +| `orleans-dissemination-values-applied` | C | `values` | `namespace`, `result` | Received values classified by the namespace apply result. | +| `orleans-dissemination-values-received` | C | `values` | `namespace`, `kind` | Values included in received broadcast batches before individual application. | +| `orleans-dissemination-values-sent` | C | `values` | `namespace`, `kind` | Values included in successfully completed broadcast sends. | + +All dissemination attributes have bounded runtime-defined values. Keys and peer addresses are available through opt-in diagnostic events instead of metric attributes. See [Monitor runtime dissemination](runtime-dissemination.md). + ## Scheduling, activations, and grains | Instrument | Type | Unit | Attributes | Description | diff --git a/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md b/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md new file mode 100644 index 00000000000..fa43aa6b655 --- /dev/null +++ b/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md @@ -0,0 +1,96 @@ +--- +title: Monitor runtime dissemination +description: Monitor convergence, retry pressure, repair, backpressure, and fallback behavior in Orleans runtime dissemination. +ms.date: 09/10/2026 +ms.topic: concept-article +--- + +# Monitor runtime dissemination + +Runtime dissemination accelerates convergence of membership and deployment-load state. Monitor its progress and failure signals alongside the authoritative membership, placement, messaging, and networking signals. + +The instruments use the standard `Microsoft.Orleans` meter: + +```dotnetcli +dotnet-counters monitor -n --counters Microsoft.Orleans +``` + +See [Runtime state dissemination](../../implementation/runtime-dissemination.md) for protocol invariants and failure semantics and the [metrics catalog](metrics-catalog.md#runtime-dissemination) for the complete instrument reference. + +## Build a convergence dashboard + +Use rates over the same interval and split by `namespace` where available: + +| Signal | Interpretation | +|---|---| +| Broadcast sent and received | Tree traffic reaches peers. Compare cluster-wide rates during active publication. | +| Values sent, received, and applied | Receipt measures delivered work. Break apply `result` into applied, duplicate, obsolete, and rejected values. | +| Broadcast scheduled with `reason=retry` | A peer pump retained work after incomplete delivery or repair. | +| Broadcast send failures | `timeout` indicates the hop lifetime expired; `error` indicates an RPC or transport failure. | +| Anti-entropy exchanges and values | Periodic repair remains active and returns bounded work. | +| Anti-entropy failures | Selected repair peers timed out or failed. Rotating peer selection uses other peers when independent repair capacity is available. | +| Publications | `accepted` measures work admitted to peer pumps. `rejected` identifies disabled, unavailable, invalid, over-budget, or queue-rejected publications which retain the component's direct path. | +| Queue admission rejected | One namespace reached its per-peer distinct-key bound. | +| Pump failures | `recovered` records an isolated iteration failure; `permanent` records a pump whose waiters were failed explicitly. | + +The `truncated=true` anti-entropy series means a response exhausted an item or byte budget while more candidates remained. Occasional truncation is bounded continuation. Sustained truncation with low applied-value throughput indicates that budgets, payload sizes, or publication churn are limiting convergence. + +## Alert on sustained loss of progress + +Page on user-visible impact first. Dissemination signals identify a contributing runtime path: + +- Alert on permanent pump failures because one destination has stopped accepting broadcast work until its peer state is recreated. +- Alert on sustained queue admission rejection because updates for new keys are using fallback paths or waiting for capacity. +- Correlate broadcast and anti-entropy failure rates with `orleans-messaging-sent-failed`, socket churn, membership changes, and deployment events. +- Compare rejected publications with the runtime component's update rate. A rollout can temporarily increase direct-path use while peer capability evidence is established. +- Investigate sustained retry scheduling when successful broadcasts and applied values remain flat. + +Timeout and retry spikes during a rolling restart or network partition can be expected. Healthy recovery produces successful exchanges, declining retry and failure rates, and continued applied or duplicate outcomes after connectivity returns. + +## Diagnose by symptom + +### One silo has persistent retries + +Compare that instance's send failures with messaging and socket metrics. Inspect `Orleans.Runtime.Dissemination.DisseminationBroadcastQueue` logs for the peer and retry delay. A permanent pump failure is logged at Error; a recovered iteration failure is logged at Warning. + +One destination occupies at most one active local broadcast slot. Its hop deadline or pump cancellation releases that slot and signals the RPC's cancellation token. A queued retry retains identities until admission. Other ready destinations use released slots in FIFO order, and backlogged peers rejoin admission per batch. + +### Traffic stops despite bounded retries + +Check local attempt duration and timeout rates against the namespace hop lifetime. Local TTL expiry ends a pump's wait and releases its slot. Repair has a separate local attempt limit from , so it can continue during broadcast saturation. + +Correlate sustained lack of progress with retry backoff, scheduler starvation, timer processing, and transport failures. Inspect pending messaging RPCs separately: normally supplies their terminal response bound even when cancellation acknowledgment is delayed. Tune concurrency together with hop lifetimes and retry cadence to control new attempt rates during transport failures. + +Retry scheduling metrics describe timer decisions. The send-gate diagnostic identifies requests which entered the FIFO; cancelling a flush observer leaves that shared request queued. Bound explicit flush and drain waits using caller cancellation tokens. Repair selection skips locally busy destinations until a later round. Successful responses completed within a repair round still apply when another selected peer misses the deadline. + +### Queue admission is rejected + +Identify the `namespace` and compare update cardinality with . Increase the limit only after confirming the process has memory headroom and the destination can drain work. Broadcast item and byte limits still bound each transmission. + +Capacity planning must include all retained peer/namespace pairs: pending-key storage scales with their summed per-peer limits, and acknowledgment ledgers scale with known keys. Active local attempts materialize at most one batch per broadcast slot, bounded by . Namespace history, metadata, and messaging buffers retained by pending RPCs remain additional costs. + +### Anti-entropy remains truncated + +Compare returned values with , , and the namespace payload limit. Persistent cursors continue from the bounded response across later exchanges, so verify that exchanges and applied values continue rather than alerting on truncation alone. + +### Rejected publications increase + +Break down by `namespace` and `reason`. Validate peer version compatibility, payload limits, and namespace enablement. `queue-rejected` identifies a full or stopping broadcast queue; correlate it with queue admission rejection and silo lifecycle activity. Deployment-load and membership integrations retain direct paths, so also inspect their component-specific logs and metrics for the authoritative outcome. + +## DiagnosticListener events + +The `Microsoft.Orleans.Dissemination` diagnostic listener exposes detailed, opt-in events: + +| Event | Use | +|---|---| +| `Dissemination.ValueApply` | Inspect one value's namespace, key, version transition, peer, result, and payload size. | +| `Dissemination.PayloadDrop` | Inspect a value rejected by a payload guard. | +| `Dissemination.BroadcastScheduled` | Observe immediate, coalesced, priority, and retry scheduling with due time and attempt. | +| `Dissemination.SendGate` | Observe broadcast FIFO entry (`Kind=broadcast`, `Stage=queued`) or a local repair attempt releasing its admission (`Kind=repair`, `Stage=released`), including timeout and cancellation. | +| `Dissemination.QueueAdmissionRejected` | Observe the peer, namespace, configured limit, and bounded rejection reason. | + +The send-gate event is emitted outside scheduler locks after queue insertion or accounting release. Admission or cancellation can race with event delivery, so it records a transition rather than a current queue-length measurement. + +Its typed payload includes `LocalSilo`, `Peer`, `Timestamp`, and bounded `Kind` and `Stage` strings. The repair-release transition can also report cancellation or failure before an admitted exchange sends its request. A later exchange may already have reused released capacity when an observer processes the event. + +Value events include keys and peer addresses; admission events include peer addresses. Enable them for a time-limited investigation and apply the deployment's telemetry data and retention policy. Metrics intentionally omit keys and peer addresses to keep long-term cardinality bounded. diff --git a/docs/site/src/content/docs/implementation/index.md b/docs/site/src/content/docs/implementation/index.md index 723ed8606e4..b5f5bc5e377 100644 --- a/docs/site/src/content/docs/implementation/index.md +++ b/docs/site/src/content/docs/implementation/index.md @@ -1,7 +1,7 @@ --- title: Orleans runtime architecture description: An advanced guide to the protocols, invariants, and extension points inside the Orleans runtime. -ms.date: 08/02/2026 +ms.date: 09/04/2026 ms.topic: overview --- @@ -38,6 +38,7 @@ Use this runtime map and the following topic list to choose the required depth. - [Runtime architecture](runtime-architecture.md) follows a call through client, messaging, placement, directory, activation, and scheduling components. - [Activation lifecycle and migration](activation-lifecycle.md) explains creation, activation, collection, deactivation, and state transfer. - [Cluster membership](cluster-management.md) describes the failure detector, membership table, ordered views, and death-vote protocol. +- [Runtime state dissemination](runtime-dissemination.md) explains deterministic broadcast, acknowledged peer state, bounded repair, mixed-version behavior, and integration boundaries. - [Grain directory](grain-directory.md) distinguishes the default `LocalGrainDirectory` DHT from the experimental distributed directory. - [Scheduling and turn execution](scheduler.md) explains `WorkItemGroup`, continuations, interleaving, and single-threaded execution. - [Messaging and delivery semantics](messaging-delivery-guarantees.md) traces requests and explains why a timeout has an unknown outcome. diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md new file mode 100644 index 00000000000..7f29f95e223 --- /dev/null +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -0,0 +1,150 @@ +--- +title: Runtime state dissemination +description: Architecture, invariants, repair, failure semantics, and integration boundaries for Orleans runtime dissemination. +ms.date: 09/10/2026 +ms.topic: concept-article +--- + +# Runtime state dissemination + +Runtime dissemination is an internal silo-to-silo protocol which accelerates convergence of frequently changing runtime state. A deterministic tree carries new versions quickly, acknowledgments record peer knowledge, and bounded anti-entropy repairs loss, reordering, partitions, and membership skew. Each participating runtime component remains authoritative for its own state. + +The dissemination protocol currently carries: + +- deployment-load statistics among Active silos; +- membership snapshots and diffs among Joining, Active, ShuttingDown, and Stopping silos. + +The deployment-load publisher and membership manager retain their direct delivery paths and validation rules. Tree broadcast and anti-entropy form one convergence feature: broadcast accelerates new updates, and repair discovers and delivers state missed by the tree. + +## Components and responsibilities + +```mermaid +flowchart LR + Producer[Runtime state producer] --> Namespace[IDisseminationNamespace] + Namespace --> Protocol[DisseminationProtocol] + Protocol --> Queue[Per-peer broadcast pumps] + Queue --> Target[Remote dissemination system target] + Target --> Apply[Namespace apply] + Apply --> Forward[Deterministic forwarding tree] + Protocol <--> Repair[Bounded anti-entropy] + Namespace --> Authority[Authoritative runtime component] + Repair --> Authority +``` + +`DisseminationProtocol` coordinates routing, peer capability evidence, anti-entropy, and application isolation. `DisseminationBroadcastQueue` owns per-peer scheduling, coalescing, retry, drain, and acknowledged-version ledgers. `DisseminationMembership` projects one membership snapshot into topology-specific member sets. Each `IDisseminationNamespace` owns serialization, current versions, retained history, payload limits, repair construction, and application semantics. + +Queue entries contain `(namespace, key)` identities. A pump acquires both destination ownership and a global broadcast slot before materializing a repair. Coalesced notifications therefore use the latest namespace state and acknowledged baseline after admission, without retaining serialized payloads for waiting peers. + +Applied state wakes each forwarding child, including same-version membership liveness advances. Duplicate deliveries wake children whose versions are still missing and whose queues contain no equivalent work. This lets a first tree delivery forward state already learned through a direct path while suppressing repeated forwarding cycles between silos with different topology views. + +## Version and application invariants + +A `DisseminationValue` names one key and a `[FromVersion, ToVersion]` transition: + +- a full value starts at version zero and can establish state without a retained baseline; +- a delta starts at the receiver baseline and must form a contiguous version chain; +- acknowledged peer versions advance monotonically from explicit receiver evidence; +- duplicate and obsolete values leave authoritative state unchanged; and +- one rejected value does not prevent later values in the batch from being considered. + +The namespace reports whether a repair is current, produced, unavailable, or unable to fit the supplied item and byte budgets. Produced repairs can be complete or a valid prefix. Prefix acknowledgments establish the receiver's new baseline and let the next batch continue the chain. + +Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. It retains 32 snapshots, prefers a smaller diff when the peer baseline is present, and falls back to a full snapshot when history or capacity makes the diff unsuitable. Apply accepts same-version snapshots only when they advance liveness and preserves newer local liveness when applying a diff. + +Deployment-load versions are sample timestamps. Each repair is a full latest value. Application runs on the deployment-load publisher's scheduler and accepts samples only for Active silo generations in the membership oracle. Terminal membership transitions remove placement statistics, and the oracle's monotonic membership state rejects later samples for departed generations. + +## Deterministic topology + +Every silo derives routing from the same ordered membership projection. Members are ordered by status and silo address. Silo addresses order by generation, port, and IP address, providing stable ordering across membership storage providers with different timestamp precision. Joining, Active, ShuttingDown, and Stopping entries participate. + +For fanout `f` and zero-based member index `i`, a forwarding node selects children starting at `f * (i + 1)` and continuing for at most `f` members. An originator sends to the first `f` members, excluding itself, plus its normal forwarding children. Stable ordering makes parent and child selection deterministic for a given membership version and fanout. + +Namespaces select a membership scope before topology construction: + +| Namespace | Scope | Operational effect | +|---|---|---| +| Deployment load | Active members | Placement data follows serving capacity. | +| Membership | All dissemination members | Joining and graceful-shutdown transitions can propagate. | + +Fanout is derived from the target hop count and bounded by the configured minimum and maximum, or selected by the code-configured callback. A membership or fanout change creates a new topology from the next snapshot; acknowledged ledgers and anti-entropy repair convergence across the transition. + +## Configuration and defaults + +The subsystem and each namespace default to disabled. Enable together with on the selected integration to activate broadcast and repair. Defaults bound concurrency, memory retention, payload size, and repair work: + +| Option | Default | Effect | +|---|---:|---| +| | 32 | Per-silo active local broadcast attempts. | +| | 8,192 | Values materialized in one batch. | +| | 1 MiB | Serialized payload bytes in one batch. | +| | 2 | Target depth used to derive fanout. | +| / | 4 / 32 | Bounds for derived fanout. | +| | 5 seconds | Repair-round cadence and retry-delay ceiling. | +| | 3 | Maximum peers selected in one repair round and independent per-silo active local repair attempt limit. | +| | 1,024 | Distinct retained keys per namespace and peer. | +| | 100 ms | Normal-priority batching window. | +| | 30 seconds | Per-hop transport and application lifetime. | +| | 10 seconds | Quiet period before a digest is offered for repair. | +| | 1 MiB | Maximum serialized value size for the namespace. | + +Each integration has its own . Operators can enable and tune membership and deployment-load dissemination independently while retaining the local concurrency and per-message bounds. + +Enablement requires representative measurements of convergence, RPC volume, serialized bytes, allocation, CPU, and tail latency across stable membership, churn, and partition recovery. Tree delivery can reduce RPC counts while increasing serialized bytes through forwarding and repair. Compare the original runtime, the updated default-off path, and the enabled path at the intended cluster sizes and publication rates, including skewed peer speeds and rolling upgrades. + +## Broadcast pumps and backpressure + +Each destination has one independent pump. Repeated notifications coalesce by namespace and key, high-priority namespaces bypass the coalescing window, and batches obey global item and byte bounds plus namespace payload bounds. An owner-wide managed admission gate combines the local broadcast limit with one active local broadcast attempt per destination. + +Admission uses an explicit FIFO linked list. Releasing a batch admits the oldest ready destinations, skipping peers with an active local attempt. A peer with a backlog or acknowledged repair prefix rejoins admission for every batch, behind other ready peers. Priority changes when a pump becomes ready, rather than allowing it to overtake already-ready peers. Destination ownership uses the full `SiloAddress`, including generation, and ends with the local attempt. + +`MaxPendingItemCount` bounds distinct retained keys per namespace and peer. A notification for an admitted key refreshes its generation at the limit. A new key is rejected until acknowledgment-driven completion, an oversized repair, or membership pruning releases capacity. Oversized repairs retain acknowledged peer versions separately so a later publication can resume from the established baseline. The runtime emits a rejection metric and a diagnostic event without including the key. A rejected tree target makes publication return `false`, allowing the producer's direct path to deliver the update. + +The aggregate pending-key bound is topology-dependent: sum `MaxPendingItemCount` over the retained peer/namespace pairs, with identity-only snapshots for active flushes. Acknowledged-version ledgers additionally scale with known namespace keys and retained peers and are pruned against membership and the authoritative key inventory. + +With fixed configuration, active local broadcast attempts materialize at most `MaxConcurrentSends` batches, each bounded by `MaxBatchBytes`. Waiting peers retain identities until admission. Namespace-owned current values and repair history, identity and acknowledgment metadata, and messaging-layer buffers for pending RPCs are separate memory costs. Broadcast and aggregate repair concurrency limits are captured when the protocol is constructed; tune them before silo startup. + +A successful RPC is transport completion. The receiver response is the evidence which advances the peer ledger. Missing or prefix acknowledgments retain work for repair. Transport timeout and failure requeue the generation with exponential backoff capped by the anti-entropy interval. A newer notification can replace that schedule with its own urgency. + +The transport hop lifetime starts after admission. Local completion, timeout, or pump cancellation releases both the broadcast slot and destination ownership. Timeout signals the RPC's cancellation token and requeues the generation with backoff, allowing subsequent attempts and other ready destinations to proceed while the earlier RPC completes under the messaging runtime's lifetime rules. Pump cancellation removes pending admission requests; a concurrent grant is either used by the caller or released exactly once. + +Admission retains its FIFO position until capacity is available or the pump stops. An explicit flush caller supplies its own bounded cancellation token to limit its wait for capacity. Cancelling that observer throws cancellation while leaving the shared pump in its FIFO position. Shutdown uses its caller's cancellation budget to stop the pump and remove its pending admission. + +Unexpected iteration failures retain work and retry. A failure in the recovery path permanently fails the pump and explicitly completes flush and drain waiters with the failure. Shutdown can drain accepted work within the caller's cancellation budget or discard a removed peer's work during membership pruning. + +## Bounded anti-entropy + +An anti-entropy round rotates through at most `AntiEntropyPeerCount` eligible peers. Rotation progress persists across membership versions so frequent table updates still allow every eligible peer to participate. The broadest participating membership scope provides the global peer budget; each request includes only namespaces for which that peer is eligible. + +Repair has its own admission gate, independent of broadcast slots. Across overlapping rounds, at most `AntiEntropyPeerCount` local repair attempts are active, with at most one per destination. Busy slots and destinations are skipped until future rotating rounds. A round creates its shared digest snapshot and per-peer requests only after acquiring slots. Request digest storage scales with namespace key inventory. Each exchange bounds its local wait using the round's lifetime and cancellation token, and releases admission before the round completes. + +Digests identify namespace-owned keys, including missing state at version zero, so repair can discover work rejected during queue admission. Recently advanced streams suppress redundant probes until `ExpectedUpdateCadence` elapses. Responses obey item, batch-byte, payload-byte, and hop-lifetime bounds. Persistent per-requester cursors rotate truncated responses so hot early keys cannot starve later candidates. Cursor state for non-members is least-recently-used and bounded to 64 entries. + +Repairs from each sender remain ordered. Competing sender chains are ranked before application, and the namespace's monotonic apply contract rejects an obsolete or incompatible chain. Responses which completed successfully within their local attempt are still applied when another peer exceeds the round deadline. Late RPC faults are observed, and subsequent rounds repair responses which missed the application window. During a partition, local attempts remain bounded and retries follow the configured cadence; after connectivity and membership views recover, rotating peer selection and cursors continue convergence. + +## Mixed-version behavior + +Peer namespace support is evidence-driven. Inbound traffic, broadcast acknowledgments, and authoritative anti-entropy responses confirm support for one silo generation. An explicit unsupported response revokes it, and membership pruning removes evidence for departed generations. + +Deployment-load publication uses dissemination for confirmed peers and the direct system target for unconfirmed peers. If dissemination is disabled, unavailable, rejected, or unable to accept the update within one refresh interval, direct fanout covers all Active peers. Membership direct gossip begins before optional dissemination and keeps the caller's cancellation and shutdown deadline. + +## Concurrency and lifetime + +Protocol dictionaries use focused locks for membership projections, response cursors, value-update timestamps, peer capability evidence, and per-peer pump state. Network calls, namespace application, logging, diagnostic callbacks, and waiter continuations run outside those locks. Waiters use asynchronous continuations. + +Caller cancellation owns public operation lifetime and is checked before each received value is applied. Per-hop value TTL bounds the local broadcast RPC wait and anti-entropy round lifetime, independently of the runtime's cancellation-acknowledgment setting. Shared admission waits are pump-cancellable and retain their FIFO position while capacity is unavailable. Explicit flush and drain observers use their caller's cancellation token, with no separate observer TTL; supply a bounded token when waiting indefinitely for admission is unacceptable. + +Stopping rejects new notifications and repair admissions. Accepted broadcast work can drain within the shutdown caller's budget; cancellation stops its queued admissions, local RPC waits, and pump timers. Each local attempt releases its managed admission lease during cleanup. Once pumps stop, the broadcast gate closes to further admissions. Repair cancellation is explicitly forwarded before disposing the round's linked sources, even when the local wait completes cancellation first. + +A stalled destination occupies at most one active local broadcast slot and one independent local repair slot. Hop deadlines release those slots and signal cancellation, allowing queued broadcasts and future repair rounds to continue. Retry backoff, rotating peer selection, per-message budgets, and local admission together bound the pace and size of new attempts. + +The messaging runtime owns pending RPC completion. `InsideRuntimeClient` monitors callback expiry using the system-target response timeout, normally unless the method overrides it. Cancellation follows the configured acknowledgment policy, and `CallbackData.OnTimeout` completes and unregisters expired requests. Remote handlers can continue after a local dissemination attempt ends; their execution follows ordinary runtime cancellation and response-timeout behavior. + +Dissemination RPC contracts carry required cancellation tokens through their implementations, callers, and scheduler handoffs. Background loops receive their owned shutdown token explicitly. Deployment-load timer callbacks forward the timer's cancellation token through publication and direct statistics RPCs, and disposing the timer cancels an active publication. Fault observers follow late transport completion after a local broadcast or repair attempt ends. + +Deployment-load mutation and subscriber delivery run on the publisher's scheduler. Concurrent dictionary reads support namespace digests and repair construction. Direct publications retain their existing equal-timestamp notification behavior; dissemination classifies equal timestamps as duplicates so repair settles without repeated subscriber delivery. + +## Observability + +The standard `Microsoft.Orleans` meter exposes publication outcomes, broadcast volume, payload bytes, apply outcomes, retries, failures, anti-entropy work, drops, pump failures, and queue admission rejection. Dimensions are bounded to namespace, direction, kind, result, reason, truncation, and pump status. Diagnostic events provide payload-level apply/drop detail and deterministic pump scheduling detail for short-lived investigation. + +See [Monitor runtime dissemination](../host/monitoring/runtime-dissemination.md) for instrument semantics, dashboards, alerts, and failure diagnosis. diff --git a/docs/site/src/content/docs/toc.yml b/docs/site/src/content/docs/toc.yml index 02dd32bf151..1c1fa62c282 100644 --- a/docs/site/src/content/docs/toc.yml +++ b/docs/site/src/content/docs/toc.yml @@ -267,6 +267,8 @@ items: href: host/monitoring/signals.md - name: Metrics href: host/monitoring/metrics.md + - name: Runtime dissemination + href: host/monitoring/runtime-dissemination.md - name: Metrics catalog href: host/monitoring/metrics-catalog.md - name: Troubleshooting workflow @@ -347,6 +349,8 @@ items: href: implementation/cluster-management.md - name: Cluster manifest retrieval href: implementation/cluster-manifest-retrieval.md + - name: Runtime state dissemination + href: implementation/runtime-dissemination.md - name: Runtime lifecycle href: implementation/orleans-lifecycle.md - name: Scheduling and turn execution diff --git a/src/Orleans.Core.Abstractions/IDs/SiloAddress.cs b/src/Orleans.Core.Abstractions/IDs/SiloAddress.cs index 23b13db9a68..48f8e2f9ac0 100644 --- a/src/Orleans.Core.Abstractions/IDs/SiloAddress.cs +++ b/src/Orleans.Core.Abstractions/IDs/SiloAddress.cs @@ -24,6 +24,7 @@ namespace Orleans.Runtime public sealed class SiloAddress : IEquatable, IComparable, + IComparable, ISpanFormattable, IParsable, IUtf8SpanParsable @@ -566,6 +567,14 @@ public int CompareTo(SiloAddress? other) return IPAddressComparer.Instance.Compare(Endpoint.Address, other.Endpoint.Address); } + + int IComparable.CompareTo(object? obj) + { + if (obj is null) return 1; + return obj is SiloAddress other + ? CompareTo(other) + : throw new ArgumentException($"Object must be of type {nameof(SiloAddress)}.", nameof(obj)); + } } /// diff --git a/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs b/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs index 246e7dbdbd6..9a53a1c6a44 100644 --- a/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs +++ b/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs @@ -93,6 +93,23 @@ public TimeSpan MaxProbeTimeout /// Membership updates are disseminated using gossip by default. public bool UseLivenessGossip { get; set; } = true; + /// + /// Gets or sets dissemination options for membership updates. + /// + /// + /// Membership dissemination includes Joining, Active, ShuttingDown, and Stopping silos and is a + /// best-effort accelerator. Direct membership gossip delivers shutdown-critical updates and reaches peers + /// during mixed-version operation. + /// + public DisseminationNamespaceOptions Dissemination { get; set; } = new() + { + ExpectedUpdateCadence = TimeSpan.FromSeconds(10), + + // Membership changes must propagate as quickly as possible, ahead of and without waiting to coalesce + // with other gossip. + Priority = DisseminationPriority.High, + }; + /// /// Gets or sets the number of silos each silo probes for liveness. /// diff --git a/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs new file mode 100644 index 00000000000..d0b44a35a82 --- /dev/null +++ b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs @@ -0,0 +1,195 @@ +using System; + +namespace Orleans.Configuration; + +/// +/// Options for configuring internal silo-to-silo dissemination. +/// +public sealed class DisseminationOptions +{ + /// + /// Gets or sets a value indicating whether the dissemination subsystem is enabled. + /// + /// by default. + public bool Enabled { get; set; } + + /// + /// Gets or sets the maximum number of concurrent local dissemination broadcast attempts. + /// + /// + /// Each attempt releases capacity when its local wait completes, including timeout or cancellation. + /// + public int MaxConcurrentSends { get; set; } = 32; + + /// + /// Gets or sets the maximum total payload bytes in one dissemination batch. + /// + public int MaxBatchBytes { get; set; } = 1024 * 1024; + + /// + /// Gets or sets the maximum number of items in one dissemination batch. + /// + public int MaxBatchItems { get; set; } = 8 * 1024; + + /// + /// Gets or sets overlay-specific dissemination options. + /// + public DisseminationOverlayOptions Overlay { get; set; } = new(); +} + +/// +/// Options for the dissemination overlay. +/// +public sealed class DisseminationOverlayOptions +{ + /// + /// Gets or sets the code-configured fanout selector. + /// + /// + /// The argument is the current member count for the selected dissemination topology. + /// The selector must return the same fanout on every silo for a given member count so that forwarding + /// routes agree across the cluster. + /// When this value is , , , + /// and are used to derive a fanout factor. + /// + public Func? FanOutFactor { get; set; } + + /// + /// Gets or sets the target number of tree hops used by the bindable fanout selector. + /// + public int TargetHopCount { get; set; } = 2; + + /// + /// Gets or sets the minimum fanout factor used by the bindable fanout selector. + /// + public int MinFanOutFactor { get; set; } = 4; + + /// + /// Gets or sets the maximum fanout factor used by the bindable fanout selector. + /// + public int MaxFanOutFactor { get; set; } = 32; + + /// + /// Gets or sets the interval between anti-entropy repair rounds. + /// + /// The interval is 5 seconds by default and must be between 1 millisecond and approximately 49.7 days. + public TimeSpan AntiEntropyInterval { get; set; } = TimeSpan.FromSeconds(5); + + /// + /// Gets or sets the number of peers contacted during each anti-entropy repair round. + /// + /// + /// This also bounds concurrent local repair attempts across overlapping rounds. Each attempt releases + /// capacity when its local wait completes, including timeout or cancellation. + /// + public int AntiEntropyPeerCount { get; set; } = 3; + + internal int GetFanOutFactor(int memberCount) + { + var count = Math.Max(1, memberCount); + var selectedFanOut = FanOutFactor?.Invoke(count) ?? GetConfiguredFanOutFactor(count); + return Math.Clamp(selectedFanOut, 1, count); + } + + internal int GetConfiguredFanOutFactor(int memberCount) + { + var count = Math.Max(1, memberCount); + var targetHopCount = Math.Max(1, TargetHopCount); + var scaled = targetHopCount switch + { + 1 => count, + 2 => Math.Sqrt(count), + 3 => Math.Cbrt(count), + _ => Math.Pow(count, 1d / targetHopCount), + }; + var min = Math.Max(1, MinFanOutFactor); + var max = Math.Max(min, MaxFanOutFactor); + return (int)Math.Ceiling(Math.Max(min, Math.Min(scaled, max))); + } +} + +/// +/// Options for a dissemination namespace. +/// +public sealed class DisseminationNamespaceOptions +{ + /// + /// Gets or sets a value indicating whether this namespace is enabled. + /// + /// by default. + public bool Enabled { get; set; } + + /// + /// Gets or sets the dissemination priority for this namespace. + /// + /// + /// namespaces bypass the coalescing window and are placed ahead of + /// lower-priority namespaces within each per-peer batch, so their updates are disseminated as quickly as + /// possible. is not applied to high-priority namespaces. + /// + public DisseminationPriority Priority { get; set; } = DisseminationPriority.Normal; + + /// + /// Gets or sets the hard maximum number of distinct pending keys retained for this namespace by each peer. + /// + /// + /// The bound applies independently to each namespace in each peer pump. A notification for a key which is + /// already retained updates that key at the limit, while a new distinct key is rejected with diagnostics until + /// acknowledged delivery or membership pruning releases capacity. Values are materialized from namespace state + /// only when they are sent, so this bounds retained identities rather than serialized batch bytes. + /// + public int MaxPendingItemCount { get; set; } = 1024; + + /// + /// Gets or sets the maximum delay for namespace coalescing. + /// + /// + /// Per-peer batches can contain values from multiple namespaces and use the shortest configured delay among + /// enabled namespaces. High-priority namespaces (see ) do not coalesce and are excluded + /// from this calculation. + /// + /// The delay is 100 milliseconds by default and must be between 1 millisecond and approximately 49.7 days. + public TimeSpan MaxCoalescingDelay { get; set; } = TimeSpan.FromMilliseconds(100); + + /// + /// Gets or sets how long a namespace value remains useful. + /// + /// + /// The lifetime bounds transport and application independently for each dissemination hop. Forwarding + /// re-materializes the value with a new hop lifetime. + /// + /// The lifetime is 30 seconds by default and must be greater than zero. + public TimeSpan StaleItemTtl { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Gets or sets the expected cadence for updates in this namespace. + /// + /// + /// Anti-entropy requests omit keys whose version advanced within this interval. Duplicate values do not + /// postpone repair probes. + /// + public TimeSpan ExpectedUpdateCadence { get; set; } = TimeSpan.FromSeconds(10); + + /// + /// Gets or sets the maximum serialized payload size for this namespace. + /// + public int MaxPayloadBytes { get; set; } = 1024 * 1024; +} + +/// +/// Describes how urgently a dissemination namespace's updates are broadcast relative to other namespaces. +/// +public enum DisseminationPriority +{ + /// + /// Updates are coalesced within the namespace's + /// and are sent after any higher-priority namespaces. + /// + Normal, + + /// + /// Updates bypass the coalescing window, are sent immediately, and are placed ahead of normal-priority + /// namespaces within each per-peer batch. + /// + High, +} diff --git a/src/Orleans.Core/OrleansContracts.txt b/src/Orleans.Core/OrleansContracts.txt index 2536412af67..02e96509eba 100644 --- a/src/Orleans.Core/OrleansContracts.txt +++ b/src/Orleans.Core/OrleansContracts.txt @@ -57,6 +57,10 @@ interface [GrainInterfaceType("Orleans.Runtime.IClusterManifestSystemTarget")] O interface [GrainInterfaceType("Orleans.Runtime.IDeploymentLoadPublisher")] Orleans.Runtime.IDeploymentLoadPublisher [Version(0)] C5255F0C: UpdateRuntimeStatistics(Orleans.Runtime.SiloAddress, Orleans.Runtime.SiloRuntimeStatistics, System.Threading.CancellationToken) -> Task +interface [GrainInterfaceType("Orleans.Runtime.IDisseminationSystemTarget")] Orleans.Runtime.IDisseminationSystemTarget [Version(0)] + EF58CB3D: ExchangeAntiEntropy(Orleans.Runtime.DisseminationAntiEntropyRequest, System.Threading.CancellationToken) -> Task + 017AA907: PushBroadcast(Orleans.Runtime.DisseminationBroadcastBatch, System.Threading.CancellationToken) -> Task + interface [GrainInterfaceType("Orleans.Runtime.IGrainCallCancellationExtension")] Orleans.Runtime.IGrainCallCancellationExtension [Version(0)] FA239824: CancelRequestAsync(Orleans.Runtime.GrainId, Orleans.Runtime.CorrelationId, System.Threading.CancellationToken) -> ValueTask diff --git a/src/Orleans.Core/Runtime/Constants.cs b/src/Orleans.Core/Runtime/Constants.cs index 8efb4156c60..d84129d9e8e 100644 --- a/src/Orleans.Core/Runtime/Constants.cs +++ b/src/Orleans.Core/Runtime/Constants.cs @@ -17,6 +17,7 @@ internal static class Constants public static readonly GrainType MembershipServiceType = SystemTargetGrainId.CreateGrainType("clustering"); public static readonly GrainType SystemMembershipTableType = SystemTargetGrainId.CreateGrainType("clustering.dev"); public static readonly GrainType DeploymentLoadPublisherSystemTargetType = SystemTargetGrainId.CreateGrainType("load-publisher"); + public static readonly GrainType DisseminationSystemTargetType = SystemTargetGrainId.CreateGrainType("dissemination"); public static readonly GrainType TestHooksSystemTargetType = SystemTargetGrainId.CreateGrainType("test.hooks"); public static readonly GrainType TransactionAgentSystemTargetType = SystemTargetGrainId.CreateGrainType("txn.agent"); public static readonly GrainType StreamProviderManagerAgentSystemTargetType = SystemTargetGrainId.CreateGrainType("stream.provider-manager"); @@ -46,6 +47,7 @@ internal static class Constants {CatalogType,"Catalog"}, {MembershipServiceType,"MembershipService"}, {DeploymentLoadPublisherSystemTargetType, "DeploymentLoadPublisherSystemTarget"}, + {DisseminationSystemTargetType, "DisseminationSystemTarget"}, {StreamProviderManagerAgentSystemTargetType,"StreamProviderManagerAgent"}, {TestHooksSystemTargetType,"TestHooksSystemTargetType"}, {TransactionAgentSystemTargetType,"TransactionAgentSystemTarget"}, diff --git a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs new file mode 100644 index 00000000000..5d4177cb2c7 --- /dev/null +++ b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs @@ -0,0 +1,283 @@ +namespace Orleans.Runtime; + +internal interface IDisseminationSystemTarget : ISystemTarget +{ + // The response carries receiver versions, which are the evidence used to sequence later repairs. + [Alias("017AA907")] + Task PushBroadcast(DisseminationBroadcastBatch batch, CancellationToken cancellationToken); + + [Alias("EF58CB3D")] + Task ExchangeAntiEntropy(DisseminationAntiEntropyRequest request, CancellationToken cancellationToken); +} + +[GenerateSerializer, Immutable] +internal readonly struct DisseminationNamespace : IEquatable, IComparable, IComparable, ISpanFormattable +{ + [Id(0)] + private readonly string? _value; + + public DisseminationNamespace(string value) + { + ArgumentException.ThrowIfNullOrEmpty(value); + _value = value; + } + + public string Value => _value ?? throw new InvalidOperationException($"A default {nameof(DisseminationNamespace)} value is invalid."); + + public static implicit operator DisseminationNamespace(string value) => new(value); + + public static implicit operator string(DisseminationNamespace value) => value.Value; + + public static bool operator ==(DisseminationNamespace left, DisseminationNamespace right) => left.Equals(right); + + public static bool operator !=(DisseminationNamespace left, DisseminationNamespace right) => !left.Equals(right); + + public bool Equals(DisseminationNamespace other) => string.Equals(_value, other._value, StringComparison.Ordinal); + + public override bool Equals(object? obj) => obj is DisseminationNamespace other && Equals(other); + + public override int GetHashCode() => _value is null ? 0 : StringComparer.Ordinal.GetHashCode(_value); + + public int CompareTo(DisseminationNamespace other) => string.Compare(_value, other._value, StringComparison.Ordinal); + + public int CompareTo(object? obj) + { + if (obj is null) + { + return 1; + } + + return obj is DisseminationNamespace other + ? CompareTo(other) + : throw new ArgumentException($"Object must be of type {nameof(DisseminationNamespace)}.", nameof(obj)); + } + + public override string ToString() => _value ?? string.Empty; + + public string ToString(string? format, IFormatProvider? formatProvider) => _value ?? string.Empty; + + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? formatProvider) + { + var value = _value.AsSpan(); + if (value.TryCopyTo(destination)) + { + charsWritten = value.Length; + return true; + } + + charsWritten = 0; + return false; + } +} + +[GenerateSerializer, Immutable] +internal readonly struct DisseminationKey(object? value) : IEquatable, IComparable, IComparable, ISpanFormattable +{ + public static readonly DisseminationKey Default = default; + + [Id(0)] + public readonly object? Value = value; + + public static implicit operator DisseminationKey(SiloAddress? value) => new(value); + + public static implicit operator DisseminationKey(string? value) => new(value); + + public static bool operator ==(DisseminationKey left, DisseminationKey right) => left.Equals(right); + + public static bool operator !=(DisseminationKey left, DisseminationKey right) => !left.Equals(right); + + public bool Equals(DisseminationKey other) => Equals(Value, other.Value); + + public override bool Equals(object? obj) => obj is DisseminationKey other && Equals(other); + + public override int GetHashCode() => Value?.GetHashCode() ?? 0; + + public int CompareTo(DisseminationKey other) => Compare(Value, other.Value); + + public int CompareTo(object? obj) + { + if (obj is null) + { + return 1; + } + + return obj is DisseminationKey other + ? CompareTo(other) + : throw new ArgumentException($"Object must be of type {nameof(DisseminationKey)}.", nameof(obj)); + } + + public override string ToString() => ToString(null, null); + + public string ToString(string? format, IFormatProvider? formatProvider) => Value switch + { + null => string.Empty, + IFormattable formattable => formattable.ToString(format, formatProvider), + _ => Value.ToString() ?? string.Empty, + }; + + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? formatProvider) + { + if (Value is null) + { + charsWritten = 0; + return true; + } + + if (Value is ISpanFormattable spanFormattable) + { + return spanFormattable.TryFormat(destination, out charsWritten, format, formatProvider); + } + + var text = ToString(format.IsEmpty ? null : format.ToString(), formatProvider); + if (text.AsSpan().TryCopyTo(destination)) + { + charsWritten = text.Length; + return true; + } + + charsWritten = 0; + return false; + } + + private static int Compare(object? left, object? right) + { + if (ReferenceEquals(left, right)) + { + return 0; + } + + if (left is null) + { + return -1; + } + + if (right is null) + { + return 1; + } + + if (Equals(left, right)) + { + return 0; + } + + return left switch + { + string leftString when right is string rightString => StringComparer.Ordinal.Compare(leftString, rightString), + SiloAddress leftSilo when right is SiloAddress rightSilo => leftSilo.CompareTo(rightSilo), + _ => Comparer.Default.Compare(left, right), + }; + } +} + +[GenerateSerializer, Immutable] +internal readonly struct DigestEntry +{ + public DigestEntry(DisseminationKey key, long version, long fingerprint = 0) + { + Key = key; + Version = version; + Fingerprint = fingerprint; + } + + [Id(0)] + public DisseminationKey Key { get; } + + [Id(1)] + public long Version { get; } + + // Fingerprints distinguish meaningful same-version state, such as membership liveness. + [Id(2)] + public long Fingerprint { get; } +} + +// FromVersion zero denotes a full value which can replace any baseline; nonzero ranges must form a chain. +[GenerateSerializer, Immutable] +internal readonly struct DisseminationValue +{ + public DisseminationValue(DisseminationKey key, long fromVersion, long toVersion, ReadOnlyMemory payload) + { + Key = key; + FromVersion = fromVersion; + ToVersion = toVersion; + Payload = payload; + } + + [Id(0)] + public DisseminationKey Key { get; } + + [Id(1)] + public long FromVersion { get; } + + [Id(2)] + public long ToVersion { get; } + + [Id(3)] + public ReadOnlyMemory Payload { get; } +} + +// Lifetime is hop-local so forwarding re-materializes both the payload and its delivery window. +[GenerateSerializer, Immutable] +internal sealed class DisseminationBroadcastValue +{ + [Id(0)] + public DisseminationValue Value { get; init; } + + [Id(1)] + public TimeSpan TimeToLive { get; init; } +} + +// Sender is the immediate hop, not the original publisher. +[GenerateSerializer, Immutable] +internal sealed class DisseminationBroadcastBatch +{ + [Id(0)] + public required SiloAddress Sender { get; init; } + + [Id(1)] + public Dictionary> Values { get; init; } = []; +} + +// Acknowledgments report the versions the receiver actually holds after processing the batch. +[GenerateSerializer, Immutable] +internal sealed class DisseminationBroadcastResponse +{ + [Id(0)] + public Dictionary> Acknowledgments { get; init; } = []; + + [Id(1)] + public List UnsupportedNamespaces { get; init; } = []; +} + +[GenerateSerializer, Immutable] +internal sealed class DisseminationAntiEntropyRequest +{ + [Id(0)] + public Dictionary> Digests { get; init; } = []; + + [Id(1)] + public required SiloAddress Sender { get; init; } + + [Id(2)] + public List SupportedNamespaces { get; init; } = []; +} + +[GenerateSerializer, Immutable] +internal sealed class DisseminationAntiEntropyResponse +{ + [Id(0)] + public required SiloAddress Sender { get; init; } + + [Id(1)] + public Dictionary> Values { get; init; } = []; + + // Truncation means at least one valid repair remains for a later round. + [Id(2)] + public bool Truncated { get; init; } + + [Id(3)] + public List SupportedNamespaces { get; init; } = []; + + [Id(4)] + public List UnsupportedNamespaces { get; init; } = []; +} diff --git a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs index 7a8b9824fb8..f4e08c1efbb 100644 --- a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs @@ -16,5 +16,18 @@ public class DeploymentLoadPublisherOptions /// The default value for . /// public static readonly TimeSpan DEFAULT_DEPLOYMENT_LOAD_PUBLISHER_REFRESH_TIME = TimeSpan.FromSeconds(1); + + /// + /// Gets or sets dissemination options for deployment load statistics. + /// + /// + /// When enabled, dissemination replaces per-refresh direct fan-out only for active peers which have + /// confirmed support for this namespace. Confirmation remains valid for that silo generation until the peer + /// explicitly rejects the namespace or leaves the eligible membership set. During rolling or mixed-version + /// operation, active peers without confirmation continue to receive direct publications. If dissemination is + /// unavailable, declines the update, throws, or cannot accept it within the refresh interval, direct + /// publication targets all active peers. + /// + public DisseminationNamespaceOptions Dissemination { get; set; } = new() { ExpectedUpdateCadence = TimeSpan.FromSeconds(5) }; } } diff --git a/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs b/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs new file mode 100644 index 00000000000..4e364923cbc --- /dev/null +++ b/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs @@ -0,0 +1,110 @@ +using System; +using Microsoft.Extensions.Options; + +namespace Orleans.Configuration; + +internal sealed class DisseminationOptionsValidator : IValidateOptions +{ + private static readonly TimeSpan MaxPeriodicTimerPeriod = TimeSpan.FromMilliseconds(uint.MaxValue - 1); + + public ValidateOptionsResult Validate(string? name, DisseminationOptions options) + { + if (options.MaxConcurrentSends <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOptions.MaxConcurrentSends)} must be greater than 0."); + } + + if (options.MaxBatchBytes <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOptions.MaxBatchBytes)} must be greater than 0."); + } + + if (options.MaxBatchItems <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOptions.MaxBatchItems)} must be greater than 0."); + } + + var overlay = options.Overlay; + if (overlay.TargetHopCount <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.TargetHopCount)} must be greater than 0."); + } + + if (overlay.MinFanOutFactor <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.MinFanOutFactor)} must be greater than 0."); + } + + if (overlay.MaxFanOutFactor < overlay.MinFanOutFactor) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.MaxFanOutFactor)} must be greater than or equal to {nameof(DisseminationOverlayOptions.MinFanOutFactor)}."); + } + + if (overlay.AntiEntropyInterval < TimeSpan.FromMilliseconds(1) + || overlay.AntiEntropyInterval > MaxPeriodicTimerPeriod) + { + return ValidateOptionsResult.Fail( + $"{nameof(DisseminationOverlayOptions.AntiEntropyInterval)} must be between 1 millisecond and {MaxPeriodicTimerPeriod}."); + } + + if (overlay.AntiEntropyPeerCount <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.AntiEntropyPeerCount)} must be greater than 0."); + } + + return ValidateOptionsResult.Success; + } +} + +internal sealed class DisseminationNamespaceOptionsValidator +{ + private static readonly TimeSpan MaxTimerPeriod = TimeSpan.FromMilliseconds(uint.MaxValue - 1); + + public static ValidateOptionsResult Validate(string owner, DisseminationNamespaceOptions options) + { + if (options.MaxPendingItemCount <= 0) + { + return ValidateOptionsResult.Fail($"{owner}.{nameof(DisseminationNamespaceOptions.MaxPendingItemCount)} must be greater than 0."); + } + + if (options.MaxCoalescingDelay < TimeSpan.FromMilliseconds(1) + || options.MaxCoalescingDelay > MaxTimerPeriod) + { + return ValidateOptionsResult.Fail( + $"{owner}.{nameof(DisseminationNamespaceOptions.MaxCoalescingDelay)} must be between 1 millisecond and {MaxTimerPeriod}."); + } + + if (options.StaleItemTtl <= TimeSpan.Zero) + { + return ValidateOptionsResult.Fail($"{owner}.{nameof(DisseminationNamespaceOptions.StaleItemTtl)} must be greater than 0."); + } + + if (options.ExpectedUpdateCadence <= TimeSpan.Zero) + { + return ValidateOptionsResult.Fail($"{owner}.{nameof(DisseminationNamespaceOptions.ExpectedUpdateCadence)} must be greater than 0."); + } + + if (options.MaxPayloadBytes <= 0) + { + return ValidateOptionsResult.Fail($"{owner}.{nameof(DisseminationNamespaceOptions.MaxPayloadBytes)} must be greater than 0."); + } + + return ValidateOptionsResult.Success; + } +} + +internal sealed class DeploymentLoadPublisherOptionsValidator : IValidateOptions +{ + public ValidateOptionsResult Validate(string? name, DeploymentLoadPublisherOptions options) => + DisseminationNamespaceOptionsValidator.Validate( + $"{nameof(DeploymentLoadPublisherOptions)}.{nameof(DeploymentLoadPublisherOptions.Dissemination)}", + options.Dissemination); +} + +internal sealed class ClusterMembershipOptionsDisseminationValidator : IValidateOptions +{ + public ValidateOptionsResult Validate(string? name, ClusterMembershipOptions options) => + DisseminationNamespaceOptionsValidator.Validate( + $"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.Dissemination)}", + options.Dissemination); +} diff --git a/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs new file mode 100644 index 00000000000..b8a24bb0b93 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs @@ -0,0 +1,128 @@ +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Serialization; + +namespace Orleans.Runtime.Dissemination; + +// A newer load sample supersedes every older sample, so this namespace never needs a version chain. +internal sealed class DeploymentLoadStatisticsDisseminationNamespace( + DeploymentLoadPublisher deploymentLoadPublisher, + IOptionsMonitor options, + Serializer serializer) : IDisseminationNamespace +{ + private readonly object _cacheLock = new(); + private readonly Dictionary _cachedValues = []; + + public DisseminationNamespace Name => DisseminationNamespaceNames.DeploymentLoad; + + public DisseminationMembershipScope MembershipScope => DisseminationMembershipScope.ActiveMembers; + + public DisseminationNamespaceOptions Options => options.CurrentValue.Dissemination; + + public DisseminationValue CreateValue(SiloAddress origin, SiloRuntimeStatistics statistics) + { + lock (_cacheLock) + { + // Reuse serialization until this silo publishes a new timestamp. + if (_cachedValues.TryGetValue(origin, out var cached) + && cached.ToVersion == statistics.DateTime.Ticks) + { + return cached; + } + + var result = new DisseminationValue( + origin, + fromVersion: 0, + toVersion: statistics.DateTime.Ticks, + serializer.SerializeToArray(statistics)); + _cachedValues[origin] = result; + return result; + } + } + + public IEnumerable Digests + { + get + { + var activeSilos = deploymentLoadPublisher.GetActiveSilosForStatisticsDigest(); + PruneCache(activeSilos); + foreach (var siloAddress in activeSilos) + { + yield return new DigestEntry(siloAddress, GetVersion(siloAddress)); + } + } + } + + public long GetVersion(DisseminationKey key) => + key.Value is SiloAddress siloAddress + && deploymentLoadPublisher.PeriodicStatistics.TryGetValue(siloAddress, out var statistics) + && !deploymentLoadPublisher.IsRuntimeStatisticsObsolete(siloAddress, statistics.DateTime.Ticks) + ? statistics.DateTime.Ticks + : 0; + + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) + { + if (request.Key.Value is not SiloAddress siloAddress + || !deploymentLoadPublisher.PeriodicStatistics.TryGetValue(siloAddress, out var statistics) + || deploymentLoadPublisher.IsRuntimeStatisticsObsolete(siloAddress, statistics.DateTime.Ticks)) + { + return DisseminationRepairResult.Unavailable(version: 0); + } + + var version = statistics.DateTime.Ticks; + if (request.ToVersion is { } targetVersion && targetVersion != version) + { + return DisseminationRepairResult.Unavailable(version); + } + + if (request.FromVersion is { } peerVersion && peerVersion >= version) + { + return DisseminationRepairResult.Current(version); + } + + if (request.MaxItemCount <= 0) + { + return DisseminationRepairResult.InsufficientCapacity(version); + } + + // Every repair is a full value from zero, making the peer's exact baseline irrelevant. + var value = CreateValue(siloAddress, statistics); + return value.Payload.Length <= request.MaxPayloadBytes + && value.Payload.Length <= request.MaxBatchBytes + ? DisseminationRepairResult.Produced(version, [value]) + : DisseminationRepairResult.InsufficientCapacity(version); + } + + public ValueTask ApplyValueAsync( + DisseminationValue value, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (value.Key.Value is not SiloAddress siloAddress) + { + return ValueTask.FromResult(DisseminationApplyResult.Rejected); + } + + if (value.FromVersion != 0 + || serializer.Deserialize(value.Payload) is not { } statistics + || value.ToVersion != statistics.DateTime.Ticks) + { + return ValueTask.FromResult(DisseminationApplyResult.Rejected); + } + + cancellationToken.ThrowIfCancellationRequested(); + return new(deploymentLoadPublisher.ApplyDisseminatedRuntimeStatisticsAsync(siloAddress, statistics, cancellationToken)); + } + + private void PruneCache(IReadOnlyCollection activeSilos) + { + var activeSiloSet = activeSilos.ToHashSet(); + lock (_cacheLock) + { + foreach (var key in _cachedValues.Keys.Where(key => !activeSiloSet.Contains(key)).ToArray()) + { + _cachedValues.Remove(key); + } + } + } +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationApplyResult.cs b/src/Orleans.Runtime/Dissemination/DisseminationApplyResult.cs new file mode 100644 index 00000000000..d0f1e741642 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationApplyResult.cs @@ -0,0 +1,9 @@ +namespace Orleans.Runtime.Dissemination; + +internal enum DisseminationApplyResult +{ + Applied, + Duplicate, + Obsolete, + Rejected, +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs new file mode 100644 index 00000000000..e57085c6aed --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs @@ -0,0 +1,1595 @@ +using System.Collections.Frozen; +using System.Runtime.InteropServices; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime.Internal; + +namespace Orleans.Runtime.Dissemination; + +// The queue remembers what each peer has acknowledged. +// Payloads are materialized from namespace state only when a peer is ready to send. +internal sealed partial class DisseminationBroadcastQueue +{ + private static readonly TimeSpan DefaultSchedulingDelay = TimeSpan.FromMilliseconds(100); + private static readonly TimeSpan MaxTransportLifetime = TimeSpan.FromMilliseconds(uint.MaxValue - 1); + private readonly TimeProvider _timeProvider; + private readonly SiloAddress _localSilo; + private readonly IInternalGrainFactory _grainFactory; + private readonly IOptionsMonitor _options; + private readonly IDisseminationNamespace[] _disseminationNamespaces; + private readonly FrozenDictionary _namespaces; + private readonly ILogger _logger; + private readonly Action? _responseObserver; + private readonly object _lock = new(); + private readonly Dictionary _peers = []; + private readonly DisseminationSendGate _sendGate; + private bool _stopped; + + public DisseminationBroadcastQueue( + TimeProvider timeProvider, + SiloAddress localSilo, + IInternalGrainFactory grainFactory, + IOptionsMonitor options, + IEnumerable disseminationNamespaces, + ILogger logger, + Action? responseObserver = null) + { + _timeProvider = timeProvider; + _localSilo = localSilo; + _grainFactory = grainFactory; + _options = options; + _disseminationNamespaces = [.. disseminationNamespaces]; + _namespaces = _disseminationNamespaces.ToFrozenDictionary(static ns => ns.Name); + _logger = logger; + _responseObserver = responseObserver; + _sendGate = new(Math.Max(1, options.CurrentValue.MaxConcurrentSends)); + } + + public bool Notify( + SiloAddress peer, + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + bool force = true) + { + PeerQueuePump pump; + lock (_lock) + { + if (_stopped) + { + return false; + } + + pump = GetOrCreatePeerUnsafe(peer); + } + + // Notification diagnostics run outside the queue lock and can reenter the queue. + var version = disseminationNamespace.GetVersion(key); + return pump.Notify(disseminationNamespace, key, version, force); + } + + public void ObservePeerVersion( + SiloAddress peer, + DisseminationNamespace namespaceName, + DisseminationKey key, + long version) + { + if (version < 0 + || !_namespaces.TryGetValue(namespaceName, out var disseminationNamespace) + || !disseminationNamespace.Options.Enabled) + { + return; + } + + // Passive evidence only sharpens an existing ledger; it must not allocate a timer for a peer with no outbound work. + lock (_lock) + { + if (_stopped || !_peers.TryGetValue(peer, out var pending)) + { + return; + } + + pending.ObservePeerVersion(disseminationNamespace, key, version); + } + } + + public async Task FlushPendingBroadcast(CancellationToken cancellationToken) + { + List peers; + lock (_lock) + { + peers = [.. _peers.Values.OrderBy(static peer => peer.Peer)]; + } + + await Task.WhenAll(peers.Select(peer => peer.FlushAsync(cancellationToken).AsTask())); + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + List peers; + lock (_lock) + { + if (_stopped) + { + return; + } + + _stopped = true; + peers = [.. _peers.Values]; + _peers.Clear(); + } + + try + { + await Task.WhenAll(peers.Select(peer => peer.StopAsync(drain: true, cancellationToken).AsTask())); + } + finally + { + _sendGate.Stop(); + } + } + + public async Task Prune( + DisseminationMembershipSnapshots membershipSnapshots, + CancellationToken cancellationToken) + { + // Namespace digests are the authoritative inventory, so clean ledger entries can disappear with their keys. + var activeKeys = new Dictionary>(); + foreach (var disseminationNamespace in _disseminationNamespaces) + { + if (disseminationNamespace.Options.Enabled) + { + activeKeys[disseminationNamespace.Name] = disseminationNamespace.Digests + .Select(static entry => entry.Key) + .ToHashSet(); + } + } + + List? removedPeers = null; + List retainedPeers; + lock (_lock) + { + retainedPeers = new(_peers.Count); + foreach (var (peer, pending) in _peers) + { + if (!_localSilo.Equals(peer) && !membershipSnapshots.AllMembers.ContainsMember(peer)) + { + (removedPeers ??= []).Add(pending); + } + else + { + retainedPeers.Add(pending); + } + } + + if (removedPeers is not null) + { + foreach (var pending in removedPeers) + { + _peers.Remove(pending.Peer); + } + } + } + + foreach (var peer in retainedPeers) + { + peer.PruneKeys(activeKeys, membershipSnapshots); + } + + if (removedPeers is not null) + { + await Task.WhenAll(removedPeers.Select(peer => peer.StopAsync(drain: false, cancellationToken).AsTask())); + } + } + + private PeerQueuePump GetOrCreatePeerUnsafe(SiloAddress peer) + { + if (!_peers.TryGetValue(peer, out var result)) + { + result = new(peer, this); + _peers.Add(peer, result); + } + + return result; + } + + private TimeSpan GetCoalescingDelay(TimeSpan namespaceDelay) + { + // A peer pump can mix namespaces, so its next wake honors the most urgent coalescing namespace. + // High-priority namespaces never coalesce, so they do not lower other namespaces' windows. + var result = namespaceDelay; + foreach (var disseminationNamespace in _disseminationNamespaces) + { + var namespaceOptions = disseminationNamespace.Options; + if (namespaceOptions.Enabled + && namespaceOptions.Priority != DisseminationPriority.High + && namespaceOptions.MaxCoalescingDelay < result) + { + result = namespaceOptions.MaxCoalescingDelay; + } + } + + return result == TimeSpan.MaxValue ? GetBoundedFallbackDelay() : result; + } + + private TimeSpan GetRetryDelay(int attempt) + { + // Retries begin at the normal coalescing cadence and back off no further than anti-entropy. + // High-priority namespaces do not coalesce, so they do not define the retry floor. + var floor = TimeSpan.MaxValue; + foreach (var disseminationNamespace in _disseminationNamespaces) + { + var namespaceOptions = disseminationNamespace.Options; + if (namespaceOptions.Enabled + && namespaceOptions.Priority != DisseminationPriority.High + && namespaceOptions.MaxCoalescingDelay < floor) + { + floor = namespaceOptions.MaxCoalescingDelay; + } + } + + if (floor == TimeSpan.MaxValue) + { + floor = GetBoundedFallbackDelay(); + } + + var cap = _options.CurrentValue.Overlay.AntiEntropyInterval; + if (floor > cap) + { + floor = cap; + } + + var multiplier = Math.Pow(2, Math.Min(Math.Max(0, attempt - 1), 20)); + return TimeSpan.FromTicks((long)Math.Min(cap.Ticks, floor.Ticks * multiplier)); + } + + private TimeSpan GetBoundedFallbackDelay() + { + var antiEntropyInterval = _options.CurrentValue.Overlay.AntiEntropyInterval; + return antiEntropyInterval < DefaultSchedulingDelay ? antiEntropyInterval : DefaultSchedulingDelay; + } + + private sealed class PeerQueuePump + { + private readonly DisseminationBroadcastQueue _owner; + private readonly Action _admissionQueued; + private readonly object _lock = new(); + private readonly CancellationTokenSource _shutdownCts = new(); + private readonly WakeTimer _flushTimer; + private readonly Task _flushTask; + private Dictionary _statesByNamespace = []; + private TaskCompletionSource _nextFlushCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + private TaskCompletionSource? _activeFlushCompletion; + private Exception? _pumpFailure; + private IDisseminationSystemTarget? _target; + // Notifications arriving during a send advance the epoch and remain dirty for the next pass. + private long _notificationEpoch; + private int _retryAttempt; + private bool _wakeScheduled; + private bool _stopping; + + public PeerQueuePump(SiloAddress peer, DisseminationBroadcastQueue owner) + { + Peer = peer; + _owner = owner; + _admissionQueued = EmitAdmissionQueued; + _flushTimer = new(owner._timeProvider); + using var _ = new ExecutionContextSuppressor(); + _flushTask = RunScheduledFlush(_shutdownCts.Token); + } + + public SiloAddress Peer { get; } + + private int DirtyCount { get; set; } + + public bool Notify(IDisseminationNamespace disseminationNamespace, DisseminationKey key, long version, bool force) + { + ScheduledFlush? scheduled = null; + var admissionRejected = false; + lock (_lock) + { + if (_stopping) + { + return false; + } + + if (_pumpFailure is { } pumpFailure) + { + throw new InvalidOperationException($"The dissemination broadcast pump for {Peer} has failed.", pumpFailure); + } + + var namespaceState = GetOrCreateNamespaceStateUnsafe(disseminationNamespace); + namespaceState.TryGetKey(key, out var keyState); + if (!force) + { + // Duplicate deliveries can repair a new child, but cannot perpetuate a cycle between + // skewed trees. Applied same-version state (membership liveness) forces a notification. + if (keyState is { Dirty: true } or { InFlight: true } && keyState.NotificationVersion >= version) + { + return true; + } + + var knownVersion = keyState?.KnownVersion; + if (knownVersion is null && namespaceState.KnownVersions.TryGetValue(key, out var recordedVersion)) + { + knownVersion = recordedVersion; + } + + if (knownVersion >= version) + { + return true; + } + } + + if (keyState is null && namespaceState.Keys.Count >= disseminationNamespace.Options.MaxPendingItemCount) + { + admissionRejected = true; + } + else + { + keyState ??= namespaceState.AddKey(key); + // A notification is only a wake-up. The namespace will choose the latest repair when this key is drained. + keyState.NotificationVersion = version; + keyState.NotificationGeneration++; + _notificationEpoch++; + var wasRetrying = _retryAttempt > 0; + _retryAttempt = 0; + var wasEmpty = DirtyCount == 0; + MarkDirtyUnsafe(namespaceState, keyState); + + // Filled queues flush immediately; otherwise one timer coalesces all pending namespaces. + var currentOptions = _owner._options.CurrentValue; + if (disseminationNamespace.Options.Priority == DisseminationPriority.High) + { + // High-priority updates skip the coalescing window and pull any pending flush forward. + _flushTimer.Change(TimeSpan.Zero); + _wakeScheduled = true; + scheduled = new(DisseminationBroadcastScheduleReason.Priority, TimeSpan.Zero, _retryAttempt, _notificationEpoch); + } + else if (DirtyCount >= currentOptions.MaxBatchItems + || namespaceState.DirtyCount >= disseminationNamespace.Options.MaxPendingItemCount) + { + _flushTimer.Change(TimeSpan.Zero); + _wakeScheduled = true; + scheduled = new(DisseminationBroadcastScheduleReason.Immediate, TimeSpan.Zero, _retryAttempt, _notificationEpoch); + } + else if (wasEmpty || wasRetrying || !_wakeScheduled) + { + var delay = _owner.GetCoalescingDelay(disseminationNamespace.Options.MaxCoalescingDelay); + _flushTimer.Change(delay); + _wakeScheduled = true; + scheduled = new(DisseminationBroadcastScheduleReason.Coalesce, delay, _retryAttempt, _notificationEpoch); + } + } + } + + if (admissionRejected) + { + DisseminationInstruments.OnQueueAdmissionRejected(disseminationNamespace.Name); + try + { + DisseminationEvents.EmitQueueAdmissionRejected( + _owner._localSilo, + Peer, + disseminationNamespace.Name, + disseminationNamespace.Options.MaxPendingItemCount); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + } + + EmitScheduled(scheduled); + return !admissionRejected; + } + + private void EmitScheduled(ScheduledFlush? scheduled) + { + if (scheduled is { } info) + { + DisseminationInstruments.OnBroadcastScheduled(info.Reason); + try + { + DisseminationEvents.EmitBroadcastScheduled(_owner._localSilo, Peer, info.Reason, info.DueTime, info.Attempt, info.Epoch); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + } + } + + private void EmitAdmissionQueued() + { + try + { + DisseminationEvents.EmitSendGate(_owner._localSilo, Peer, kind: "broadcast", stage: "queued"); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + } + + // Captures a scheduling decision made under the lock so the diagnostic can be emitted after the lock is released. + private readonly record struct ScheduledFlush( + DisseminationBroadcastScheduleReason Reason, + TimeSpan DueTime, + int Attempt, + long Epoch); + + public void ObservePeerVersion( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version) + { + lock (_lock) + { + if (_stopping) + { + return; + } + + // Peer knowledge is evidence-driven and monotonic across acknowledgments, pushes, and anti-entropy. + var namespaceState = GetOrCreateNamespaceStateUnsafe(disseminationNamespace); + if (namespaceState.TryGetKey(key, out var keyState) && keyState is not null) + { + if (keyState.KnownVersion is not { } knownVersion || version > knownVersion) + { + keyState.KnownVersion = version; + } + } + else + { + namespaceState.ObserveKnownVersion(key, version); + } + } + } + + public void PruneKeys( + Dictionary> activeKeys, + DisseminationMembershipSnapshots membershipSnapshots) + { + TaskCompletionSource? droppedFlushCompletion = null; + var droppedDirtyCount = 0; + lock (_lock) + { + foreach (var (namespaceName, namespaceState) in _statesByNamespace.ToArray()) + { + if (!membershipSnapshots.GetSnapshot(namespaceState.Namespace.MembershipScope).ContainsMember(Peer)) + { + droppedDirtyCount += namespaceState.DirtyCount; + DirtyCount -= namespaceState.DirtyCount; + _statesByNamespace.Remove(namespaceName); + continue; + } + + activeKeys.TryGetValue(namespaceName, out var namespaceKeys); + foreach (var (key, keyState) in namespaceState.Keys.ToArray()) + { + if (!keyState.Dirty + && !keyState.InFlight + && (namespaceKeys is null || !namespaceKeys.Contains(key))) + { + namespaceState.Keys.Remove(key); + } + } + + namespaceState.PruneKnownVersions(namespaceKeys); + if (namespaceState.Keys.Count == 0 && namespaceState.KnownVersions.Count == 0) + { + _statesByNamespace.Remove(namespaceName); + } + } + + if (droppedDirtyCount > 0 && DirtyCount == 0 && !_nextFlushCompletion.Task.IsCompleted) + { + droppedFlushCompletion = _nextFlushCompletion; + _nextFlushCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + } + + droppedFlushCompletion?.TrySetResult(); + } + + public async ValueTask FlushAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Task? flushCompletion; + var wake = false; + lock (_lock) + { + if (_pumpFailure is { } pumpFailure) + { + throw new InvalidOperationException($"The dissemination broadcast pump for {Peer} has failed.", pumpFailure); + } + + if (DirtyCount > 0) + { + flushCompletion = _nextFlushCompletion.Task; + wake = true; + _wakeScheduled = true; + } + else + { + flushCompletion = _activeFlushCompletion?.Task; + } + } + + if (flushCompletion is null) + { + return; + } + + if (wake) + { + _flushTimer.Wake(); + } + + flushCompletion.Ignore(); + await flushCompletion.WaitAsync(cancellationToken); + } + + public async ValueTask StopAsync(bool drain, CancellationToken cancellationToken) + { + Task? flushCompletion; + TaskCompletionSource? droppedFlushCompletion = null; + var wake = false; + var alreadyStopping = false; + lock (_lock) + { + if (_stopping) + { + alreadyStopping = true; + flushCompletion = null; + } + else + { + _stopping = true; + if (drain) + { + if (_pumpFailure is { } pumpFailure) + { + flushCompletion = Task.FromException(pumpFailure); + } + else if (DirtyCount > 0) + { + flushCompletion = _nextFlushCompletion.Task; + wake = true; + _wakeScheduled = true; + } + else + { + flushCompletion = _activeFlushCompletion?.Task; + } + } + else + { + droppedFlushCompletion = _nextFlushCompletion; + _nextFlushCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + ClearPendingUnsafe(); + flushCompletion = null; + } + } + } + + if (alreadyStopping) + { + await _flushTask.WaitAsync(cancellationToken); + return; + } + + droppedFlushCompletion?.TrySetResult(); + try + { + if (wake) + { + _flushTimer.Wake(); + } + + if (flushCompletion is not null) + { + flushCompletion.Ignore(); + await flushCompletion.WaitAsync(cancellationToken); + } + } + finally + { + if (!drain || cancellationToken.IsCancellationRequested) + { + await _shutdownCts.CancelAsync(); + } + + _flushTimer.Dispose(); + try + { + await _flushTask; + } + catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested) + { + } + + _shutdownCts.Dispose(); + } + } + + private async Task RunScheduledFlush(CancellationToken cancellationToken) + { + try + { + while (await _flushTimer.WaitAsync(cancellationToken)) + { + try + { + await RunScheduledFlushIteration(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + if (!TryRecoverFromUnexpectedIterationFailure(exception)) + { + return; + } + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + } + + private async Task RunScheduledFlushIteration(CancellationToken cancellationToken) + { + TaskCompletionSource flushCompletion; + List work; + long notificationEpoch; + // Move one dirty generation to in-flight atomically; a concurrent notification can mark it dirty again. + lock (_lock) + { + _wakeScheduled = false; + if (DirtyCount == 0) + { + return; + } + + flushCompletion = _nextFlushCompletion; + _nextFlushCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + _activeFlushCompletion = flushCompletion; + notificationEpoch = _notificationEpoch; + work = DrainDirtyUnsafe(); + } + + var result = SendWorkResult.None; + try + { + result = await SendValues(work, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + DisseminationInstruments.OnPumpFailure(DisseminationPumpFailureStatus.Recovered); + LogDebugBroadcastFlushFailed(_owner._logger, exception); + Requeue(work); + result = new(RequiresBackoff: true, MadeProgress: false); + } + finally + { + ScheduledFlush? scheduled = null; + lock (_lock) + { + if (result.MadeProgress) + { + _retryAttempt = 0; + } + + if (DirtyCount > 0 && !_stopping) + { + // Do not overwrite a timer which a newer notification already chose. + if (notificationEpoch == _notificationEpoch) + { + if (result.RequiresBackoff) + { + _retryAttempt++; + var delay = _owner.GetRetryDelay(_retryAttempt); + _flushTimer.Change(delay); + scheduled = new(DisseminationBroadcastScheduleReason.Retry, delay, _retryAttempt, _notificationEpoch); + } + else if (HasHighPriorityDirtyUnsafe()) + { + // Leftover high-priority work must not wait for another coalescing window. + _flushTimer.Change(TimeSpan.Zero); + scheduled = new(DisseminationBroadcastScheduleReason.Priority, TimeSpan.Zero, _retryAttempt, _notificationEpoch); + } + else + { + var delay = _owner.GetCoalescingDelay(TimeSpan.MaxValue); + _flushTimer.Change(delay); + scheduled = new(DisseminationBroadcastScheduleReason.Coalesce, delay, _retryAttempt, _notificationEpoch); + } + + _wakeScheduled = true; + } + else if (!_wakeScheduled) + { + if (HasHighPriorityDirtyUnsafe()) + { + _flushTimer.Change(TimeSpan.Zero); + scheduled = new(DisseminationBroadcastScheduleReason.Priority, TimeSpan.Zero, _retryAttempt, _notificationEpoch); + } + else + { + var delay = _owner.GetCoalescingDelay(TimeSpan.MaxValue); + _flushTimer.Change(delay); + scheduled = new(DisseminationBroadcastScheduleReason.Coalesce, delay, _retryAttempt, _notificationEpoch); + } + + _wakeScheduled = true; + } + } + + if (ReferenceEquals(_activeFlushCompletion, flushCompletion)) + { + _activeFlushCompletion = null; + } + } + + EmitScheduled(scheduled); + flushCompletion.TrySetResult(); + } + } + + private bool TryRecoverFromUnexpectedIterationFailure(Exception exception) + { + try + { + ScheduledFlush? scheduled = null; + TaskCompletionSource? activeFlushCompletion; + lock (_lock) + { + if (DirtyCount > 0 && !_stopping) + { + _retryAttempt++; + var delay = _owner.GetRetryDelay(_retryAttempt); + _flushTimer.Change(delay); + _wakeScheduled = true; + scheduled = new(DisseminationBroadcastScheduleReason.Retry, delay, _retryAttempt, _notificationEpoch); + } + + activeFlushCompletion = _activeFlushCompletion; + _activeFlushCompletion = null; + } + + DisseminationInstruments.OnPumpFailure(DisseminationPumpFailureStatus.Recovered); + LogWarningBroadcastPumpIterationFailed(_owner._logger, exception, Peer, scheduled?.DueTime); + EmitScheduled(scheduled); + activeFlushCompletion?.TrySetResult(); + return true; + } + catch (Exception recoveryException) + { + var failure = new AggregateException( + $"The dissemination broadcast pump for {Peer} could not recover from an iteration failure.", + exception, + recoveryException); + FailPump(failure); + DisseminationInstruments.OnPumpFailure(DisseminationPumpFailureStatus.Permanent); + LogErrorBroadcastPumpFailed(_owner._logger, failure, Peer); + return false; + } + } + + private void FailPump(Exception exception) + { + TaskCompletionSource nextFlushCompletion; + TaskCompletionSource? activeFlushCompletion; + lock (_lock) + { + _pumpFailure = exception; + nextFlushCompletion = _nextFlushCompletion; + _nextFlushCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + activeFlushCompletion = _activeFlushCompletion; + _activeFlushCompletion = null; + ClearPendingUnsafe(); + } + + activeFlushCompletion?.TrySetException(exception); + nextFlushCompletion.TrySetException(exception); + } + + private async ValueTask SendValues( + List initialWork, + CancellationToken cancellationToken) + { + // Every pass re-materializes repairs from the latest acknowledged version instead of retaining serialized messages. + var pending = new Queue(initialWork); + var requiresBackoff = false; + var madeProgress = false; + while (pending.Count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + // Dropping identities needs no transport capacity, including while shutdown drains disabled work. + var enabled = _owner._options.CurrentValue.Enabled; + while (pending.TryPeek(out var nextWork)) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!IsWorkActive(nextWork)) + { + pending.Dequeue(); + } + else if (!enabled || !nextWork.Namespace.Options.Enabled) + { + pending.Dequeue(); + CompleteUnsupported(nextWork); + } + else + { + break; + } + } + + if (pending.Count == 0) + { + break; + } + + var lease = await _owner._sendGate.AcquireAsync(Peer, _admissionQueued, cancellationToken); + try + { + cancellationToken.ThrowIfCancellationRequested(); + // Admission precedes serialization; waiting destinations retain only identities. + var currentOptions = _owner._options.CurrentValue; + var batch = new Dictionary>(); + var sentKeys = new List(); + var itemCount = 0; + var byteCount = 0; + + while (pending.Count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + var work = pending.Peek(); + if (!IsWorkActive(work)) + { + pending.Dequeue(); + continue; + } + + if (!currentOptions.Enabled || !work.Namespace.Options.Enabled) + { + pending.Dequeue(); + CompleteUnsupported(work); + continue; + } + + var knownVersion = GetKnownVersion(work); + var request = new DisseminationRepairRequest( + work.Key, + knownVersion, + toVersion: null, + currentOptions.MaxBatchItems - itemCount, + currentOptions.MaxBatchBytes - byteCount, + work.Namespace.Options.MaxPayloadBytes); + var repair = work.Namespace.CreateRepair(request); + if (repair.Status is DisseminationRepairStatus.Current) + { + // The namespace confirms that the peer is already current, so no RPC is needed. + pending.Dequeue(); + CompleteCurrent(work, repair.Version); + continue; + } + + if (repair.Status is DisseminationRepairStatus.InsufficientCapacity && itemCount > 0) + { + // Give this key a fresh budget in the next batch. + break; + } + + if (repair.Status is DisseminationRepairStatus.InsufficientCapacity) + { + // A value which cannot fit in an empty batch waits for a future publication to change it. + pending.Dequeue(); + CompleteUnsendable(work); + continue; + } + + if (repair.Status is DisseminationRepairStatus.Unavailable + && !IsActiveKey(work.Namespace, work.Key)) + { + // A key absent from current digests was removed, not transiently unavailable. + pending.Dequeue(); + CompleteRemoved(work); + continue; + } + + if (repair.Status is not DisseminationRepairStatus.Produced + || !ValidateRepair(request, repair)) + { + // Invalid or temporarily unavailable repairs retain the dirty key and enter backoff. + pending.Dequeue(); + Requeue([work]); + requiresBackoff = true; + continue; + } + + pending.Dequeue(); + ref var namespaceValues = ref CollectionsMarshal.GetValueRefOrAddDefault( + batch, + work.Namespace.Name, + out _); + namespaceValues ??= []; + foreach (var value in repair.Values) + { + namespaceValues.Add(new DisseminationBroadcastValue + { + Value = value, + TimeToLive = work.Namespace.Options.StaleItemTtl, + }); + itemCount++; + byteCount += value.Payload.Length; + } + + sentKeys.Add(new(work, knownVersion, repair.Version)); + if (itemCount >= currentOptions.MaxBatchItems || byteCount >= currentOptions.MaxBatchBytes) + { + break; + } + } + + if (itemCount == 0) + { + continue; + } + + var responseTask = SendBatch(batch, cancellationToken); + batch = null!; + var response = await responseTask; + if (response is null) + { + Requeue(sentKeys.Select(static sent => sent.Work)); + Requeue(pending); + requiresBackoff = true; + break; + } + + // RPC completion is not application evidence; only the returned receiver versions advance the ledger. + _owner._responseObserver?.Invoke(Peer, response); + var acknowledgments = CreateAcknowledgmentLookup(response.Acknowledgments); + var unsupportedNamespaces = response.UnsupportedNamespaces.ToHashSet(); + foreach (var sent in sentKeys) + { + if (unsupportedNamespaces.Contains(sent.Work.Namespace.Name)) + { + CompleteUnsupported(sent.Work); + continue; + } + + if (!acknowledgments.TryGetValue( + new(sent.Work.Namespace.Name, sent.Work.Key), + out var acknowledgedVersion)) + { + if (!CompleteFromExistingEvidence(sent)) + { + requiresBackoff = true; + } + + continue; + } + + var completion = CompleteAcknowledged(sent, acknowledgedVersion); + madeProgress |= completion.MadeProgress; + requiresBackoff |= completion.RequiresBackoff; + if (completion.ImmediateRetry is { } retry) + { + // An acknowledged prefix rejoins admission before continuing from the receiver's new version. + pending.Enqueue(retry); + } + } + } + finally + { + lease.Dispose(); + } + } + + return new(requiresBackoff, madeProgress); + } + + private async ValueTask SendBatch( + Dictionary> valuesByNamespace, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var transportLifetime = valuesByNamespace.Values + .SelectMany(static values => values) + .Min(static value => value.TimeToLive); + if (transportLifetime <= TimeSpan.Zero) + { + return null; + } + + if (transportLifetime > MaxTransportLifetime) + { + transportLifetime = MaxTransportLifetime; + } + + using var lifetimeCancellation = new CancellationTokenSource(transportLifetime, _owner._timeProvider); + using var sendCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + lifetimeCancellation.Token); + try + { + var batch = new DisseminationBroadcastBatch + { + Sender = _owner._localSilo, + Values = valuesByNamespace, + }; + sendCancellation.Token.ThrowIfCancellationRequested(); + var sendTask = GetTarget().PushBroadcast(batch, sendCancellation.Token); + try + { + var response = await sendTask.WaitAsync(sendCancellation.Token); + DisseminationInstruments.OnBroadcastSent(batch.Values, "tree"); + return response; + } + catch (OperationCanceledException) when (sendCancellation.IsCancellationRequested) + { + ObserveLateSend(sendTask); + throw; + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (OperationCanceledException) when (lifetimeCancellation.IsCancellationRequested) + { + DisseminationInstruments.OnBroadcastSendFailure(DisseminationFailureReason.Timeout); + LogDebugBroadcastTransportLifetimeExpired(_owner._logger, Peer, transportLifetime); + return null; + } + catch (Exception exception) + { + DisseminationInstruments.OnBroadcastSendFailure(DisseminationFailureReason.Error); + LogDebugDisseminationSendFailed(_owner._logger, exception, Peer); + return null; + } + } + + private void ObserveLateSend(Task sendTask) + { + // Fault observation follows transport completion after the operation's cancellation. + sendTask.ContinueWith( + task => LogDebugDisseminationSendFailed(_owner._logger, task.Exception!, Peer), + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default).Ignore(); + } + + private IDisseminationSystemTarget GetTarget() => + _target ??= _owner._grainFactory.GetSystemTarget( + Constants.DisseminationSystemTargetType, + Peer); + + private List DrainDirtyUnsafe() + { + // Snapshot dirty work into an in-flight generation; any later notification will set Dirty again. + // Drain higher-priority namespaces first so their values fill the earliest batches. + var result = new List(DirtyCount); + foreach (var namespaceState in _statesByNamespace.Values + .OrderByDescending(static state => (int)state.Namespace.Options.Priority)) + { + foreach (var (key, keyState) in namespaceState.Keys) + { + if (!keyState.Dirty) + { + continue; + } + + keyState.Dirty = false; + keyState.InFlight = true; + namespaceState.DirtyCount--; + DirtyCount--; + result.Add(new( + namespaceState.Namespace, + key, + keyState.NotificationGeneration, + keyState.KnownVersion)); + } + } + + return result; + } + + private long? GetKnownVersion(PendingKeyWork work) + { + lock (_lock) + { + return TryGetKeyStateUnsafe(work, out _, out var keyState) + ? keyState.KnownVersion + : work.KnownVersion; + } + } + + private bool IsWorkActive(PendingKeyWork work) + { + lock (_lock) + { + return TryGetKeyStateUnsafe(work, out _, out var keyState) && keyState.InFlight; + } + } + + private void CompleteCurrent(PendingKeyWork work, long version) + { + lock (_lock) + { + if (!TryGetKeyStateUnsafe(work, out var namespaceState, out var keyState)) + { + return; + } + + if (keyState.KnownVersion is not { } knownVersion || version > knownVersion) + { + keyState.KnownVersion = version; + } + + keyState.InFlight = false; + if (keyState.NotificationGeneration == work.NotificationGeneration && !keyState.Dirty) + { + namespaceState.RetireKey(work.Key, keyState); + } + } + } + + private AcknowledgmentCompletion CompleteAcknowledged(SentKey sent, long acknowledgedVersion) + { + lock (_lock) + { + if (!TryGetKeyStateUnsafe(sent.Work, out var namespaceState, out var keyState)) + { + return default; + } + + var previousVersion = keyState.KnownVersion; + if (previousVersion is null || acknowledgedVersion > previousVersion) + { + keyState.KnownVersion = acknowledgedVersion; + } + + var madeProgress = sent.FromVersion is null + ? acknowledgedVersion > 0 + : acknowledgedVersion > sent.FromVersion; + // Retire only the generation we sent; a newer notification remains queued even if this repair reached its target. + if (keyState.NotificationGeneration != sent.Work.NotificationGeneration + || keyState.Dirty) + { + keyState.InFlight = false; + return new(null, RequiresBackoff: false, madeProgress); + } + + if (keyState.KnownVersion >= sent.ResolvedVersion) + { + keyState.InFlight = false; + namespaceState.RetireKey(sent.Work.Key, keyState); + return new(null, RequiresBackoff: false, madeProgress); + } + + if (madeProgress) + { + // Partial repairs stay in flight and are re-materialized immediately from the acknowledged prefix. + return new( + sent.Work with { NotificationGeneration = keyState.NotificationGeneration, KnownVersion = keyState.KnownVersion }, + RequiresBackoff: false, + MadeProgress: true); + } + + keyState.InFlight = false; + MarkDirtyUnsafe(namespaceState, keyState); + return new(null, RequiresBackoff: true, MadeProgress: false); + } + } + + private bool CompleteFromExistingEvidence(SentKey sent) + { + // A concurrent digest observation can satisfy a response which omitted this key's acknowledgment. + lock (_lock) + { + if (!TryGetKeyStateUnsafe(sent.Work, out var namespaceState, out var keyState)) + { + return true; + } + + if (keyState.KnownVersion >= sent.ResolvedVersion) + { + keyState.InFlight = false; + if (keyState.NotificationGeneration == sent.Work.NotificationGeneration && !keyState.Dirty) + { + namespaceState.RetireKey(sent.Work.Key, keyState); + } + + return true; + } + + keyState.InFlight = false; + MarkDirtyUnsafe(namespaceState, keyState); + return false; + } + } + + private void CompleteUnsupported(PendingKeyWork work) + { + // Treat this as peer capability evidence and forget the namespace until a new publication recreates it. + lock (_lock) + { + if (!_statesByNamespace.TryGetValue(work.Namespace.Name, out var namespaceState)) + { + return; + } + + DirtyCount -= namespaceState.DirtyCount; + _statesByNamespace.Remove(work.Namespace.Name); + if (DirtyCount == 0) + { + var completion = _nextFlushCompletion; + _nextFlushCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + completion.TrySetResult(); + } + } + } + + private void CompleteRemoved(PendingKeyWork work) + { + // Removed keys leave no retry state behind unless a newer notification raced with the repair. + lock (_lock) + { + if (!TryGetKeyStateUnsafe(work, out var namespaceState, out var keyState)) + { + return; + } + + if (keyState.NotificationGeneration != work.NotificationGeneration || keyState.Dirty) + { + keyState.InFlight = false; + return; + } + + namespaceState.Keys.Remove(work.Key); + if (namespaceState.Keys.Count == 0) + { + _statesByNamespace.Remove(work.Namespace.Name); + } + } + } + + private void CompleteUnsendable(PendingKeyWork work) + { + // Release admission capacity while preserving peer knowledge for a later publication. + lock (_lock) + { + if (TryGetKeyStateUnsafe(work, out var namespaceState, out var keyState)) + { + keyState.InFlight = false; + if (keyState.NotificationGeneration == work.NotificationGeneration && !keyState.Dirty) + { + namespaceState.RetireKey(work.Key, keyState); + } + } + } + } + + private void Requeue(IEnumerable work) + { + lock (_lock) + { + if (_stopping) + { + foreach (var item in work) + { + if (TryGetKeyStateUnsafe(item, out _, out var keyState)) + { + keyState.InFlight = false; + } + } + + return; + } + + foreach (var item in work) + { + if (TryGetKeyStateUnsafe(item, out var namespaceState, out var keyState)) + { + keyState.InFlight = false; + MarkDirtyUnsafe(namespaceState, keyState); + } + } + } + } + + private void ClearPendingUnsafe() + { + _statesByNamespace.Clear(); + DirtyCount = 0; + _retryAttempt = 0; + _wakeScheduled = false; + } + + private PeerNamespaceState GetOrCreateNamespaceStateUnsafe(IDisseminationNamespace disseminationNamespace) + { + if (!_statesByNamespace.TryGetValue(disseminationNamespace.Name, out var result)) + { + result = new(disseminationNamespace); + _statesByNamespace.Add(disseminationNamespace.Name, result); + } + + return result; + } + + private bool TryGetKeyStateUnsafe( + PendingKeyWork work, + out PeerNamespaceState namespaceState, + out PeerKeyState keyState) + { + if (_statesByNamespace.TryGetValue(work.Namespace.Name, out namespaceState!) + && namespaceState.Keys.TryGetValue(work.Key, out keyState!)) + { + return true; + } + + namespaceState = null!; + keyState = null!; + return false; + } + + private void MarkDirtyUnsafe(PeerNamespaceState namespaceState, PeerKeyState keyState) + { + if (keyState.Dirty) + { + return; + } + + keyState.Dirty = true; + namespaceState.DirtyCount++; + DirtyCount++; + } + + private bool HasHighPriorityDirtyUnsafe() + { + foreach (var namespaceState in _statesByNamespace.Values) + { + if (namespaceState.DirtyCount > 0 + && namespaceState.Namespace.Options.Priority == DisseminationPriority.High) + { + return true; + } + } + + return false; + } + + private static Dictionary CreateAcknowledgmentLookup( + Dictionary> acknowledgments) + { + var result = new Dictionary(); + foreach (var (namespaceName, entries) in acknowledgments) + { + foreach (var entry in entries) + { + var key = new DigestKey(namespaceName, entry.Key); + if (!result.TryGetValue(key, out var version) || entry.Version > version) + { + result[key] = entry.Version; + } + } + } + + return result; + } + + private static bool IsActiveKey( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key) => + disseminationNamespace.Digests.Any(entry => entry.Key == key); + + private static bool ValidateRepair( + in DisseminationRepairRequest request, + in DisseminationRepairResult repair) + { + // Defensively enforce a bounded, contiguous chain toward the namespace's resolved target. + if (repair.Status is not DisseminationRepairStatus.Produced + || repair.Values.IsDefaultOrEmpty + || repair.Version <= 0 + || request.ToVersion is { } requestedToVersion && repair.Version > requestedToVersion + || repair.Values.Length > request.MaxItemCount) + { + return false; + } + + var byteCount = 0; + var expectedFromVersion = request.FromVersion; + for (var i = 0; i < repair.Values.Length; i++) + { + var value = repair.Values[i]; + if (value.Key != request.Key + || value.FromVersion < 0 + || value.ToVersion <= value.FromVersion + || value.ToVersion > repair.Version + || value.Payload.Length > request.MaxPayloadBytes) + { + return false; + } + + if (i == 0) + { + if (expectedFromVersion is null && value.FromVersion != 0 + || expectedFromVersion is { } fromVersion + && value.FromVersion != 0 + && value.FromVersion != fromVersion) + { + return false; + } + } + else if (value.FromVersion != 0 && value.FromVersion != expectedFromVersion) + { + return false; + } + + expectedFromVersion = value.ToVersion; + byteCount += value.Payload.Length; + if (byteCount > request.MaxBatchBytes) + { + return false; + } + } + + var lastVersion = repair.Values[^1].ToVersion; + return repair.IsComplete ? lastVersion == repair.Version : lastVersion < repair.Version; + } + + private sealed class PeerNamespaceState(IDisseminationNamespace disseminationNamespace) + { + public IDisseminationNamespace Namespace { get; } = disseminationNamespace; + + public Dictionary Keys { get; } = []; + + public Dictionary KnownVersions { get; } = []; + + public int DirtyCount { get; set; } + + public bool TryGetKey(DisseminationKey key, out PeerKeyState? keyState) => + Keys.TryGetValue(key, out keyState); + + public PeerKeyState AddKey(DisseminationKey key) + { + var result = new PeerKeyState(); + if (KnownVersions.TryGetValue(key, out var knownVersion)) + { + result.KnownVersion = knownVersion; + } + + Keys.Add(key, result); + return result; + } + + public void ObserveKnownVersion(DisseminationKey key, long version) + { + if (!KnownVersions.TryGetValue(key, out var knownVersion) || version > knownVersion) + { + KnownVersions[key] = version; + } + } + + public void RetireKey(DisseminationKey key, PeerKeyState keyState) + { + if (keyState.KnownVersion is { } knownVersion) + { + ObserveKnownVersion(key, knownVersion); + } + + Keys.Remove(key); + } + + public void PruneKnownVersions(HashSet? activeKeys) + { + foreach (var key in KnownVersions.Keys.ToArray()) + { + if (activeKeys is null || !activeKeys.Contains(key)) + { + KnownVersions.Remove(key); + } + } + } + } + + private sealed class PeerKeyState + { + // A null version means no known baseline. Dirty can coexist with InFlight when a newer notification arrives mid-send. + public long? KnownVersion { get; set; } + + public long NotificationGeneration { get; set; } + + public long NotificationVersion { get; set; } + + public bool Dirty { get; set; } + + public bool InFlight { get; set; } + } + + private readonly record struct PendingKeyWork( + IDisseminationNamespace Namespace, + DisseminationKey Key, + long NotificationGeneration, + long? KnownVersion); + + private readonly record struct SentKey( + PendingKeyWork Work, + long? FromVersion, + long ResolvedVersion); + + private readonly record struct DigestKey( + DisseminationNamespace Namespace, + DisseminationKey Key); + + private readonly record struct AcknowledgmentCompletion( + PendingKeyWork? ImmediateRetry, + bool RequiresBackoff, + bool MadeProgress); + + private readonly record struct SendWorkResult(bool RequiresBackoff, bool MadeProgress) + { + public static SendWorkResult None => default; + } + } + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination send to {Peer} failed.")] + private static partial void LogDebugDisseminationSendFailed( + ILogger logger, + Exception exception, + SiloAddress peer); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination broadcast batch flush failed.")] + private static partial void LogDebugBroadcastFlushFailed( + ILogger logger, + Exception exception); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Dissemination broadcast pump for {Peer} encountered an unexpected iteration failure and will retry after {RetryDelay}.")] + private static partial void LogWarningBroadcastPumpIterationFailed( + ILogger logger, + Exception exception, + SiloAddress peer, + TimeSpan? retryDelay); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Dissemination broadcast pump for {Peer} failed permanently. Pending flush and drain waiters will fail explicitly.")] + private static partial void LogErrorBroadcastPumpFailed( + ILogger logger, + Exception exception, + SiloAddress peer); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination broadcast scheduling diagnostic for {Peer} failed.")] + private static partial void LogDebugBroadcastDiagnosticFailed( + ILogger logger, + Exception exception, + SiloAddress peer); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination broadcast transport to {Peer} exceeded its remaining hop lifetime of {Lifetime}.")] + private static partial void LogDebugBroadcastTransportLifetimeExpired( + ILogger logger, + SiloAddress peer, + TimeSpan lifetime); +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationEvents.cs b/src/Orleans.Runtime/Dissemination/DisseminationEvents.cs new file mode 100644 index 00000000000..c17150aff95 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationEvents.cs @@ -0,0 +1,192 @@ +using System; +using System.Diagnostics; + +namespace Orleans.Runtime.Dissemination; + +internal static class DisseminationEvents +{ + public const string ListenerName = "Microsoft.Orleans.Dissemination"; + public static readonly DiagnosticListener Listener = new(ListenerName); + + public static void EmitValue(DisseminationNamespace namespaceName, DisseminationValue value, SiloAddress localSilo, SiloAddress? peer, DisseminationApplyResult result, int payloadBytes) + { + if (Listener.IsEnabled("Dissemination.ValueApply")) + { + Listener.Write("Dissemination.ValueApply", new DisseminationValueEvent + { + Namespace = namespaceName, + LocalSilo = localSilo, + Peer = peer, + Key = value.Key, + FromVersion = value.FromVersion, + ToVersion = value.ToVersion, + Result = result.ToString(), + PayloadBytes = payloadBytes, + Timestamp = DateTimeOffset.UtcNow, + }); + } + } + + public static void EmitPayloadDrop(DisseminationNamespace namespaceName, DisseminationValue value, SiloAddress localSilo, string reason, int payloadBytes) + { + if (Listener.IsEnabled("Dissemination.PayloadDrop")) + { + Listener.Write("Dissemination.PayloadDrop", new DisseminationValueEvent + { + Namespace = namespaceName, + LocalSilo = localSilo, + Key = value.Key, + FromVersion = value.FromVersion, + ToVersion = value.ToVersion, + Result = reason, + PayloadBytes = payloadBytes, + Timestamp = DateTimeOffset.UtcNow, + }); + } + } + + public const string BroadcastScheduledEventName = "Dissemination.BroadcastScheduled"; + public const string QueueAdmissionRejectedEventName = "Dissemination.QueueAdmissionRejected"; + public const string SendGateEventName = "Dissemination.SendGate"; + public const string NamespacePendingLimitReason = "namespace_pending_limit"; + + public static void EmitSendGate(SiloAddress localSilo, SiloAddress peer, string kind, string stage) + { + if (Listener.IsEnabled(SendGateEventName)) + { + Listener.Write(SendGateEventName, new DisseminationSendGateEvent + { + LocalSilo = localSilo, + Peer = peer, + Kind = kind, + Stage = stage, + Timestamp = DateTimeOffset.UtcNow, + }); + } + } + + public static void EmitBroadcastScheduled( + SiloAddress localSilo, + SiloAddress peer, + DisseminationBroadcastScheduleReason reason, + TimeSpan dueTime, + int attempt, + long epoch) + { + if (Listener.IsEnabled(BroadcastScheduledEventName)) + { + Listener.Write(BroadcastScheduledEventName, new DisseminationBroadcastScheduledEvent + { + LocalSilo = localSilo, + Peer = peer, + Reason = reason, + DueTime = dueTime, + Attempt = attempt, + Epoch = epoch, + Timestamp = DateTimeOffset.UtcNow, + }); + } + } + + public static void EmitQueueAdmissionRejected( + SiloAddress localSilo, + SiloAddress peer, + DisseminationNamespace namespaceName, + int limit) + { + if (Listener.IsEnabled(QueueAdmissionRejectedEventName)) + { + Listener.Write(QueueAdmissionRejectedEventName, new DisseminationQueueAdmissionRejectedEvent + { + LocalSilo = localSilo, + Peer = peer, + Namespace = namespaceName, + Limit = limit, + Reason = NamespacePendingLimitReason, + Timestamp = DateTimeOffset.UtcNow, + }); + } + } +} + +// Why a peer pump (re)armed its flush timer, exposed for deterministic tests and diagnostics. +internal enum DisseminationBroadcastScheduleReason +{ + // A batch filled, so the pump flushes without further coalescing. + Immediate, + + // The pump is coalescing and will flush after the namespace delay. + Coalesce, + + // A prior send failed and the pump re-armed after backoff. + Retry, + + // A high-priority namespace bypasses the coalescing window and flushes immediately. + Priority, +} + +internal sealed class DisseminationBroadcastScheduledEvent +{ + public required SiloAddress LocalSilo { get; init; } + + public required SiloAddress Peer { get; init; } + + public DisseminationBroadcastScheduleReason Reason { get; init; } + + public TimeSpan DueTime { get; init; } + + public int Attempt { get; init; } + + public long Epoch { get; init; } + + public DateTimeOffset Timestamp { get; init; } +} + +internal sealed class DisseminationValueEvent +{ + public DisseminationNamespace Namespace { get; init; } + + public required SiloAddress LocalSilo { get; init; } + + public SiloAddress? Peer { get; init; } + + public DisseminationKey Key { get; init; } + + public long FromVersion { get; init; } + + public long ToVersion { get; init; } + + public string Result { get; init; } = string.Empty; + + public int PayloadBytes { get; init; } + + public DateTimeOffset Timestamp { get; init; } +} + +internal sealed class DisseminationQueueAdmissionRejectedEvent +{ + public required SiloAddress LocalSilo { get; init; } + + public required SiloAddress Peer { get; init; } + + public DisseminationNamespace Namespace { get; init; } + + public int Limit { get; init; } + + public string Reason { get; init; } = string.Empty; + + public DateTimeOffset Timestamp { get; init; } +} + +internal sealed class DisseminationSendGateEvent +{ + public required SiloAddress LocalSilo { get; init; } + + public required SiloAddress Peer { get; init; } + + public required string Kind { get; init; } + + public required string Stage { get; init; } + + public DateTimeOffset Timestamp { get; init; } +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationInstruments.cs b/src/Orleans.Runtime/Dissemination/DisseminationInstruments.cs new file mode 100644 index 00000000000..824d0faf41b --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationInstruments.cs @@ -0,0 +1,235 @@ +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace Orleans.Runtime.Dissemination; + +internal static class DisseminationInstruments +{ + internal const string MeterName = "Microsoft.Orleans"; + internal const string BroadcastSendFailuresName = "orleans-dissemination-broadcast-send-failures"; + internal const string BroadcastScheduledName = "orleans-dissemination-broadcast-scheduled"; + internal const string AntiEntropyFailuresName = "orleans-dissemination-anti-entropy-failures"; + internal const string PumpFailuresName = "orleans-dissemination-pump-failures"; + internal const string PublicationsName = "orleans-dissemination-publications"; + internal const string QueueAdmissionRejectedName = "orleans-dissemination-queue-admission-rejected"; + internal const string ValuesReceivedName = "orleans-dissemination-values-received"; + + private static readonly Meter Meter = new(MeterName); + private static readonly Counter BroadcastSent = Meter.CreateCounter("orleans-dissemination-broadcast-sent", "messages"); + private static readonly Counter BroadcastReceived = Meter.CreateCounter("orleans-dissemination-broadcast-received", "messages"); + private static readonly Counter ValuesSent = Meter.CreateCounter("orleans-dissemination-values-sent", "values"); + private static readonly Counter ValuesReceived = Meter.CreateCounter(ValuesReceivedName, "values"); + private static readonly Counter ValuesApplied = Meter.CreateCounter("orleans-dissemination-values-applied", "values"); + private static readonly Counter BytesSent = Meter.CreateCounter("orleans-dissemination-bytes-sent", "bytes"); + private static readonly Counter BroadcastSendFailures = Meter.CreateCounter(BroadcastSendFailuresName, "attempts"); + private static readonly Counter BroadcastScheduled = Meter.CreateCounter(BroadcastScheduledName, "schedules"); + private static readonly Counter AntiEntropyExchanges = Meter.CreateCounter("orleans-dissemination-anti-entropy-exchanges", "operations"); + private static readonly Counter AntiEntropyDigests = Meter.CreateCounter("orleans-dissemination-anti-entropy-digests", "digests"); + private static readonly Counter AntiEntropyValues = Meter.CreateCounter("orleans-dissemination-anti-entropy-values", "values"); + private static readonly Counter AntiEntropyFailures = Meter.CreateCounter(AntiEntropyFailuresName, "operations"); + private static readonly Counter PumpFailures = Meter.CreateCounter(PumpFailuresName, "failures"); + private static readonly Counter Publications = Meter.CreateCounter(PublicationsName, "operations"); + private static readonly Counter PayloadDropped = Meter.CreateCounter("orleans-dissemination-payload-dropped", "values"); + private static readonly Counter QueueAdmissionRejected = Meter.CreateCounter( + QueueAdmissionRejectedName, + "keys"); + + public static void OnBroadcastSent(DisseminationNamespace namespaceName, string kind, int itemCount, int byteCount) + { + var broadcastSentEnabled = BroadcastSent.Enabled; + var valuesSentEnabled = ValuesSent.Enabled; + var bytesSentEnabled = BytesSent.Enabled; + if (!broadcastSentEnabled && !valuesSentEnabled && !bytesSentEnabled) + { + return; + } + + var namespaceTag = Tag("namespace", namespaceName); + var kindTag = Tag("kind", kind); + if (broadcastSentEnabled) + { + BroadcastSent.Add(1, namespaceTag, kindTag); + } + + if (valuesSentEnabled) + { + ValuesSent.Add(itemCount, namespaceTag, kindTag); + } + + if (bytesSentEnabled) + { + BytesSent.Add(byteCount, namespaceTag, kindTag); + } + } + + public static void OnBroadcastSent(Dictionary> valuesByNamespace, string kind) + { + if (!BroadcastSent.Enabled && !ValuesSent.Enabled && !BytesSent.Enabled) + { + return; + } + + foreach (var (namespaceName, values) in valuesByNamespace) + { + OnBroadcastSent(namespaceName, kind, values.Count, values.Sum(static item => item.Value.Payload.Length)); + } + } + + public static void OnBroadcastReceived(DisseminationNamespace namespaceName, string kind, int itemCount) + { + var broadcastReceivedEnabled = BroadcastReceived.Enabled; + var valuesReceivedEnabled = ValuesReceived.Enabled; + if (!broadcastReceivedEnabled && !valuesReceivedEnabled) + { + return; + } + + var namespaceTag = Tag("namespace", namespaceName); + if (broadcastReceivedEnabled) + { + BroadcastReceived.Add(1, namespaceTag, Tag("kind", kind)); + } + + if (valuesReceivedEnabled) + { + ValuesReceived.Add(itemCount, namespaceTag, Tag("kind", kind)); + } + } + + public static void OnBroadcastSendFailure(DisseminationFailureReason reason) + { + if (BroadcastSendFailures.Enabled) + { + BroadcastSendFailures.Add(1, Tag("reason", GetFailureReason(reason))); + } + } + + public static void OnBroadcastScheduled(DisseminationBroadcastScheduleReason reason) + { + if (BroadcastScheduled.Enabled) + { + BroadcastScheduled.Add(1, Tag("reason", GetScheduleReason(reason))); + } + } + + public static void OnValueApplied(DisseminationNamespace namespaceName, DisseminationApplyResult result) + { + if (!ValuesApplied.Enabled) + { + return; + } + + ValuesApplied.Add(1, Tag("namespace", namespaceName), Tag("result", GetApplyResult(result))); + } + + public static void OnAntiEntropyExchange(string direction, int digestCount, int itemCount, bool truncated) + { + var antiEntropyExchangesEnabled = AntiEntropyExchanges.Enabled; + var antiEntropyDigestsEnabled = AntiEntropyDigests.Enabled; + var antiEntropyValuesEnabled = AntiEntropyValues.Enabled; + if (!antiEntropyExchangesEnabled && !antiEntropyDigestsEnabled && !antiEntropyValuesEnabled) + { + return; + } + + var directionTag = Tag("direction", direction); + if (antiEntropyExchangesEnabled) + { + AntiEntropyExchanges.Add(1, directionTag, Tag("truncated", truncated)); + } + + if (antiEntropyDigestsEnabled) + { + AntiEntropyDigests.Add(digestCount, directionTag); + } + + if (antiEntropyValuesEnabled) + { + AntiEntropyValues.Add(itemCount, directionTag); + } + } + + public static void OnAntiEntropyFailure(DisseminationFailureReason reason, int count = 1) + { + if (AntiEntropyFailures.Enabled && count > 0) + { + AntiEntropyFailures.Add(count, Tag("reason", GetFailureReason(reason))); + } + } + + public static void OnPumpFailure(DisseminationPumpFailureStatus status) + { + if (PumpFailures.Enabled) + { + PumpFailures.Add( + 1, + Tag("status", status == DisseminationPumpFailureStatus.Recovered ? "recovered" : "permanent")); + } + } + + public static void OnPublication(DisseminationNamespace namespaceName, bool accepted, string reason) + { + if (Publications.Enabled) + { + Publications.Add( + 1, + Tag("namespace", namespaceName), + Tag("result", accepted ? "accepted" : "rejected"), + Tag("reason", reason)); + } + } + + public static void OnPayloadDropped(DisseminationNamespace namespaceName, string reason) + { + if (PayloadDropped.Enabled) + { + PayloadDropped.Add(1, Tag("namespace", namespaceName), Tag("reason", reason)); + } + } + + public static void OnQueueAdmissionRejected(DisseminationNamespace namespaceName) + { + if (QueueAdmissionRejected.Enabled) + { + QueueAdmissionRejected.Add( + 1, + Tag("namespace", namespaceName), + Tag("reason", DisseminationEvents.NamespacePendingLimitReason)); + } + } + + private static string GetFailureReason(DisseminationFailureReason reason) => + reason == DisseminationFailureReason.Timeout ? "timeout" : "error"; + + private static string GetApplyResult(DisseminationApplyResult result) => result switch + { + DisseminationApplyResult.Applied => "applied", + DisseminationApplyResult.Duplicate => "duplicate", + DisseminationApplyResult.Obsolete => "obsolete", + DisseminationApplyResult.Rejected => "rejected", + _ => "unknown", + }; + + private static string GetScheduleReason(DisseminationBroadcastScheduleReason reason) => reason switch + { + DisseminationBroadcastScheduleReason.Immediate => "immediate", + DisseminationBroadcastScheduleReason.Coalesce => "coalesce", + DisseminationBroadcastScheduleReason.Retry => "retry", + DisseminationBroadcastScheduleReason.Priority => "priority", + _ => "unknown", + }; + + private static KeyValuePair Tag(string name, object value) => new(name, value); +} + +internal enum DisseminationFailureReason +{ + Timeout, + Error, +} + +internal enum DisseminationPumpFailureStatus +{ + Recovered, + Permanent, +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs b/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs new file mode 100644 index 00000000000..332caf0becf --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs @@ -0,0 +1,130 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime.MembershipService; + +namespace Orleans.Runtime.Dissemination; + +internal sealed class DisseminationMembership( + IMembershipManager membershipManager, + ILocalSiloDetails localSiloDetails, + IOptions options) +{ + private readonly object _membershipLock = new(); + private DisseminationMembershipSnapshots? _currentSnapshots; + + public DisseminationMembershipSnapshots CurrentSnapshots + { + get + { + var membershipSnapshot = membershipManager.CurrentSnapshot; + var current = Volatile.Read(ref _currentSnapshots); + if (current is not null && current.MembershipVersion == membershipSnapshot.Version) + { + return current; + } + + lock (_membershipLock) + { + current = Volatile.Read(ref _currentSnapshots); + if (current is not null && current.MembershipVersion >= membershipSnapshot.Version) + { + return current; + } + + current = ComputeMembership(membershipSnapshot, localSiloDetails.SiloAddress, options.Value.Overlay, current); + Volatile.Write(ref _currentSnapshots, current); + return current; + } + } + } + + public DisseminationMembershipSnapshot CurrentSnapshot => CurrentSnapshots.AllMembers; + + public DisseminationMembershipSnapshot GetSnapshot(DisseminationMembershipScope scope) => + CurrentSnapshots.GetSnapshot(scope); + + public Task RefreshMembership(CancellationToken cancellationToken) => + membershipManager.Refresh(targetVersion: null, cancellationToken: cancellationToken); + + public async ValueTask GetSnapshotsContainingMember( + SiloAddress member, + DisseminationMembershipScope scope, + CancellationToken cancellationToken) + { + var snapshots = CurrentSnapshots; + if (snapshots.GetSnapshot(scope).ContainsMember(member)) + { + return snapshots; + } + + await RefreshMembership(cancellationToken); + snapshots = CurrentSnapshots; + return snapshots.GetSnapshot(scope).ContainsMember(member) ? snapshots : null; + } + + private static DisseminationMembershipSnapshots ComputeMembership( + MembershipTableSnapshot snapshot, + SiloAddress localSilo, + DisseminationOverlayOptions overlayOptions, + DisseminationMembershipSnapshots? previous) + { + var members = snapshot.Entries.Values + .Where(static entry => IsDisseminationMember(entry.Status)) + .OrderBy(static entry => GetStatusRank(entry.Status)) + // Storage providers can round StartTime differently from the originating silo's local snapshot. + // SiloAddress includes the generation and provides a stable oldest-first order everywhere. + .ThenBy(static entry => entry.SiloAddress) + .ToArray(); + var allMembers = ImmutableArray.CreateBuilder(members.Length); + var activeMembers = ImmutableArray.CreateBuilder(members.Length); + foreach (var member in members) + { + allMembers.Add(member.SiloAddress); + if (member.Status == SiloStatus.Active) + { + activeMembers.Add(member.SiloAddress); + } + } + + return new( + new DisseminationMembershipSnapshot( + snapshot.Version, + localSilo, + activeMembers.ToImmutable(), + overlayOptions, + previous?.ActiveMembers), + new DisseminationMembershipSnapshot( + snapshot.Version, + localSilo, + allMembers.MoveToImmutable(), + overlayOptions, + previous?.AllMembers)); + } + + private static bool IsDisseminationMember(SiloStatus status) => + status is SiloStatus.Joining or SiloStatus.Active or SiloStatus.ShuttingDown or SiloStatus.Stopping; + + private static int GetStatusRank(SiloStatus status) => status switch + { + SiloStatus.Active => 0, + SiloStatus.Joining => 1, + SiloStatus.ShuttingDown => 2, + SiloStatus.Stopping => 3, + _ => 4, + }; +} + +internal sealed class DisseminationMembershipSnapshots( + DisseminationMembershipSnapshot activeMembers, + DisseminationMembershipSnapshot allMembers) +{ + public MembershipVersion MembershipVersion => AllMembers.MembershipVersion; + + public DisseminationMembershipSnapshot ActiveMembers { get; } = activeMembers; + + public DisseminationMembershipSnapshot AllMembers { get; } = allMembers; + + public DisseminationMembershipSnapshot GetSnapshot(DisseminationMembershipScope scope) => + scope == DisseminationMembershipScope.ActiveMembers ? ActiveMembers : AllMembers; +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs b/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs new file mode 100644 index 00000000000..e6165a37cb0 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs @@ -0,0 +1,144 @@ +using System.Collections.Frozen; +using System.Collections.Immutable; +using System.Runtime.InteropServices; +using Orleans.Configuration; + +namespace Orleans.Runtime.Dissemination; + +internal sealed class DisseminationMembershipSnapshot +{ + private readonly FrozenSet _set; + private readonly SiloAddress[] _antiEntropyPeers; + private readonly AntiEntropyPeerSelection _antiEntropySelection; + + public DisseminationMembershipSnapshot( + MembershipVersion membershipVersion, + SiloAddress localSilo, + ImmutableArray members, + DisseminationOverlayOptions overlayOptions, + DisseminationMembershipSnapshot? previous = null) + { + MembershipVersion = membershipVersion; + Members = members.IsDefault ? [] : members; + // Membership versions also advance without changing the eligible peers. Preserve progress so that + // frequent updates cannot keep repair rounds confined to the first few members. + _antiEntropySelection = previous?._antiEntropySelection ?? new(); + var fanout = Members.Length <= 1 ? 1 : overlayOptions.GetFanOutFactor(Members.Length); + var memberSet = new HashSet(Members.Length); + var localIndex = -1; + for (var i = 0; i < Members.Length; i++) + { + var member = Members[i]; + if (!memberSet.Add(member)) + { + throw new ArgumentException("Membership snapshot members must be unique.", nameof(members)); + } + + if (Equals(member, localSilo)) + { + localIndex = i; + } + } + + _set = memberSet.ToFrozenSet(); + ForwardingTreeTargets = localIndex < 0 ? [] : ComputeForwardingTreeTargets(localIndex, fanout); + + if (localIndex < 0) + { + _antiEntropyPeers = []; + } + else + { + _antiEntropyPeers = new SiloAddress[Members.Length - 1]; + var candidateIndex = 0; + for (var i = 0; i < Members.Length; i++) + { + if (i != localIndex) + { + _antiEntropyPeers[candidateIndex++] = Members[i]; + } + } + } + + OriginatorTreeTargets = ComputeOriginatorTreeTargets(localSilo, localIndex, fanout); + } + + public MembershipVersion MembershipVersion { get; } + + public ImmutableArray Members { get; } + + public ImmutableArray OriginatorTreeTargets { get; } + + public ImmutableArray ForwardingTreeTargets { get; } + + public bool ContainsMember(SiloAddress silo) => _set.Contains(silo); + + public ImmutableArray SelectAntiEntropyPeers(int peerCount) + { + ArgumentOutOfRangeException.ThrowIfNegative(peerCount); + var candidates = _antiEntropyPeers; + var count = Math.Min(peerCount, candidates.Length); + if (count <= 0) + { + return []; + } + + var peers = new SiloAddress[count]; + lock (_antiEntropySelection) + { + for (var i = 0; i < count; i++) + { + peers[i] = candidates[(_antiEntropySelection.Cursor + i) % candidates.Length]; + } + + _antiEntropySelection.Cursor = (_antiEntropySelection.Cursor + count) % candidates.Length; + } + + return ImmutableCollectionsMarshal.AsImmutableArray(peers); + } + + private ImmutableArray ComputeOriginatorTreeTargets(SiloAddress localSilo, int localIndex, int fanout) + { + if (localIndex < 0) + { + return []; + } + + var result = ImmutableArray.CreateBuilder(Math.Min(fanout * 2, Members.Length)); + var count = Math.Min(fanout, Members.Length); + for (var i = 0; i < count; i++) + { + var member = Members[i]; + if (!Equals(member, localSilo)) + { + result.Add(member); + } + } + + result.AddRange(ForwardingTreeTargets); + return result.ToImmutable(); + } + + private ImmutableArray ComputeForwardingTreeTargets(int index, int fanout) + { + var result = ImmutableArray.CreateBuilder(Math.Min(fanout, Members.Length)); + var firstChild = (long)fanout * (index + 1); + for (var i = 0; i < fanout; i++) + { + var childIndex = firstChild + i; + if (childIndex >= Members.Length) + { + break; + } + + result.Add(Members[(int)childIndex]); + } + + return result.ToImmutable(); + } + + private sealed class AntiEntropyPeerSelection + { + public int Cursor; + } +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationNamespaceNames.cs b/src/Orleans.Runtime/Dissemination/DisseminationNamespaceNames.cs new file mode 100644 index 00000000000..f2d8279b717 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationNamespaceNames.cs @@ -0,0 +1,7 @@ +namespace Orleans.Runtime.Dissemination; + +internal static class DisseminationNamespaceNames +{ + public static readonly DisseminationNamespace DeploymentLoad = new("load"); + public static readonly DisseminationNamespace Membership = new("membership"); +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs new file mode 100644 index 00000000000..7c5b2d44a00 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -0,0 +1,1340 @@ +using System.Collections.Frozen; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.Configuration; + +namespace Orleans.Runtime.Dissemination; + +// The protocol coordinates routing and application while namespaces remain authoritative for values and repair history. +internal sealed partial class DisseminationProtocol +{ + private const int MaxRetainedNonMemberResponseCursors = 64; + private static readonly TimeSpan MaxAntiEntropyRoundLifetime = TimeSpan.FromMilliseconds(uint.MaxValue - 1); + private readonly SiloAddress _localSilo; + private readonly IInternalGrainFactory _grainFactory; + private readonly DisseminationMembership _membership; + private readonly IOptionsMonitor _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly DisseminationBroadcastQueue _broadcastQueue; + private readonly DisseminationSendGate _antiEntropySendGate; + private readonly CancellationTokenSource _antiEntropyShutdown = new(); + private readonly object _antiEntropyResponseCursorLock = new(); + private readonly Dictionary _antiEntropyResponseCursors = []; + private long _antiEntropyResponseCursorAccess; + private readonly object _valueUpdateLock = new(); + private readonly Dictionary _lastValueUpdates = []; + private readonly object _peerSupportLock = new(); + private readonly Dictionary> _confirmedPeerNamespaces = []; + private readonly FrozenDictionary _namespaces; + + public DisseminationProtocol( + ILocalSiloDetails localSiloDetails, + IInternalGrainFactory grainFactory, + DisseminationMembership membership, + IOptionsMonitor options, + IEnumerable disseminationNamespaces, + TimeProvider timeProvider, + ILogger logger, + ILogger broadcastQueueLogger) + { + _localSilo = localSiloDetails.SiloAddress; + _grainFactory = grainFactory; + _membership = membership; + _options = options; + _timeProvider = timeProvider; + _logger = logger; + _namespaces = disseminationNamespaces.ToFrozenDictionary(static ns => ns.Name); + _antiEntropySendGate = new(Math.Max(1, options.CurrentValue.Overlay.AntiEntropyPeerCount)); + + _broadcastQueue = new DisseminationBroadcastQueue( + _timeProvider, + _localSilo, + _grainFactory, + _options, + _namespaces.Values, + broadcastQueueLogger, + ObserveBroadcastResponse); + } + + public async ValueTask Publish( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var options = _options.CurrentValue; + if (!options.Enabled || !disseminationNamespace.Options.Enabled) + { + DisseminationInstruments.OnPublication( + disseminationNamespace.Name, + accepted: false, + reason: "disabled"); + return false; + } + + // Before replacing the legacy fallback, prove that an unknown peer can receive a complete, bounded repair. + if (!TryValidatePublish( + disseminationNamespace, + key, + version, + options, + out var publishedVersion, + out var reason)) + { + DisseminationInstruments.OnPublication( + disseminationNamespace.Name, + accepted: false, + reason: reason); + return false; + } + + var membership = await GetMembershipSnapshotForRouting( + disseminationNamespace.MembershipScope, + _localSilo, + cancellationToken); + if (membership is null) + { + DisseminationInstruments.OnPublication( + disseminationNamespace.Name, + accepted: false, + "membership-unavailable"); + return false; + } + + // Notifications carry identity only; each peer pump asks the namespace for the latest repair at send time. + RecordValueUpdate(disseminationNamespace.Name, key, publishedVersion); + var accepted = true; + foreach (var peer in membership.OriginatorTreeTargets) + { + accepted &= _broadcastQueue.Notify(peer, disseminationNamespace, key); + } + + DisseminationInstruments.OnPublication( + disseminationNamespace.Name, + accepted, + reason: accepted ? "none" : "queue-rejected"); + return accepted; + } + + public async Task ReceiveBroadcast( + DisseminationBroadcastBatch batch, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var options = _options.CurrentValue; + if (!options.Enabled) + { + return new DisseminationBroadcastResponse + { + UnsupportedNamespaces = [.. batch.Values.Keys], + }; + } + + var receivedKeys = new Dictionary>(); + var unsupportedNamespaces = new List(); + foreach (var (namespaceName, values) in batch.Values) + { + if (!TryGetEnabledNamespace(namespaceName, out var disseminationNamespace)) + { + unsupportedNamespaces.Add(namespaceName); + continue; + } + + DisseminationInstruments.OnBroadcastReceived(disseminationNamespace.Name, "tree", values.Count); + ConfirmPeerNamespaces(batch.Sender, [namespaceName]); + var namespaceKeys = new Dictionary(); + receivedKeys.Add(disseminationNamespace, namespaceKeys); + foreach (var item in values) + { + cancellationToken.ThrowIfCancellationRequested(); + namespaceKeys.TryAdd(item.Value.Key, false); + // The sender necessarily owns this version; use that fact only if an outbound ledger already exists. + _broadcastQueue.ObservePeerVersion( + batch.Sender, + namespaceName, + item.Value.Key, + item.Value.ToVersion); + var result = await ApplyReceivedValue( + disseminationNamespace, + item, + batch.Sender, + options, + cancellationToken); + if (result is DisseminationApplyResult.Applied) + { + namespaceKeys[item.Value.Key] = true; + } + } + } + + // Membership may be part of this batch, so apply everything before deriving the forwarding tree. + var membershipSnapshots = _membership.CurrentSnapshots; + PrunePeerNamespaceConfirmations(membershipSnapshots.AllMembers); + await _broadcastQueue.Prune(membershipSnapshots, cancellationToken); + // Changed state always wakes children, including same-version liveness updates. Duplicate deliveries + // wake only children which still need this version and have no equivalent queued work. + foreach (var (disseminationNamespace, keys) in receivedKeys) + { + var membership = membershipSnapshots.GetSnapshot(disseminationNamespace.MembershipScope); + foreach (var (key, applied) in keys) + { + if (disseminationNamespace.GetVersion(key) <= 0) + { + continue; + } + + foreach (var peer in membership.ForwardingTreeTargets) + { + if (!Equals(peer, batch.Sender)) + { + _broadcastQueue.Notify(peer, disseminationNamespace, key, force: applied); + } + } + } + } + + // Once downstream work is queued, report the versions this receiver actually holds. + var acknowledgments = new Dictionary>(); + foreach (var (disseminationNamespace, keys) in receivedKeys) + { + var namespaceAcknowledgments = new List(keys.Count); + foreach (var key in keys.Keys) + { + namespaceAcknowledgments.Add(new DigestEntry(key, disseminationNamespace.GetVersion(key))); + } + + acknowledgments.Add(disseminationNamespace.Name, namespaceAcknowledgments); + } + + return new DisseminationBroadcastResponse + { + Acknowledgments = acknowledgments, + UnsupportedNamespaces = unsupportedNamespaces, + }; + } + + public async Task RunAntiEntropyRound(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var options = _options.CurrentValue; + if (!options.Enabled) + { + return; + } + + var membershipSnapshots = _membership.CurrentSnapshots; + PrunePeerNamespaceConfirmations(membershipSnapshots.AllMembers); + await _broadcastQueue.Prune(membershipSnapshots, cancellationToken); + + // A round never queues behind prior rounds: busy destinations and slots wait for a future rotation. + var leases = AcquireAntiEntropyPeers(membershipSnapshots, options.Overlay.AntiEntropyPeerCount); + try + { + if (leases.Count == 0) + { + return; + } + + using var roundCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + _antiEntropyShutdown.Token); + var roundCancellationToken = roundCancellation.Token; + roundCancellationToken.ThrowIfCancellationRequested(); + // Push traffic suppresses redundant checks; admitted peers share this round's digest snapshot. + var requestDigests = CreateAntiEntropyRequestDigests(_timeProvider.GetTimestamp()); + var requests = CreateAntiEntropyRequests(membershipSnapshots, requestDigests, leases.Keys); + if (requests.Count == 0) + { + return; + } + + var roundLifetime = GetAntiEntropyRoundLifetime(requests.Values); + using var lifetimeCancellation = new CancellationTokenSource(roundLifetime, _timeProvider); + using var exchangeCancellation = CancellationTokenSource.CreateLinkedTokenSource( + roundCancellationToken, + lifetimeCancellation.Token); + var responseTasks = new List>(requests.Count); + DisseminationAntiEntropyResponse?[] responses; + try + { + foreach (var (peer, request) in requests) + { + var responseTask = ExchangeAntiEntropyRequest( + peer, + request, + leases[peer], + GetDigestCount(request.Digests), + exchangeCancellation.Token, + roundCancellationToken, + lifetimeCancellation.Token); + leases.Remove(peer); + responseTasks.Add(responseTask); + } + + // Each exchange bounds its local wait, so all leases are released before the round completes. + responses = await Task.WhenAll(responseTasks); + } + finally + { + // The local wait can win cancellation before the linked transport source is signaled. + await exchangeCancellation.CancelAsync(); + } + + roundCancellationToken.ThrowIfCancellationRequested(); + await ApplyAntiEntropyResponses(responses, options, roundCancellationToken); + } + finally + { + foreach (var lease in leases.Values) + { + lease.Dispose(); + } + } + } + + private Dictionary AcquireAntiEntropyPeers( + DisseminationMembershipSnapshots membershipSnapshots, + int peerCount) + { + // AllMembers is a superset of ActiveMembers, preserving a single budget across namespace scopes. + var selectionScope = _namespaces.Values.Any(static disseminationNamespace => + disseminationNamespace.Options.Enabled + && disseminationNamespace.MembershipScope == DisseminationMembershipScope.AllMembers) + ? DisseminationMembershipScope.AllMembers + : DisseminationMembershipScope.ActiveMembers; + var result = new Dictionary(); + foreach (var peer in membershipSnapshots.GetSnapshot(selectionScope).SelectAntiEntropyPeers(peerCount)) + { + if (_antiEntropySendGate.TryAcquire(peer, out var lease)) + { + result.Add(peer, lease); + } + } + + return result; + } + + private Dictionary CreateAntiEntropyRequests( + DisseminationMembershipSnapshots membershipSnapshots, + Dictionary> requestDigests, + IEnumerable peers) + { + var result = new Dictionary(); + var namespacesByScope = new Dictionary(); + foreach (var scope in Enum.GetValues()) + { + var scopedNamespaces = _namespaces.Values + .Where(disseminationNamespace => disseminationNamespace.Options.Enabled + && disseminationNamespace.MembershipScope == scope) + .ToArray(); + if (scopedNamespaces.Length > 0) + { + namespacesByScope.Add(scope, scopedNamespaces); + } + } + + if (namespacesByScope.Count == 0) + { + return result; + } + + foreach (var peer in peers) + { + var peerDigests = new Dictionary>(); + var supportedNamespaces = new List(); + foreach (var (scope, scopedNamespaces) in namespacesByScope) + { + if (!membershipSnapshots.GetSnapshot(scope).ContainsMember(peer)) + { + continue; + } + + foreach (var disseminationNamespace in scopedNamespaces) + { + supportedNamespaces.Add(disseminationNamespace.Name); + if (requestDigests.TryGetValue(disseminationNamespace.Name, out var digest)) + { + peerDigests.Add(disseminationNamespace.Name, digest); + } + } + } + + if (supportedNamespaces.Count > 0) + { + result.Add(peer, new DisseminationAntiEntropyRequest + { + Sender = _localSilo, + Digests = peerDigests, + SupportedNamespaces = supportedNamespaces, + }); + } + } + + return result; + } + + private TimeSpan GetAntiEntropyRoundLifetime( + IEnumerable requests) + { + var result = MaxAntiEntropyRoundLifetime; + foreach (var request in requests) + { + foreach (var namespaceName in request.SupportedNamespaces) + { + if (_namespaces.TryGetValue(namespaceName, out var disseminationNamespace) + && disseminationNamespace.Options.StaleItemTtl < result) + { + result = disseminationNamespace.Options.StaleItemTtl; + } + } + } + + return result; + } + + private Dictionary> CreateAntiEntropyRequestDigests(long now) + { + // New or quiet streams need periodic repair; this pass also forgets streams which disappeared. + Dictionary lastValueUpdates; + lock (_valueUpdateLock) + { + lastValueUpdates = new(_lastValueUpdates); + } + + Dictionary>? digestsByNamespace = null; + var currentValueStreams = new HashSet(); + + foreach (var disseminationNamespace in _namespaces.Values) + { + var namespaceOptions = disseminationNamespace.Options; + if (!namespaceOptions.Enabled) + { + continue; + } + + List? digestEntries = null; + foreach (var digest in disseminationNamespace.Digests) + { + var digestKey = new DigestKey(disseminationNamespace.Name, digest.Key); + currentValueStreams.Add(digestKey); + + if (!lastValueUpdates.TryGetValue(digestKey, out var lastUpdate) + || lastUpdate.Version != digest.Version + || _timeProvider.GetElapsedTime(lastUpdate.Timestamp, now) >= namespaceOptions.ExpectedUpdateCadence) + { + (digestEntries ??= []).Add(digest); + } + } + + if (digestEntries is not null) + { + (digestsByNamespace ??= [])[disseminationNamespace.Name] = digestEntries; + } + } + + lock (_valueUpdateLock) + { + List? removedKeys = null; + foreach (var key in _lastValueUpdates.Keys) + { + if (!currentValueStreams.Contains(key)) + { + (removedKeys ??= []).Add(key); + } + } + + if (removedKeys is not null) + { + foreach (var key in removedKeys) + { + _lastValueUpdates.Remove(key); + } + } + } + + return digestsByNamespace ?? []; + } + + private async Task ExchangeAntiEntropyRequest( + SiloAddress peer, + DisseminationAntiEntropyRequest request, + DisseminationSendGate.Lease lease, + int requestDigestCount, + CancellationToken cancellationToken, + CancellationToken callerCancellationToken, + CancellationToken lifetimeCancellationToken) + { + try + { + cancellationToken.ThrowIfCancellationRequested(); + var exchangeTask = _grainFactory.GetSystemTarget(Constants.DisseminationSystemTargetType, peer) + .ExchangeAntiEntropy(request, cancellationToken); + exchangeTask.Ignore(); + var response = await exchangeTask.WaitAsync(cancellationToken).ConfigureAwait(false); + // Record exchange metrics after the peer returns so truncation and repair counts reflect the response. + DisseminationInstruments.OnAntiEntropyExchange( + "out", + requestDigestCount, + GetValueCount(response.Values), + response.Truncated); + return response; + } + catch (OperationCanceledException) when (callerCancellationToken.IsCancellationRequested) + { + return null; + } + catch (OperationCanceledException) when (lifetimeCancellationToken.IsCancellationRequested) + { + DisseminationInstruments.OnAntiEntropyFailure(DisseminationFailureReason.Timeout); + return null; + } + catch (Exception exception) + { + // Anti-entropy transport failures are isolated to the peer; random peer selection naturally spreads retries. + DisseminationInstruments.OnAntiEntropyFailure(DisseminationFailureReason.Error); + LogDebugDisseminationSendFailed(_logger, exception, peer); + return null; + } + finally + { + lease.Dispose(); + EmitAntiEntropyAdmissionReleased(peer); + } + } + + private void EmitAntiEntropyAdmissionReleased(SiloAddress peer) + { + try + { + DisseminationEvents.EmitSendGate(_localSilo, peer, kind: "repair", stage: "released"); + } + catch (Exception exception) + { + LogDebugDisseminationAdmissionDiagnosticFailed(_logger, exception, peer); + } + } + + private async Task ApplyAntiEntropyResponses( + DisseminationAntiEntropyResponse?[] responses, + DisseminationOptions options, + CancellationToken cancellationToken) + { + // Keep each sender's chain intact and rank all repairs which completed within the round's hop lifetime. + Dictionary>? repairs = null; + foreach (var response in responses) + { + if (response is null) + { + continue; + } + + ConfirmPeerNamespaces(response.Sender, response.SupportedNamespaces); + RevokePeerNamespaces(response.Sender, response.UnsupportedNamespaces); + // A response only includes namespaces which produced repairs. Absence is not evidence that an + // up-to-date or unrelated namespace is unsupported, so confirmations are additive here. + ConfirmPeerNamespaces(response.Sender, response.Values.Keys); + foreach (var (namespaceName, values) in response.Values) + { + if (!TryGetEnabledNamespace(namespaceName, out var disseminationNamespace)) + { + continue; + } + + foreach (var stream in values.GroupBy(static item => item.Value.Key)) + { + repairs ??= []; + var items = stream.ToList(); + var key = new DigestKey(namespaceName, stream.Key); + if (!repairs.TryGetValue(key, out var candidates)) + { + candidates = []; + repairs.Add(key, candidates); + } + + var terminalVersion = items.Max(static item => item.Value.ToVersion); + _broadcastQueue.ObservePeerVersion( + response.Sender, + namespaceName, + stream.Key, + terminalVersion); + candidates.Add(new(disseminationNamespace, items, response.Sender)); + } + } + } + + if (repairs is null) + { + return; + } + + foreach (var candidates in repairs.Values) + { + // Try the furthest-reaching repair first, preferring a full value when candidates tie. + candidates.Sort(CompareAntiEntropyRepairs); + + foreach (var candidate in candidates) + { + foreach (var item in candidate.Items) + { + await ApplyReceivedValue( + candidate.Namespace, + item, + candidate.Sender, + options, + cancellationToken); + } + } + } + } + + public ValueTask ReceiveAntiEntropy( + DisseminationAntiEntropyRequest request, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ConfirmPeerNamespaces(request.Sender, request.SupportedNamespaces); + ConfirmPeerNamespaces(request.Sender, request.Digests.Keys); + // Incoming digests are passive evidence for existing peer pumps, not a reason to create new ones. + foreach (var (namespaceName, entries) in request.Digests) + { + foreach (var entry in entries) + { + _broadcastQueue.ObservePeerVersion( + request.Sender, + namespaceName, + entry.Key, + entry.Version); + } + } + + var options = _options.CurrentValue; + var response = CreateAntiEntropyResponse(request, options, cancellationToken); + PruneAntiEntropyResponseCursors(_membership.CurrentSnapshots.AllMembers, request.Sender); + return new(response); + } + + private DisseminationAntiEntropyResponse CreateAntiEntropyResponse( + DisseminationAntiEntropyRequest request, + DisseminationOptions options, + CancellationToken cancellationToken) + { + if (!options.Enabled) + { + return new DisseminationAntiEntropyResponse + { + Sender = _localSilo, + Values = [], + Truncated = false, + UnsupportedNamespaces = [.. request.SupportedNamespaces], + }; + } + + var supportedNamespaces = new List(); + var unsupportedNamespaces = new List(); + foreach (var namespaceName in request.SupportedNamespaces) + { + if (TryGetEnabledNamespace(namespaceName, out _)) + { + supportedNamespaces.Add(namespaceName); + } + else + { + unsupportedNamespaces.Add(namespaceName); + } + } + + var valueCount = 0; + var byteCount = 0; + var truncated = false; + var valuesByNamespace = new Dictionary>(); + var candidates = new List(); + foreach (var (namespaceName, remoteDigest) in request.Digests) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!TryGetEnabledNamespace(namespaceName, out var requestedNamespace)) + { + continue; + } + + if (remoteDigest.Count == 0) + { + continue; + } + + var remoteVersions = CreateDigestLookup(remoteDigest); + foreach (var localDigest in requestedNamespace.Digests) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!remoteVersions.TryGetValue(localDigest.Key, out var peerDigest)) + { + continue; + } + + if (localDigest.Version < peerDigest.Version + || localDigest.Version == peerDigest.Version + && localDigest.Fingerprint == peerDigest.Fingerprint) + { + continue; + } + + candidates.Add(new(requestedNamespace, localDigest, peerDigest)); + } + } + + var start = GetAntiEntropyResponseStart(request.Sender, candidates.Count); + var examined = 0; + for (var i = 0; i < candidates.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + var candidate = candidates[(start + i) % candidates.Count]; + examined++; + var requestedNamespace = candidate.Namespace; + var localDigest = candidate.LocalDigest; + var peerDigest = candidate.PeerDigest; + var repairRequest = new DisseminationRepairRequest( + localDigest.Key, + peerDigest.Version, + toVersion: null, + options.MaxBatchItems - valueCount, + options.MaxBatchBytes - byteCount, + requestedNamespace.Options.MaxPayloadBytes); + var repair = requestedNamespace.CreateRepair(repairRequest); + if (repair.Status is DisseminationRepairStatus.InsufficientCapacity) + { + if (valueCount > 0) + { + // Probe with a fresh batch budget to distinguish truncation from a permanently oversized key. + var emptyBatchRequest = new DisseminationRepairRequest( + localDigest.Key, + peerDigest.Version, + toVersion: null, + options.MaxBatchItems, + options.MaxBatchBytes, + requestedNamespace.Options.MaxPayloadBytes); + var emptyBatchRepair = requestedNamespace.CreateRepair(emptyBatchRequest); + if (emptyBatchRepair.Status is DisseminationRepairStatus.Produced + && ValidateRepair( + requestedNamespace, + emptyBatchRequest, + emptyBatchRepair, + options)) + { + // Resume with this candidate next round because the current response budget, not the + // candidate itself, prevented it from being included. + examined--; + truncated = true; + break; + } + } + + continue; + } + + if (repair.Status is not DisseminationRepairStatus.Produced + || !ValidateRepair(requestedNamespace, repairRequest, repair, options)) + { + continue; + } + + if (!valuesByNamespace.TryGetValue(requestedNamespace.Name, out var namespaceValues)) + { + namespaceValues = []; + valuesByNamespace.Add(requestedNamespace.Name, namespaceValues); + } + + foreach (var value in repair.Values) + { + namespaceValues.Add(CreateBroadcastValue(requestedNamespace, value)); + ++valueCount; + byteCount += value.Payload.Length; + } + + if (!repair.IsComplete) + { + // A valid prefix consumes this response; the caller can continue in its next round. + truncated = true; + break; + } + } + + if (truncated) + { + AdvanceAntiEntropyResponseCursor(request.Sender, start, examined, candidates.Count); + } + else + { + ClearAntiEntropyResponseCursor(request.Sender); + } + + DisseminationInstruments.OnAntiEntropyExchange("in", GetDigestCount(request.Digests), valueCount, truncated); + return new DisseminationAntiEntropyResponse + { + Sender = _localSilo, + Values = valuesByNamespace, + Truncated = truncated, + SupportedNamespaces = supportedNamespaces, + UnsupportedNamespaces = unsupportedNamespaces, + }; + } + + private int GetAntiEntropyResponseStart(SiloAddress peer, int candidateCount) + { + if (candidateCount == 0) + { + return 0; + } + + lock (_antiEntropyResponseCursorLock) + { + if (!_antiEntropyResponseCursors.TryGetValue(peer, out var cursor)) + { + return 0; + } + + _antiEntropyResponseCursors[peer] = cursor with + { + LastAccess = ++_antiEntropyResponseCursorAccess, + }; + return cursor.Position % candidateCount; + } + } + + private void AdvanceAntiEntropyResponseCursor( + SiloAddress peer, + int start, + int examined, + int candidateCount) + { + lock (_antiEntropyResponseCursorLock) + { + if (candidateCount == 0) + { + _antiEntropyResponseCursors.Remove(peer); + } + else + { + _antiEntropyResponseCursors[peer] = new( + (start + Math.Max(1, examined)) % candidateCount, + ++_antiEntropyResponseCursorAccess); + } + } + } + + private void ClearAntiEntropyResponseCursor(SiloAddress peer) + { + lock (_antiEntropyResponseCursorLock) + { + _antiEntropyResponseCursors.Remove(peer); + } + } + + private void PruneAntiEntropyResponseCursors( + DisseminationMembershipSnapshot membership, + SiloAddress currentRequester) + { + lock (_antiEntropyResponseCursorLock) + { + var nonMemberCount = 0; + foreach (var peer in _antiEntropyResponseCursors.Keys) + { + if (!membership.ContainsMember(peer)) + { + nonMemberCount++; + } + } + + if (nonMemberCount <= MaxRetainedNonMemberResponseCursors) + { + return; + } + + foreach (var cursor in _antiEntropyResponseCursors + .Where(entry => !Equals(entry.Key, currentRequester) && !membership.ContainsMember(entry.Key)) + .OrderBy(static entry => entry.Value.LastAccess) + .ToArray()) + { + _antiEntropyResponseCursors.Remove(cursor.Key); + if (--nonMemberCount <= MaxRetainedNonMemberResponseCursors) + { + break; + } + } + } + } + + private async ValueTask ApplyReceivedValue( + IDisseminationNamespace disseminationNamespace, + DisseminationBroadcastValue item, + SiloAddress sender, + DisseminationOptions options, + CancellationToken cancellationToken) + { + try + { + return await ApplyReceivedValueCore( + disseminationNamespace, + item, + sender, + options, + cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + LogDebugDisseminationValueApplyFailed( + _logger, + exception, + sender, + disseminationNamespace.Name, + item.Value.Key, + item.Value.ToVersion); + EmitApplyResult(disseminationNamespace.Name, item, sender, DisseminationApplyResult.Rejected); + return DisseminationApplyResult.Rejected; + } + } + + private async ValueTask ApplyReceivedValueCore( + IDisseminationNamespace disseminationNamespace, + DisseminationBroadcastValue item, + SiloAddress sender, + DisseminationOptions options, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var namespaceName = disseminationNamespace.Name; + if (!ValidatePayloadSize(disseminationNamespace, item.Value, options)) + { + return DisseminationApplyResult.Rejected; + } + + if (IsExpired(item)) + { + EmitApplyResult(namespaceName, item, sender, DisseminationApplyResult.Obsolete); + return DisseminationApplyResult.Obsolete; + } + + // Reject gaps before deserializing; full values from version zero can replace any baseline. + if (TryGetTerminalApplyResult(disseminationNamespace, item.Value, out var terminalResult)) + { + EmitApplyResult(namespaceName, item, sender, terminalResult); + return terminalResult; + } + + var result = await disseminationNamespace.ApplyValueAsync(item.Value, cancellationToken); + EmitApplyResult(namespaceName, item, sender, result); + if (result is DisseminationApplyResult.Applied) + { + RecordValueUpdate(namespaceName, item.Value.Key, item.Value.ToVersion); + } + + return result; + } + + internal Task FlushPendingBroadcast(CancellationToken cancellationToken) => + _broadcastQueue.FlushPendingBroadcast(cancellationToken); + + internal async Task StopAsync(CancellationToken cancellationToken) + { + _antiEntropySendGate.Stop(); + await _antiEntropyShutdown.CancelAsync(); + await _broadcastQueue.StopAsync(cancellationToken); + } + + internal IReadOnlyList GetUnconfirmedPeers( + IDisseminationNamespace disseminationNamespace) + { + var membershipSnapshots = _membership.CurrentSnapshots; + PrunePeerNamespaceConfirmations(membershipSnapshots.AllMembers); + var participants = membershipSnapshots.GetSnapshot(disseminationNamespace.MembershipScope).Members; + lock (_peerSupportLock) + { + List? result = null; + foreach (var peer in participants) + { + if (Equals(peer, _localSilo)) + { + continue; + } + + if (!_confirmedPeerNamespaces.TryGetValue(peer, out var namespaces) + || !namespaces.ContainsKey(disseminationNamespace.Name)) + { + (result ??= []).Add(peer); + } + } + + return result ?? []; + } + } + + private void ObserveBroadcastResponse( + SiloAddress peer, + DisseminationBroadcastResponse response) + { + ConfirmPeerNamespaces(peer, response.Acknowledgments.Keys); + RevokePeerNamespaces(peer, response.UnsupportedNamespaces); + } + + private void ConfirmPeerNamespaces( + SiloAddress peer, + IEnumerable namespaceNames) + { + if (Equals(peer, _localSilo) + || !_membership.CurrentSnapshots.AllMembers.ContainsMember(peer)) + { + return; + } + + var now = _timeProvider.GetTimestamp(); + lock (_peerSupportLock) + { + Dictionary? confirmedNamespaces = null; + foreach (var namespaceName in namespaceNames) + { + if (!_namespaces.TryGetValue(namespaceName, out var disseminationNamespace) + || !disseminationNamespace.Options.Enabled) + { + continue; + } + + if (confirmedNamespaces is null + && !_confirmedPeerNamespaces.TryGetValue(peer, out confirmedNamespaces)) + { + confirmedNamespaces = []; + _confirmedPeerNamespaces.Add(peer, confirmedNamespaces); + } + + confirmedNamespaces![namespaceName] = now; + } + } + } + + private void RevokePeerNamespaces( + SiloAddress peer, + IEnumerable namespaceNames) + { + lock (_peerSupportLock) + { + if (!_confirmedPeerNamespaces.TryGetValue(peer, out var confirmedNamespaces)) + { + return; + } + + foreach (var namespaceName in namespaceNames) + { + confirmedNamespaces.Remove(namespaceName); + } + + if (confirmedNamespaces.Count == 0) + { + _confirmedPeerNamespaces.Remove(peer); + } + } + } + + private void PrunePeerNamespaceConfirmations(DisseminationMembershipSnapshot membership) + { + lock (_peerSupportLock) + { + foreach (var peer in _confirmedPeerNamespaces.Keys + .Where(peer => !membership.ContainsMember(peer)) + .ToArray()) + { + _confirmedPeerNamespaces.Remove(peer); + } + } + } + + private void RecordValueUpdate(DisseminationNamespace namespaceName, DisseminationKey key, long version) + { + // Recent successful updates suppress anti-entropy until the namespace's expected cadence elapses. + var digestKey = new DigestKey(namespaceName, key); + var update = new ValueUpdate(version, _timeProvider.GetTimestamp()); + lock (_valueUpdateLock) + { + if (!_lastValueUpdates.TryGetValue(digestKey, out var previous) || version > previous.Version) + { + _lastValueUpdates[digestKey] = update; + } + } + } + + private async ValueTask GetMembershipSnapshotForRouting( + DisseminationMembershipScope scope, + SiloAddress member, + CancellationToken cancellationToken) + { + // Prune peer ledgers against the same membership view used to choose tree targets. + var memberships = await _membership.GetSnapshotsContainingMember(member, scope, cancellationToken); + await _broadcastQueue.Prune(memberships ?? _membership.CurrentSnapshots, cancellationToken); + return memberships?.GetSnapshot(scope); + } + + private bool TryGetEnabledNamespace(DisseminationNamespace namespaceName, [NotNullWhen(true)] out IDisseminationNamespace? disseminationNamespace) + { + if (_namespaces.TryGetValue(namespaceName, out disseminationNamespace) + && disseminationNamespace.Options.Enabled) + { + return true; + } + + disseminationNamespace = null; + return false; + } + + private bool ValidatePayloadSize( + IDisseminationNamespace disseminationNamespace, + DisseminationValue value, + DisseminationOptions options) + { + if (value.Payload.Length <= disseminationNamespace.Options.MaxPayloadBytes + && value.Payload.Length <= options.MaxBatchBytes) + { + return true; + } + + var namespaceName = disseminationNamespace.Name; + DisseminationEvents.EmitPayloadDrop(namespaceName, value, _localSilo, "oversize", value.Payload.Length); + DisseminationInstruments.OnPayloadDropped(namespaceName, "oversize"); + return false; + } + + private bool TryValidatePublish( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + DisseminationOptions options, + out long publishedVersion, + [NotNullWhen(false)] out string? failureReason) + { + if (version <= 0) + { + publishedVersion = 0; + failureReason = "invalid-version"; + return false; + } + + if (disseminationNamespace.GetVersion(key) < version) + { + publishedVersion = 0; + failureReason = "unavailable"; + return false; + } + + // Publishing is safe only if the namespace can materialize a complete repair for an unknown peer. + var request = new DisseminationRepairRequest( + key, + fromVersion: null, + toVersion: null, + maxItemCount: int.MaxValue, + maxBatchBytes: int.MaxValue, + disseminationNamespace.Options.MaxPayloadBytes); + var repair = disseminationNamespace.CreateRepair(request); + if (repair.Status is DisseminationRepairStatus.InsufficientCapacity) + { + publishedVersion = 0; + failureReason = "oversize"; + return false; + } + + if (repair.Status is not DisseminationRepairStatus.Produced + || !repair.IsComplete + || repair.Version < version + || !ValidateRepair(disseminationNamespace, request, repair, options)) + { + publishedVersion = 0; + failureReason = "invalid-repair"; + return false; + } + + publishedVersion = repair.Version; + failureReason = null; + return true; + } + + private bool ValidateRepair( + IDisseminationNamespace disseminationNamespace, + in DisseminationRepairRequest request, + in DisseminationRepairResult repair, + DisseminationOptions options) + { + // Keep namespace-specific serialization behind one common range and budget contract. + if (repair.Status is not DisseminationRepairStatus.Produced + || repair.Values.IsDefaultOrEmpty + || repair.Version <= 0 + || request.ToVersion is { } requestedToVersion && repair.Version > requestedToVersion + || repair.Values.Length > request.MaxItemCount) + { + return false; + } + + var byteCount = 0; + var expectedFromVersion = request.FromVersion; + foreach (var value in repair.Values) + { + if (value.Key != request.Key + || !IsValidVersionRange(value) + || value.ToVersion > repair.Version + || !ValidatePayloadSize(disseminationNamespace, value, options)) + { + return false; + } + + if (expectedFromVersion is null && value.FromVersion != 0 + || expectedFromVersion is { } fromVersion + && value.FromVersion != 0 + && value.FromVersion != fromVersion) + { + return false; + } + + expectedFromVersion = value.ToVersion; + byteCount += value.Payload.Length; + if (byteCount > request.MaxBatchBytes) + { + return false; + } + } + + var lastVersion = repair.Values[^1].ToVersion; + return repair.IsComplete ? lastVersion == repair.Version : lastVersion < repair.Version; + } + + private static bool IsValidVersionRange(DisseminationValue value) => + value is { FromVersion: >= 0, ToVersion: > 0 } && value.ToVersion > value.FromVersion; + + private static bool TryGetTerminalApplyResult( + IDisseminationNamespace disseminationNamespace, + DisseminationValue value, + out DisseminationApplyResult result) + { + if (!IsValidVersionRange(value)) + { + result = DisseminationApplyResult.Rejected; + return true; + } + + var localVersion = disseminationNamespace.GetVersion(value.Key); + if (value.ToVersion < localVersion) + { + result = DisseminationApplyResult.Obsolete; + return true; + } + + if (value.ToVersion == localVersion || value.FromVersion == 0 || value.FromVersion == localVersion) + { + result = default; + return false; + } + + result = DisseminationApplyResult.Rejected; + return true; + } + + private bool IsExpired(DisseminationBroadcastValue item) => + item.TimeToLive <= TimeSpan.Zero; + + private DisseminationBroadcastValue CreateBroadcastValue( + IDisseminationNamespace disseminationNamespace, + DisseminationValue value) => + new() + { + Value = value, + TimeToLive = disseminationNamespace.Options.StaleItemTtl, + }; + + private void EmitApplyResult(DisseminationNamespace namespaceName, DisseminationBroadcastValue item, SiloAddress sender, DisseminationApplyResult result) + { + try + { + DisseminationEvents.EmitValue(namespaceName, item.Value, _localSilo, sender, result, item.Value.Payload.Length); + } + catch (Exception exception) + { + LogDebugDisseminationDiagnosticFailed(_logger, exception, namespaceName, item.Value.Key); + } + + DisseminationInstruments.OnValueApplied(namespaceName, result); + } + + private static int GetDigestCount(Dictionary> digest) => digest.Values.Sum(entries => entries.Count); + + private static Dictionary CreateDigestLookup(List digest) + { + var result = new Dictionary(digest.Count); + foreach (var entry in digest) + { + result[entry.Key] = entry; + } + + return result; + } + + private static int GetValueCount(Dictionary> valuesByNamespace) => valuesByNamespace.Values.Sum(values => values.Count); + + // Prefer the highest terminal version, then a universal full value, then stable peer order. + private static int CompareAntiEntropyRepairs(AntiEntropyRepair left, AntiEntropyRepair right) + { + var result = right.Items[^1].Value.ToVersion.CompareTo(left.Items[^1].Value.ToVersion); + if (result != 0) + { + return result; + } + + var leftIsFullValue = left.Items[0].Value.FromVersion == 0; + var rightIsFullValue = right.Items[0].Value.FromVersion == 0; + if (leftIsFullValue != rightIsFullValue) + { + return leftIsFullValue ? -1 : 1; + } + + return left.Sender.CompareTo(right.Sender); + } + + private readonly record struct DigestKey(DisseminationNamespace Namespace, DisseminationKey Key); + + private readonly record struct ValueUpdate(long Version, long Timestamp); + + private readonly record struct AntiEntropyResponseCursor(int Position, long LastAccess); + + private readonly record struct AntiEntropyRepair( + IDisseminationNamespace Namespace, + List Items, + SiloAddress Sender); + + private readonly record struct AntiEntropyResponseCandidate( + IDisseminationNamespace Namespace, + DigestEntry LocalDigest, + DigestEntry PeerDigest); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination send to {Peer} failed.")] + private static partial void LogDebugDisseminationSendFailed(ILogger logger, Exception exception, SiloAddress peer); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination admission diagnostic for {Peer} failed.")] + private static partial void LogDebugDisseminationAdmissionDiagnosticFailed(ILogger logger, Exception exception, SiloAddress peer); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Failed to apply dissemination value from {Sender} for namespace {Namespace}, key {Key}, version {Version}.")] + private static partial void LogDebugDisseminationValueApplyFailed(ILogger logger, Exception exception, SiloAddress sender, DisseminationNamespace @namespace, DisseminationKey key, long version); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination apply diagnostic failed for namespace {Namespace}, key {Key}.")] + private static partial void LogDebugDisseminationDiagnosticFailed( + ILogger logger, + Exception exception, + DisseminationNamespace @namespace, + DisseminationKey key); + +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationSendGate.cs b/src/Orleans.Runtime/Dissemination/DisseminationSendGate.cs new file mode 100644 index 00000000000..fa143c95a95 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationSendGate.cs @@ -0,0 +1,151 @@ +using System.Diagnostics.CodeAnalysis; + +namespace Orleans.Runtime.Dissemination; + +internal sealed class DisseminationSendGate(int maxConcurrency) +{ + private readonly object _lock = new(); + private readonly HashSet _activePeers = []; + private readonly LinkedList _requests = []; + private bool _stopped; + + public ValueTask AcquireAsync(SiloAddress peer, CancellationToken cancellationToken) => + AcquireAsync(peer, onQueued: null, cancellationToken); + + public ValueTask AcquireAsync(SiloAddress peer, Action? onQueued, CancellationToken cancellationToken) + { + Request request; + lock (_lock) + { + cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_stopped, this); + // Release fills available slots before unlocking, so any remaining queued peers are busy. + if (_activePeers.Count < maxConcurrency && _activePeers.Add(peer)) + { + return ValueTask.FromResult(new Lease(this, peer)); + } + + request = new(this, peer); + request.Node = _requests.AddLast(request); + } + + return WaitAsync(request, onQueued, cancellationToken); + } + + public bool TryAcquire(SiloAddress peer, [NotNullWhen(true)] out Lease? lease) + { + lock (_lock) + { + if (!_stopped && _activePeers.Count < maxConcurrency && _activePeers.Add(peer)) + { + lease = new(this, peer); + return true; + } + } + + lease = null; + return false; + } + + public void Stop() + { + lock (_lock) + { + _stopped = true; + while (_requests.First is { } node) + { + _requests.Remove(node); + node.Value.Completion.TrySetCanceled(); + } + } + } + + private static async ValueTask WaitAsync(Request request, Action? onQueued, CancellationToken cancellationToken) + { + using var registration = cancellationToken.UnsafeRegister( + static (state, token) => + { + var request = (Request)state!; + request.Owner.Cancel(request, token); + }, + request); + try + { + onQueued?.Invoke(); + } + catch + { + // A diagnostic failure must not leave an orphaned request or a concurrently granted lease. + request.Owner.Cancel(request, cancellationToken); + if (request.Completion.Task.IsCompletedSuccessfully) + { + request.Completion.Task.Result.Dispose(); + } + + throw; + } + + var lease = await request.Completion.Task.ConfigureAwait(false); + // A grant can win the lock just before cancellation removes its request. + if (cancellationToken.IsCancellationRequested) + { + lease.Dispose(); + cancellationToken.ThrowIfCancellationRequested(); + } + + return lease; + } + + private void Cancel(Request request, CancellationToken cancellationToken) + { + lock (_lock) + { + if (request.Node is { List: not null } node) + { + _requests.Remove(node); + request.Completion.TrySetCanceled(cancellationToken); + } + } + } + + private void Release(SiloAddress peer) + { + lock (_lock) + { + _activePeers.Remove(peer); + if (_stopped) + { + return; + } + + // FIFO among ready destinations: skip peers which already have an active local attempt. + var node = _requests.First; + while (node is not null && _activePeers.Count < maxConcurrency) + { + var next = node.Next; + if (_activePeers.Add(node.Value.Peer)) + { + _requests.Remove(node); + node.Value.Completion.SetResult(new Lease(this, node.Value.Peer)); + } + + node = next; + } + } + } + + private sealed class Request(DisseminationSendGate owner, SiloAddress peer) + { + public DisseminationSendGate Owner { get; } = owner; + public SiloAddress Peer { get; } = peer; + public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public LinkedListNode? Node { get; set; } + } + + internal sealed class Lease(DisseminationSendGate owner, SiloAddress peer) : IDisposable + { + private DisseminationSendGate? _owner = owner; + + public void Dispose() => Interlocked.Exchange(ref _owner, null)?.Release(peer); + } +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs b/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs new file mode 100644 index 00000000000..82052c59bba --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs @@ -0,0 +1,145 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime.Scheduler; + +namespace Orleans.Runtime.Dissemination; + +internal sealed partial class DisseminationSystemTarget : SystemTarget, IDisseminationSystemTarget, IDisseminationService, ILifecycleParticipant +{ + private readonly DisseminationProtocol _protocol; + private readonly IOptionsMonitor _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly CancellationTokenSource _shutdownCts = new(); + private Task? _antiEntropyTask; + + public DisseminationSystemTarget( + ILocalSiloDetails localSiloDetails, + IInternalGrainFactory grainFactory, + DisseminationMembership membership, + IOptionsMonitor options, + IEnumerable disseminationNamespaces, + TimeProvider timeProvider, + ILogger logger, + ILogger broadcastQueueLogger, + SystemTargetShared shared) + : base(Constants.DisseminationSystemTargetType, shared) + { + _options = options; + _timeProvider = timeProvider; + _logger = logger; + _protocol = new DisseminationProtocol(localSiloDetails, grainFactory, membership, options, disseminationNamespaces, timeProvider, logger, broadcastQueueLogger); + shared.ActivationDirectory.RecordNewTarget(this); + } + + async ValueTask IDisseminationService.Publish( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken) => + await this.RunOrQueueTask(token => _protocol.Publish( + disseminationNamespace, + key, + version, + token).AsTask(), cancellationToken); + + IReadOnlyList IDisseminationService.GetUnconfirmedPeers( + IDisseminationNamespace disseminationNamespace) => + _protocol.GetUnconfirmedPeers(disseminationNamespace); + + Task IDisseminationSystemTarget.PushBroadcast( + DisseminationBroadcastBatch batch, + CancellationToken cancellationToken) => + _protocol.ReceiveBroadcast(batch, cancellationToken); + + async Task IDisseminationSystemTarget.ExchangeAntiEntropy( + DisseminationAntiEntropyRequest request, + CancellationToken cancellationToken) => + await _protocol.ReceiveAntiEntropy(request, cancellationToken); + + void ILifecycleParticipant.Participate(ISiloLifecycle observer) + { + observer.Subscribe( + nameof(DisseminationSystemTarget), + ServiceLifecycleStage.RuntimeServices, + StartAsync, + StopAsync); + } + + private Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_antiEntropyTask is { IsCompleted: false }) + { + return Task.CompletedTask; + } + + var shutdownToken = _shutdownCts.Token; + _antiEntropyTask = this.RunOrQueueTask(() => RunAntiEntropyLoop(shutdownToken)); + return Task.CompletedTask; + } + + private async Task StopAsync(CancellationToken cancellationToken) + { + await _shutdownCts.CancelAsync(); + try + { + await this.RunOrQueueTask(() => _protocol.StopAsync(cancellationToken)); + } + finally + { + // Always observe the loop and release the cancellation source, even if draining the protocol threw. + if (_antiEntropyTask is not null) + { + try + { + await _antiEntropyTask.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) + { + // Expected during silo shutdown. + } + catch (Exception exception) + { + LogDebugAntiEntropyLoopFailed(_logger, exception); + } + + _antiEntropyTask = null; + } + + _shutdownCts.Dispose(); + } + } + + private async Task RunAntiEntropyLoop(CancellationToken cancellationToken) + { + try + { + while (true) + { + await Task.Delay( + _options.CurrentValue.Overlay.AntiEntropyInterval, + _timeProvider, + cancellationToken); + try + { + await _protocol.RunAntiEntropyRound(cancellationToken); + } + catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested) + { + LogDebugAntiEntropyLoopFailed(_logger, exception); + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // Expected during silo shutdown. + } + } + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination anti-entropy loop iteration failed.")] + private static partial void LogDebugAntiEntropyLoopFailed(ILogger logger, Exception exception); +} diff --git a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs new file mode 100644 index 00000000000..9016f7f20eb --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs @@ -0,0 +1,96 @@ +using System.Collections.Immutable; +using Orleans.Configuration; + +namespace Orleans.Runtime.Dissemination; + +// A namespace owns current state plus any history and caching needed to repair a peer from an acknowledged version. +internal interface IDisseminationNamespace +{ + DisseminationNamespace Name { get; } + + DisseminationMembershipScope MembershipScope => DisseminationMembershipScope.AllMembers; + + DisseminationNamespaceOptions Options { get; } + + IEnumerable Digests { get; } + + long GetVersion(DisseminationKey key); + + DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request); + + ValueTask ApplyValueAsync( + DisseminationValue value, + CancellationToken cancellationToken); +} + +internal enum DisseminationMembershipScope +{ + ActiveMembers, + AllMembers, +} + +// A null FromVersion means no known peer baseline; a null ToVersion asks for the highest repairable version. +internal readonly struct DisseminationRepairRequest( + DisseminationKey key, + long? fromVersion, + long? toVersion, + int maxItemCount, + int maxBatchBytes, + int maxPayloadBytes) +{ + public DisseminationKey Key { get; } = key; + + public long? FromVersion { get; } = fromVersion; + + public long? ToVersion { get; } = toVersion; + + public int MaxItemCount { get; } = maxItemCount; + + public int MaxBatchBytes { get; } = maxBatchBytes; + + public int MaxPayloadBytes { get; } = maxPayloadBytes; +} + +// Version reports the namespace's resolved or current version. +// For Produced results, IsComplete says whether Values reaches that version or only forms a prefix. +internal readonly struct DisseminationRepairResult( + DisseminationRepairStatus status, + long version, + ImmutableArray values, + bool isComplete) +{ + public DisseminationRepairStatus Status { get; } = status; + + public long Version { get; } = version; + + public ImmutableArray Values { get; } = values.IsDefault ? [] : values; + + public bool IsComplete { get; } = isComplete; + + public static DisseminationRepairResult Current(long version) => + new(DisseminationRepairStatus.Current, version, [], isComplete: true); + + public static DisseminationRepairResult Produced( + long version, + ImmutableArray values, + bool isComplete = true) => + new(DisseminationRepairStatus.Produced, version, values, isComplete); + + public static DisseminationRepairResult Unavailable(long version) => + new(DisseminationRepairStatus.Unavailable, version, [], isComplete: false); + + public static DisseminationRepairResult InsufficientCapacity(long version) => + new(DisseminationRepairStatus.InsufficientCapacity, version, [], isComplete: false); +} + +internal enum DisseminationRepairStatus +{ + // The peer is already at or beyond the resolved version. + Current, + // Values contains a valid repair, possibly a prefix when IsComplete is false. + Produced, + // The key or requested target cannot currently be reconstructed. + Unavailable, + // No valid repair fits within the supplied item or byte budget. + InsufficientCapacity, +} diff --git a/src/Orleans.Runtime/Dissemination/IDisseminationService.cs b/src/Orleans.Runtime/Dissemination/IDisseminationService.cs new file mode 100644 index 00000000000..19652a5768a --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/IDisseminationService.cs @@ -0,0 +1,13 @@ +namespace Orleans.Runtime.Dissemination; + +// Publishing announces that a version is repairable; serialized payload ownership stays with the namespace. +internal interface IDisseminationService +{ + ValueTask Publish( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken); + + IReadOnlyList GetUnconfirmedPeers(IDisseminationNamespace disseminationNamespace); +} diff --git a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs new file mode 100644 index 00000000000..d3374798532 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs @@ -0,0 +1,482 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime.MembershipService; +using Orleans.Serialization; + +namespace Orleans.Runtime.Dissemination; + +// Membership retains a bounded snapshot history so each peer can receive either a compact diff +// or a universal full snapshot. +internal sealed class MembershipDisseminationNamespace( + IMembershipManager membershipManager, + IOptionsMonitor options, + Serializer serializer) : IDisseminationNamespace +{ + private const int MaxSnapshotHistory = 32; + private readonly object _historyLock = new(); + private readonly SortedDictionary _snapshotHistory = new(); + private readonly Dictionary> _snapshotPayloads = []; + private readonly Dictionary<(long FromVersion, long ToVersion), ReadOnlyMemory> _diffPayloads = []; + + public DisseminationNamespace Name => DisseminationNamespaceNames.Membership; + + public DisseminationMembershipScope MembershipScope => DisseminationMembershipScope.AllMembers; + + public DisseminationNamespaceOptions Options => options.CurrentValue.Dissemination; + + public async ValueTask PublishAsync( + IDisseminationService disseminationService, + MembershipTableSnapshot snapshot, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + // Remember the exact snapshot before waking peer pumps so it is immediately repairable. + RememberSnapshot(snapshot); + return await disseminationService.Publish( + this, + DisseminationKey.Default, + snapshot.Version.Value, + cancellationToken); + } + + public IEnumerable Digests + { + get + { + var snapshot = membershipManager.CurrentSnapshot; + RememberSnapshot(snapshot); + // Version alone misses same-version liveness advances, so the digest fingerprints heartbeat state too. + yield return new DigestEntry( + DisseminationKey.Default, + snapshot.Version.Value, + GetFingerprint(snapshot)); + } + } + + public long GetVersion(DisseminationKey key) => + key == DisseminationKey.Default + ? membershipManager.CurrentSnapshot.Version.Value + : 0; + + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) + { + if (request.Key != DisseminationKey.Default) + { + return DisseminationRepairResult.Unavailable(version: 0); + } + + // Select history and cached bytes atomically because membership can change without advancing its version. + lock (_historyLock) + { + var currentSnapshot = membershipManager.CurrentSnapshot; + RememberSnapshotUnsafe(currentSnapshot); + var targetVersion = request.ToVersion ?? currentSnapshot.Version.Value; + if (targetVersion > currentSnapshot.Version.Value + || !_snapshotHistory.TryGetValue(targetVersion, out var targetSnapshot)) + { + return DisseminationRepairResult.Unavailable(currentSnapshot.Version.Value); + } + + // Prefer a retained peer baseline when available; otherwise the full snapshot remains the fallback. + MembershipTableSnapshot? baseSnapshot = null; + if (request.FromVersion is { } fromVersion + && fromVersion > 0 + && fromVersion < targetVersion) + { + _snapshotHistory.TryGetValue(fromVersion, out baseSnapshot); + } + + var resolvedVersion = targetSnapshot.Version.Value; + if (request.FromVersion is { } peerVersion && peerVersion > resolvedVersion) + { + return DisseminationRepairResult.Current(resolvedVersion); + } + + if (request.MaxItemCount <= 0) + { + return DisseminationRepairResult.InsufficientCapacity(resolvedVersion); + } + + var snapshotValue = CreateSnapshotValue(targetSnapshot); + var selectedValue = snapshotValue; + if (baseSnapshot is not null) + { + // Use the smaller representation, retaining the full snapshot as the capacity fallback. + var diffValue = CreateDiffValue(baseSnapshot, targetSnapshot); + if (diffValue.Payload.Length < snapshotValue.Payload.Length) + { + selectedValue = diffValue; + } + } + + if (selectedValue.Payload.Length > request.MaxPayloadBytes + || selectedValue.Payload.Length > request.MaxBatchBytes) + { + if (selectedValue.FromVersion != 0 + && snapshotValue.Payload.Length <= request.MaxPayloadBytes + && snapshotValue.Payload.Length <= request.MaxBatchBytes) + { + selectedValue = snapshotValue; + } + else + { + return DisseminationRepairResult.InsufficientCapacity(resolvedVersion); + } + } + + return DisseminationRepairResult.Produced(resolvedVersion, [selectedValue]); + } + } + + public async ValueTask ApplyValueAsync( + DisseminationValue value, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (value.Key != DisseminationKey.Default) + { + return DisseminationApplyResult.Rejected; + } + + if (serializer.Deserialize(value.Payload) is not { } update) + { + return DisseminationApplyResult.Rejected; + } + + if (update.Diff is { } diff) + { + return update.Snapshot is null + && value.FromVersion == diff.BaseVersion.Value + && value.ToVersion == diff.Version.Value + ? await ApplyDiff(diff, cancellationToken) + : DisseminationApplyResult.Rejected; + } + + if (update.Snapshot is not { } snapshot) + { + return DisseminationApplyResult.Rejected; + } + + if (value.FromVersion != 0 || value.ToVersion != snapshot.Version.Value) + { + return DisseminationApplyResult.Rejected; + } + + var currentSnapshot = membershipManager.CurrentSnapshot; + if (snapshot.Version < currentSnapshot.Version) + { + return DisseminationApplyResult.Obsolete; + } + + if (snapshot.Version == currentSnapshot.Version) + { + // Same-version snapshots can still advance IAmAlive state. + if (!snapshot.IsSuccessorTo(currentSnapshot)) + { + return DisseminationApplyResult.Duplicate; + } + + await membershipManager.ProcessGossipSnapshot(snapshot, cancellationToken); + RememberSnapshot(membershipManager.CurrentSnapshot); + return DisseminationApplyResult.Applied; + } + + await membershipManager.ProcessGossipSnapshot(snapshot, cancellationToken); + RememberSnapshot(membershipManager.CurrentSnapshot); + return DisseminationApplyResult.Applied; + } + + private DisseminationValue CreateSnapshotValue(MembershipTableSnapshot snapshot) => new( + DisseminationKey.Default, + fromVersion: 0, + toVersion: snapshot.Version.Value, + GetSnapshotPayload(snapshot)); + + private DisseminationValue CreateDiffValue( + MembershipTableSnapshot baseSnapshot, + MembershipTableSnapshot snapshot) => new( + DisseminationKey.Default, + fromVersion: baseSnapshot.Version.Value, + toVersion: snapshot.Version.Value, + GetDiffPayload(baseSnapshot, snapshot)); + + private async ValueTask ApplyDiff(MembershipTableSnapshotDiff diff, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var current = membershipManager.CurrentSnapshot; + if (current.Version.Value > diff.Version.Value) + { + return DisseminationApplyResult.Obsolete; + } + + if (current.Version.Value != diff.Version.Value + && current.Version.Value != diff.BaseVersion.Value) + { + return DisseminationApplyResult.Rejected; + } + + var entries = current.Entries.ToBuilder(); + foreach (var silo in diff.RemovedSilos) + { + entries.Remove(silo); + } + + foreach (var entry in diff.UpdatedEntries) + { + entries[entry.SiloAddress] = PreserveIAmAliveTime(current, entry); + } + + var snapshot = new MembershipTableSnapshot(diff.Version, entries.ToImmutable()); + if (current.Version.Value == diff.Version.Value && !snapshot.IsSuccessorTo(current)) + { + return DisseminationApplyResult.Duplicate; + } + + await membershipManager.ProcessGossipSnapshot(snapshot, cancellationToken); + RememberSnapshot(membershipManager.CurrentSnapshot); + return DisseminationApplyResult.Applied; + } + + private void RememberSnapshot(MembershipTableSnapshot snapshot) + { + lock (_historyLock) + { + RememberSnapshotUnsafe(snapshot); + } + } + + private void RememberSnapshotUnsafe(MembershipTableSnapshot snapshot) + { + if (_snapshotHistory.TryGetValue(snapshot.Version.Value, out var previous) + && !MembershipSnapshotsEqual(previous, snapshot)) + { + // Replacing a same-version snapshot invalidates bytes derived from the older liveness state. + InvalidatePayloads(snapshot.Version.Value); + } + + _snapshotHistory[snapshot.Version.Value] = snapshot; + while (_snapshotHistory.Count > MaxSnapshotHistory) + { + using var enumerator = _snapshotHistory.Keys.GetEnumerator(); + if (enumerator.MoveNext()) + { + var removedVersion = enumerator.Current; + _snapshotHistory.Remove(removedVersion); + InvalidatePayloads(removedVersion); + } + } + } + + private void InvalidatePayloads(long version) + { + _snapshotPayloads.Remove(version); + foreach (var key in _diffPayloads.Keys + .Where(key => key.FromVersion == version || key.ToVersion == version) + .ToArray()) + { + _diffPayloads.Remove(key); + } + } + + private ReadOnlyMemory GetSnapshotPayload(MembershipTableSnapshot snapshot) + { + lock (_historyLock) + { + if (!_snapshotPayloads.TryGetValue(snapshot.Version.Value, out var payload)) + { + payload = serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = snapshot }); + _snapshotPayloads.Add(snapshot.Version.Value, payload); + } + + return payload; + } + } + + private ReadOnlyMemory GetDiffPayload( + MembershipTableSnapshot baseSnapshot, + MembershipTableSnapshot snapshot) + { + var key = (baseSnapshot.Version.Value, snapshot.Version.Value); + lock (_historyLock) + { + if (!_diffPayloads.TryGetValue(key, out var payload)) + { + payload = serializer.SerializeToArray(new MembershipTableSnapshotUpdate + { + Diff = CreateDiff(baseSnapshot, snapshot), + }); + _diffPayloads.Add(key, payload); + } + + return payload; + } + } + + private static MembershipTableSnapshotDiff CreateDiff(MembershipTableSnapshot baseSnapshot, MembershipTableSnapshot snapshot) + { + // Include every current entry: peers at the same table version can still have different liveness baselines. + var updated = ImmutableArray.CreateBuilder(); + foreach (var entry in snapshot.Entries) + { + updated.Add(entry.Value); + } + + var removed = ImmutableArray.CreateBuilder(); + foreach (var entry in baseSnapshot.Entries) + { + if (!snapshot.Entries.ContainsKey(entry.Key)) + { + removed.Add(entry.Key); + } + } + + return new MembershipTableSnapshotDiff( + baseSnapshot.Version, + snapshot.Version, + updated.ToImmutable(), + removed.ToImmutable()); + } + + private static long GetFingerprint(MembershipTableSnapshot snapshot) + { + // Keep the hash deterministic across hosts and focused on state which can change without a version bump. + const ulong offset = 14695981039346656037; + const ulong prime = 1099511628211; + var hash = offset; + foreach (var entry in snapshot.Entries.OrderBy(static entry => entry.Key)) + { + hash = unchecked((hash ^ (uint)entry.Key.GetConsistentHashCode()) * prime); + hash = unchecked((hash ^ (ulong)entry.Value.EffectiveIAmAliveTime.Ticks) * prime); + } + + return unchecked((long)hash); + } + + private static MembershipEntry PreserveIAmAliveTime(MembershipTableSnapshot previousSnapshot, MembershipEntry entry) + { + // A repair must not move locally observed liveness backward. + if (previousSnapshot.Entries.TryGetValue(entry.SiloAddress, out var previousEntry) + && previousEntry.IAmAliveTime > entry.IAmAliveTime) + { + return CopyWithIAmAliveTime(entry, previousEntry.IAmAliveTime); + } + + return entry; + } + + private static MembershipEntry CopyWithIAmAliveTime(MembershipEntry entry, DateTime iAmAliveTime) => new() + { + SiloAddress = entry.SiloAddress, + Status = entry.Status, + SuspectTimes = entry.SuspectTimes is null ? null : new(entry.SuspectTimes), + ProxyPort = entry.ProxyPort, + HostName = entry.HostName, + SiloName = entry.SiloName, + RoleName = entry.RoleName, + UpdateZone = entry.UpdateZone, + FaultZone = entry.FaultZone, + StartTime = entry.StartTime, + IAmAliveTime = iAmAliveTime, + }; + + private static bool MembershipEntriesEqual(MembershipEntry left, MembershipEntry right) => + left.SiloAddress.Equals(right.SiloAddress) + && left.Status == right.Status + && EqualSuspectTimes(left.SuspectTimes, right.SuspectTimes) + && left.ProxyPort == right.ProxyPort + && string.Equals(left.HostName, right.HostName, StringComparison.Ordinal) + && string.Equals(left.SiloName, right.SiloName, StringComparison.Ordinal) + && string.Equals(left.RoleName, right.RoleName, StringComparison.Ordinal) + && left.UpdateZone == right.UpdateZone + && left.FaultZone == right.FaultZone + && left.StartTime == right.StartTime + && left.IAmAliveTime == right.IAmAliveTime; + + private static bool MembershipSnapshotsEqual( + MembershipTableSnapshot left, + MembershipTableSnapshot right) + { + if (ReferenceEquals(left, right)) + { + return true; + } + + if (left.Version != right.Version || left.Entries.Count != right.Entries.Count) + { + return false; + } + + foreach (var (siloAddress, entry) in left.Entries) + { + if (!right.Entries.TryGetValue(siloAddress, out var other) + || !MembershipEntriesEqual(entry, other)) + { + return false; + } + } + + return true; + } + + private static bool EqualSuspectTimes(List>? left, List>? right) + { + if (left is null || right is null) + { + return left is null && right is null; + } + + if (left.Count != right.Count) + { + return false; + } + + for (var i = 0; i < left.Count; i++) + { + if (!Equals(left[i].Item1, right[i].Item1) || left[i].Item2 != right[i].Item2) + { + return false; + } + } + + return true; + } +} + +[GenerateSerializer, Immutable] +internal sealed class MembershipTableSnapshotUpdate +{ + [Id(0)] + public MembershipTableSnapshot? Snapshot { get; init; } + + [Id(1)] + public MembershipTableSnapshotDiff? Diff { get; init; } +} + +[GenerateSerializer, Immutable] +internal sealed class MembershipTableSnapshotDiff +{ + public MembershipTableSnapshotDiff( + MembershipVersion baseVersion, + MembershipVersion version, + ImmutableArray updatedEntries, + ImmutableArray removedSilos) + { + BaseVersion = baseVersion; + Version = version; + UpdatedEntries = updatedEntries; + RemovedSilos = removedSilos; + } + + [Id(0)] + public MembershipVersion BaseVersion { get; } + + [Id(1)] + public MembershipVersion Version { get; } + + [Id(2)] + public ImmutableArray UpdatedEntries { get; } + + [Id(3)] + public ImmutableArray RemovedSilos { get; } +} diff --git a/src/Orleans.Runtime/Dissemination/WakeTimer.cs b/src/Orleans.Runtime/Dissemination/WakeTimer.cs new file mode 100644 index 00000000000..cb3bf0078e8 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/WakeTimer.cs @@ -0,0 +1,215 @@ +namespace Orleans.Runtime.Dissemination; + +/// +/// A thread-safe, wakeable, one-shot timer. +/// +/// +/// The timer has one reusable underlying . Call to arm or re-arm it +/// with a due time, call to complete the current or next wait immediately, and call +/// to wait until either the due time elapses or the timer is explicitly woken. +/// +internal sealed class WakeTimer : IDisposable +{ + private readonly object _lock = new(); + private readonly TimeProvider _timeProvider; + private readonly ITimer _timer; + private TaskCompletionSource? _waiter; + private long _armedAtTimestamp; + private TimeSpan _dueTime; + private bool _armed; + private bool _signaled; + private bool _disposed; + + /// + /// Initializes a new instance of the class. + /// + /// The time provider used to create the underlying timer. + public WakeTimer(TimeProvider timeProvider) + { + _timeProvider = timeProvider; + _timer = timeProvider.CreateTimer( + static state => ((WakeTimer)state!).OnTimer(), + this, + Timeout.InfiniteTimeSpan, + Timeout.InfiniteTimeSpan); + } + + /// + /// Arms or re-arms the timer to fire once after the provided due time. + /// + /// + /// The delay before the timer fires. schedules an immediate wake, and + /// prevents the timer from firing until changed or woken. + /// + /// + /// The timer always uses an infinite period, so every call is a one-shot schedule. Calling this method while + /// another caller is waiting causes that waiter to observe the new due time rather than completing immediately. + /// + public void Change(TimeSpan dueTime) + { + lock (_lock) + { + if (_disposed) + { + return; + } + + _armed = dueTime != Timeout.InfiniteTimeSpan; + _armedAtTimestamp = _armed ? _timeProvider.GetTimestamp() : 0; + _dueTime = dueTime; + _timer.Change(dueTime, Timeout.InfiniteTimeSpan); + } + } + + /// + /// Immediately completes the current wait, or causes the next wait to complete immediately. + /// + public void Wake() + { + TaskCompletionSource? waiter; + lock (_lock) + { + if (_disposed) + { + return; + } + + waiter = CompleteWaitUnsafe(); + if (waiter is null) + { + _signaled = true; + } + } + + waiter?.TrySetResult(true); + } + + /// + /// Waits until the timer fires, is explicitly woken, or is disposed. + /// + /// A token which cancels this wait without disarming the timer. + /// + /// when the due time elapsed or was called; + /// when the timer was disposed. + /// + /// + /// Only one waiter is supported at a time. If the timer has already fired or been woken before this method is + /// called, the method returns immediately. Cancelling a wait does not disarm the timer, + /// so a later waiter can still observe the scheduled wakeup. + /// + public async ValueTask WaitAsync(CancellationToken cancellationToken) + { + TaskCompletionSource waiter; + lock (_lock) + { + if (_disposed) + { + return false; + } + + if (_signaled) + { + _signaled = false; + return true; + } + + if (_waiter is not null) + { + throw new InvalidOperationException("Only one waiter can wait on a WakeTimer at a time."); + } + + waiter = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiter = waiter; + } + + using var registration = cancellationToken.UnsafeRegister( + static state => + { + var (timer, waiter, token) = + ((WakeTimer Timer, TaskCompletionSource Waiter, CancellationToken Token))state!; + timer.CancelWait(waiter, token); + }, + (this, waiter, cancellationToken)); + + return await waiter.Task; + } + + /// + /// Disposes the timer, completing any current waiter with . + /// + public void Dispose() + { + TaskCompletionSource? waiter; + lock (_lock) + { + if (_disposed) + { + return; + } + + _disposed = true; + waiter = _waiter; + _waiter = null; + _armed = false; + } + + try + { + waiter?.TrySetResult(false); + } + finally + { + _timer.Dispose(); + } + } + + private TaskCompletionSource? CompleteWaitUnsafe() + { + var waiter = _waiter; + _waiter = null; + _armed = false; + _timer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + return waiter; + } + + private void OnTimer() + { + TaskCompletionSource? waiter; + lock (_lock) + { + if (_disposed || !_armed) + { + return; + } + + var remaining = _dueTime - _timeProvider.GetElapsedTime(_armedAtTimestamp); + if (remaining > TimeSpan.Zero) + { + _timer.Change(remaining, Timeout.InfiniteTimeSpan); + return; + } + + waiter = CompleteWaitUnsafe(); + if (waiter is null) + { + _signaled = true; + } + } + + waiter?.TrySetResult(true); + } + + private void CancelWait(TaskCompletionSource waiter, CancellationToken cancellationToken) + { + lock (_lock) + { + if (!ReferenceEquals(_waiter, waiter)) + { + return; + } + + _waiter = null; + waiter.TrySetCanceled(cancellationToken); + } + } +} diff --git a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs index df1bc1a6e4b..b9387444b38 100644 --- a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs +++ b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs @@ -20,6 +20,7 @@ using Orleans.Runtime; using Orleans.Runtime.Configuration; using Orleans.Runtime.ConsistentRing; +using Orleans.Runtime.Dissemination; using Orleans.Runtime.GrainDirectory; using Orleans.Runtime.MembershipService; using Orleans.Runtime.Messaging; @@ -172,9 +173,16 @@ internal static void AddDefaultServices(ISiloBuilder builder) services.AddSingleton(); services.AddFromExisting, DeploymentLoadPublisher>(); + services.AddSingleton(); + services.AddFromExisting(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddFromExisting(); + services.AddFromExisting, DisseminationSystemTarget>(); + services.TryAddSingleton(); services.AddFromExisting(); services.AddFromExisting, IMembershipManager>(); @@ -182,6 +190,8 @@ internal static void AddDefaultServices(ISiloBuilder builder) services.AddFromExisting(); services.AddFromExisting, MembershipSystemTarget>(); services.AddSingleton(); + services.AddSingleton(); + services.AddFromExisting(); services.AddSingleton(); services.AddSingleton(); services.TryAddFromExisting(); @@ -350,6 +360,7 @@ internal static void AddDefaultServices(ISiloBuilder builder) services.ConfigureFormatter(); services.ConfigureFormatter(); services.ConfigureFormatter(); + services.ConfigureFormatter(); // This validator needs to construct the IMembershipOracle and the IMembershipTable // so move it in the end so other validator are called first @@ -358,6 +369,9 @@ internal static void AddDefaultServices(ISiloBuilder builder) services.AddTransient(); services.AddTransient(); services.AddTransient, SiloMessagingOptionsValidator>(); + services.AddTransient, DisseminationOptionsValidator>(); + services.AddTransient, DeploymentLoadPublisherOptionsValidator>(); + services.AddTransient, ClusterMembershipOptionsDisseminationValidator>(); services.AddTransient>(static sp => sp.GetRequiredService>()); // Enable hosted client. diff --git a/src/Orleans.Runtime/MembershipService/MembershipGossiper.cs b/src/Orleans.Runtime/MembershipService/MembershipGossiper.cs index 8d6c3749deb..2b39c0535a6 100644 --- a/src/Orleans.Runtime/MembershipService/MembershipGossiper.cs +++ b/src/Orleans.Runtime/MembershipService/MembershipGossiper.cs @@ -4,14 +4,18 @@ using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Orleans.Runtime.Dissemination; namespace Orleans.Runtime.MembershipService; -internal partial class MembershipGossiper(IServiceProvider serviceProvider, ILogger logger) : IMembershipGossiper +internal partial class MembershipGossiper( + IServiceProvider serviceProvider, + ILocalSiloDetails localSiloDetails, + ILogger logger) : IMembershipGossiper { private MembershipSystemTarget? _membershipSystemTarget; - public Task GossipToRemoteSilos( + public async Task GossipToRemoteSilos( List gossipPartners, MembershipTableSnapshot snapshot, SiloAddress updatedSilo, @@ -19,12 +23,44 @@ public Task GossipToRemoteSilos( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - if (gossipPartners.Count == 0) return Task.CompletedTask; + if (gossipPartners.Count == 0) return; LogDebugGossipingStatusToPartners(logger, updatedSilo, updatedStatus, gossipPartners.Count); + // Direct gossip starts first and owns shutdown-critical delivery. var systemTarget = _membershipSystemTarget ??= serviceProvider.GetRequiredService(); - return systemTarget.GossipToRemoteSilos(gossipPartners, snapshot, updatedSilo, updatedStatus, cancellationToken); + var directGossip = systemTarget.GossipToRemoteSilos(gossipPartners, snapshot, updatedSilo, updatedStatus, cancellationToken); + if (snapshot.Entries.TryGetValue(localSiloDetails.SiloAddress, out var localEntry) + && localEntry.Status is SiloStatus.Joining or SiloStatus.Active or SiloStatus.ShuttingDown or SiloStatus.Stopping) + { + await Task.WhenAll(directGossip, TryGossipViaDissemination(snapshot, cancellationToken)); + } + else + { + await directGossip; + } + } + + private async Task TryGossipViaDissemination(MembershipTableSnapshot snapshot, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + var disseminationNamespace = serviceProvider.GetRequiredService(); + if (disseminationNamespace.Options.Enabled) + { + var dissemination = serviceProvider.GetRequiredService(); + await disseminationNamespace.PublishAsync(dissemination, snapshot, cancellationToken); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + LogDebugMembershipDisseminationFailed(logger, exception); + } } [LoggerMessage( @@ -32,4 +68,9 @@ public Task GossipToRemoteSilos( Message = "Gossiping {Silo} status {Status} to {NumPartners} partners" )] private static partial void LogDebugGossipingStatusToPartners(ILogger logger, SiloAddress silo, SiloStatus status, int numPartners); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Membership dissemination failed. Direct membership gossip continues delivery.")] + private static partial void LogDebugMembershipDisseminationFailed(ILogger logger, Exception exception); } diff --git a/src/Orleans.Runtime/OrleansContracts.txt b/src/Orleans.Runtime/OrleansContracts.txt index 3dd8082bb7d..073b71e1154 100644 --- a/src/Orleans.Runtime/OrleansContracts.txt +++ b/src/Orleans.Runtime/OrleansContracts.txt @@ -23,6 +23,8 @@ class [GrainType("developmentleaseprovider")] Orleans.Runtime.Development.Develo interface [GrainInterfaceType("Orleans.Runtime.Development.IDevelopmentLeaseProviderGrain")] Orleans.Runtime.Development.IDevelopmentLeaseProviderGrain [Version(0)] 847FCE12: Reset(System.Threading.CancellationToken) -> Task +class [GrainType("disseminationsystemtarget")] Orleans.Runtime.Dissemination.DisseminationSystemTarget + class [GrainType("graincallcancellationmanager")] Orleans.Runtime.GrainCallCancellationManager class [GrainType("clientdirectory")] Orleans.Runtime.GrainDirectory.ClientDirectory diff --git a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs index a3223cc86b7..bc3977f0f3b 100644 --- a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs +++ b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs @@ -4,10 +4,12 @@ using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Orleans.Configuration; using Orleans.Runtime.Diagnostics; +using Orleans.Runtime.Dissemination; using Orleans.Internal; using Orleans.Runtime.Scheduler; using Orleans.Statistics; @@ -26,6 +28,7 @@ internal sealed partial class DeploymentLoadPublisher : SystemTarget, IDeploymen private readonly IActivationWorkingSet _activationWorkingSet; private readonly IEnvironmentStatisticsProvider _environmentStatisticsProvider; private readonly IOptions _loadSheddingOptions; + private readonly IServiceProvider _serviceProvider; private readonly ConcurrentDictionary _periodicStats; private readonly TimeSpan _statisticsRefreshTime; private readonly List _siloStatisticsChangeListeners; @@ -48,6 +51,7 @@ public DeploymentLoadPublisher( IActivationWorkingSet activationWorkingSet, IEnvironmentStatisticsProvider environmentStatisticsProvider, IOptions loadSheddingOptions, + IServiceProvider serviceProvider, SystemTargetShared shared) : base(Constants.DeploymentLoadPublisherSystemTargetType, shared) { @@ -59,6 +63,7 @@ public DeploymentLoadPublisher( _activationWorkingSet = activationWorkingSet; _environmentStatisticsProvider = environmentStatisticsProvider; _loadSheddingOptions = loadSheddingOptions; + _serviceProvider = serviceProvider; _statisticsRefreshTime = options.Value.DeploymentLoadPublisherRefreshTime; _periodicStats = new ConcurrentDictionary(); _siloStatisticsChangeListeners = new List(); @@ -85,7 +90,11 @@ await this.RunOrQueueTask(() => } await RefreshClusterStatistics(cancellationToken); - await PublishStatistics(cancellationToken); + await this.RunOrQueueTask(async token => + { + await PublishStatistics(token); + return true; + }, cancellationToken); LogDebugStartedDeploymentLoadPublisher(_logger); } @@ -112,24 +121,20 @@ internal async Task PublishStatistics(CancellationToken cancellationToken) DeploymentLoadPublisherEvents.EmitPublished(_siloDetails.SiloAddress, myStats); // Inform other cluster members about our refreshed statistics. - var members = _siloStatusOracle.GetApproximateSiloStatuses(true).Keys; - var tasks = new List(members.Count); - foreach (var siloAddress in members) + var members = _siloStatusOracle.GetApproximateSiloStatuses(true).Keys.ToArray(); + IReadOnlyCollection directRecipients = members; + if (await TryPublishStatisticsViaDissemination(myStats, cancellationToken)) { - // No need to make a grain call to ourselves. - if (siloAddress.Equals(_siloDetails.SiloAddress)) - { - continue; - } - try { - var deploymentLoadPublisher = _grainFactory.GetSystemTarget(Constants.DeploymentLoadPublisherSystemTargetType, siloAddress); - tasks.Add(deploymentLoadPublisher.UpdateRuntimeStatistics(_siloDetails.SiloAddress, myStats, cancellationToken)); + var dissemination = _serviceProvider.GetRequiredService(); + var disseminationNamespace = _serviceProvider.GetRequiredService(); + var unconfirmedPeers = dissemination.GetUnconfirmedPeers(disseminationNamespace).ToHashSet(); + directRecipients = members.Where(unconfirmedPeers.Contains).ToArray(); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - tasks.Add(Task.FromCanceled(cancellationToken)); + throw; } catch (Exception exception) { @@ -137,7 +142,7 @@ internal async Task PublishStatistics(CancellationToken cancellationToken) } } - await Task.WhenAll(tasks); + await PublishStatisticsDirectly(myStats, directRecipients, cancellationToken); DeploymentLoadPublisherEvents.EmitClusterRefreshed(_siloDetails.SiloAddress, _periodicStats); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -160,23 +165,131 @@ public Task UpdateRuntimeStatistics( return Task.CompletedTask; } - private void UpdateRuntimeStatisticsInternal(SiloAddress siloAddress, SiloRuntimeStatistics siloStats) + internal Task ApplyDisseminatedRuntimeStatisticsAsync( + SiloAddress siloAddress, + SiloRuntimeStatistics siloStats, + CancellationToken cancellationToken) => + this.RunOrQueueTask( + token => + { + token.ThrowIfCancellationRequested(); + return Task.FromResult(UpdateRuntimeStatisticsInternal(siloAddress, siloStats, isDisseminated: true)); + }, + cancellationToken); + + internal bool IsRuntimeStatisticsObsolete(SiloAddress siloAddress, long timestampTicks) => + _siloStatusOracle.GetApproximateSiloStatus(siloAddress) != SiloStatus.Active + || (_periodicStats.TryGetValue(siloAddress, out var old) && old.DateTime.Ticks > timestampTicks); + + internal IReadOnlyCollection GetActiveSilosForStatisticsDigest() => + _siloStatusOracle.GetApproximateSiloStatuses(onlyActive: true).Keys; + + internal async Task TryPublishStatisticsViaDissemination( + SiloRuntimeStatistics myStats, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_statisticsRefreshTime <= TimeSpan.Zero) + { + return false; + } + + var disseminationNamespace = _serviceProvider.GetRequiredService(); + if (!disseminationNamespace.Options.Enabled) + { + return false; + } + + var timeProvider = _serviceProvider.GetRequiredService(); + using var timeoutCancellation = new CancellationTokenSource(_statisticsRefreshTime, timeProvider); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellation.Token); + try + { + var dissemination = _serviceProvider.GetRequiredService(); + return await dissemination.Publish( + disseminationNamespace, + _siloDetails.SiloAddress, + myStats.DateTime.Ticks, + cancellation.Token); + } + catch (OperationCanceledException) when ( + timeoutCancellation.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + LogDebugRuntimeStatisticsDisseminationTimedOut(_logger, _statisticsRefreshTime); + return false; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + LogWarningRuntimeStatisticsUpdateFailure1(_logger, exception); + return false; + } + } + + private async Task PublishStatisticsDirectly( + SiloRuntimeStatistics myStats, + IReadOnlyCollection members, + CancellationToken cancellationToken) + { + var tasks = new List(members.Count); + foreach (var siloAddress in members) + { + if (siloAddress.Equals(_siloDetails.SiloAddress)) + { + continue; + } + + try + { + var deploymentLoadPublisher = _grainFactory.GetSystemTarget( + Constants.DeploymentLoadPublisherSystemTargetType, siloAddress); + tasks.Add(deploymentLoadPublisher.UpdateRuntimeStatistics(_siloDetails.SiloAddress, myStats, cancellationToken)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + tasks.Add(Task.FromCanceled(cancellationToken)); + } + catch (Exception exception) + { + LogWarningRuntimeStatisticsUpdateFailure1(_logger, exception); + } + } + + await Task.WhenAll(tasks); + } + + private DisseminationApplyResult UpdateRuntimeStatisticsInternal( + SiloAddress siloAddress, + SiloRuntimeStatistics siloStats, + bool isDisseminated = false) { LogTraceUpdateRuntimeStatistics(_logger, siloAddress); if (_siloStatusOracle.GetApproximateSiloStatus(siloAddress) != SiloStatus.Active) { - return; + return DisseminationApplyResult.Rejected; } // Take only if newer. - if (_periodicStats.TryGetValue(siloAddress, out var old) && old.DateTime > siloStats.DateTime) + if (_periodicStats.TryGetValue(siloAddress, out var old)) { - return; + if (old.DateTime > siloStats.DateTime) + { + return DisseminationApplyResult.Obsolete; + } + + if (isDisseminated && old.DateTime == siloStats.DateTime) + { + return DisseminationApplyResult.Duplicate; + } } _periodicStats[siloAddress] = siloStats; NotifyAllStatisticsChangeEventsSubscribers(siloAddress, siloStats); DeploymentLoadPublisherEvents.EmitReceived(siloAddress, _siloDetails.SiloAddress, siloStats); + return DisseminationApplyResult.Applied; } internal async Task RefreshClusterStatistics(CancellationToken cancellationToken) @@ -309,6 +422,11 @@ Task DisposePublishTimer(CancellationToken ct) )] private static partial void LogDebugStartedDeploymentLoadPublisher(ILogger logger); + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Deployment load dissemination did not accept the update within {Timeout}. Publishing directly.")] + private static partial void LogDebugRuntimeStatisticsDisseminationTimedOut(ILogger logger, TimeSpan timeout); + [LoggerMessage( Level = LogLevel.Trace, Message = "PublishStatistics" diff --git a/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs b/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs index 00dd3d6ce4d..fb5b6f098de 100644 --- a/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs +++ b/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs @@ -2,13 +2,16 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; using NSubstitute; using Orleans; using Orleans.Configuration; using Orleans.Core.Diagnostics; using Orleans.Internal; using Orleans.Runtime; +using Orleans.Runtime.Dissemination; using Orleans.Runtime.Scheduler; +using Orleans.Serialization; using Orleans.Statistics; using Orleans.Timers; using TestExtensions; @@ -22,6 +25,199 @@ namespace NonSilo.Tests.Runtime; [TestArea("Runtime")] public class DeploymentLoadPublisherTests { + [Fact] + public async Task PublishStatistics_DefaultDisabled_PreservesDirectPublication() + { + using var rig = CreateTestRig(TimeSpan.FromSeconds(5)); + + await rig.Publisher.PublishStatistics(TestContext.Current.CancellationToken); + + await rig.DirectTarget.Received(1).UpdateRuntimeStatistics( + rig.LocalSilo, rig.Publisher.LocalRuntimeStatistics, TestContext.Current.CancellationToken); + Assert.Empty(rig.Dissemination.ReceivedCalls()); + Assert.False(new DisseminationOptions().Enabled); + Assert.False(new DeploymentLoadPublisherOptions().Dissemination.Enabled); + Assert.False(new ClusterMembershipOptions().Dissemination.Enabled); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task PublishStatistics_ConfirmedPeersUseDissemination(bool confirmed) + { + using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: true); + var remoteSilo = SiloAddress.FromParsableString("127.0.0.1:200@100"); + rig.Dissemination.GetUnconfirmedPeers(Arg.Any()) + .Returns(confirmed ? [] : new[] { remoteSilo }); + + await rig.Publisher.PublishStatistics(TestContext.Current.CancellationToken); + + await rig.Dissemination.Received(1).Publish( + Arg.Any(), rig.LocalSilo, + rig.Publisher.LocalRuntimeStatistics.DateTime.Ticks, Arg.Any()); + await rig.DirectTarget.Received(confirmed ? 0 : 1).UpdateRuntimeStatistics( + rig.LocalSilo, rig.Publisher.LocalRuntimeStatistics, TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PublishStatistics_DisseminationCancelsIndependently_FallsBackToDirectPublication() + { + using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: true); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + rig.Dissemination.Publish( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => ValueTask.FromCanceled(cancellation.Token)); + + await rig.Publisher.PublishStatistics(TestContext.Current.CancellationToken); + + await rig.DirectTarget.Received(1).UpdateRuntimeStatistics( + rig.LocalSilo, rig.Publisher.LocalRuntimeStatistics, TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PublishStatistics_CallerCancels_DoesNotFallBackToDirectPublication() + { + using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: true); + using var cancellation = new CancellationTokenSource(); + rig.Dissemination.Publish( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => + { + cancellation.Cancel(); + return ValueTask.FromCanceled(call.ArgAt(3)); + }); + + await Assert.ThrowsAnyAsync(() => rig.Publisher.PublishStatistics(cancellation.Token)); + + Assert.Empty(rig.DirectTarget.ReceivedCalls()); + } + + [Fact] + public async Task PublishStatistics_DeadlineCancelsNativePublicationBeforeDirectDelivery() + { + using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: true); + var timeProvider = (FakeTimeProvider)rig.ServiceProvider.GetRequiredService(); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var completed = false; + rig.Dissemination.Publish( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => new ValueTask(PublishAsync(call.ArgAt(3)))); + var publication = rig.Publisher.PublishStatistics(TestContext.Current.CancellationToken); + var token = await started.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + timeProvider.Advance(TimeSpan.FromSeconds(5)); + await publication.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.True(token.IsCancellationRequested); + Assert.True(completed); + await rig.DirectTarget.Received(1).UpdateRuntimeStatistics( + rig.LocalSilo, rig.Publisher.LocalRuntimeStatistics, TestContext.Current.CancellationToken); + + async Task PublishAsync(CancellationToken cancellationToken) + { + started.SetResult(cancellationToken); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return true; + } + finally + { + completed = true; + } + } + } + + [Fact] + public async Task ApplyLoadStatistics_UsesPublisherSchedulerAndSuppressesDisseminationDuplicates() + { + using var rig = CreateTestRig(TimeSpan.Zero); + var listener = Substitute.For(); + listener.When(value => value.SiloStatisticsChangeNotification(rig.LocalSilo, Arg.Any())) + .Do(_ => + { + Assert.Same(rig.Publisher, RuntimeContext.Current); + rig.Publisher.UnsubscribeStatisticsChangeEvents(listener); + }); + var remaining = Substitute.For(); + rig.Publisher.SubscribeToStatisticsChangeEvents(listener); + rig.Publisher.SubscribeToStatisticsChangeEvents(remaining); + var ns = rig.ServiceProvider.GetRequiredService(); + var value = ns.CreateValue(rig.LocalSilo, rig.InitialStatistics); + + var applied = await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken); + var duplicate = await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken); + + Assert.Equal(DisseminationApplyResult.Applied, applied); + Assert.Equal(DisseminationApplyResult.Duplicate, duplicate); + listener.Received(1).SiloStatisticsChangeNotification(rig.LocalSilo, Arg.Any()); + remaining.Received(1).SiloStatisticsChangeNotification(rig.LocalSilo, Arg.Any()); + Assert.Equal(rig.InitialStatistics.DateTime, rig.Publisher.PeriodicStatistics[rig.LocalSilo].DateTime); + } + + [Fact] + public async Task UpdateRuntimeStatistics_DefaultDisabled_PreservesDuplicateNotifications() + { + using var rig = CreateTestRig(TimeSpan.Zero); + var listener = Substitute.For(); + rig.Publisher.SubscribeToStatisticsChangeEvents(listener); + + await rig.Publisher.UpdateRuntimeStatistics(rig.LocalSilo, rig.InitialStatistics, TestContext.Current.CancellationToken); + await rig.Publisher.UpdateRuntimeStatistics(rig.LocalSilo, rig.InitialStatistics, TestContext.Current.CancellationToken); + + listener.Received(2).SiloStatisticsChangeNotification(rig.LocalSilo, rig.InitialStatistics); + Assert.Same(rig.InitialStatistics, rig.Publisher.PeriodicStatistics[rig.LocalSilo]); + } + + [Theory] + [InlineData(SiloStatus.Dead)] + [InlineData(SiloStatus.None)] + public async Task ApplyLoadStatistics_DepartedGenerationRemainsRemoved(SiloStatus status) + { + using var rig = CreateTestRig(TimeSpan.Zero); + await rig.Publisher.UpdateRuntimeStatistics(rig.LocalSilo, rig.InitialStatistics, TestContext.Current.CancellationToken); + var oracle = rig.ServiceProvider.GetRequiredService(); + oracle.GetApproximateSiloStatus(rig.LocalSilo).Returns(status); + rig.Publisher.SiloStatusChangeNotification(rig.LocalSilo, SiloStatus.Dead); + var ns = rig.ServiceProvider.GetRequiredService(); + + var result = await ns.ApplyValueAsync( + ns.CreateValue(rig.LocalSilo, rig.InitialStatistics), TestContext.Current.CancellationToken); + + Assert.Equal(DisseminationApplyResult.Rejected, result); + Assert.False(rig.Publisher.PeriodicStatistics.ContainsKey(rig.LocalSilo)); + Assert.Equal(0, ns.GetVersion(rig.LocalSilo)); + } + + [Fact] + public async Task ApplyLoadStatistics_PreCanceledToken_PreservesLocalState() + { + using var rig = CreateTestRig(TimeSpan.Zero); + using var cancellation = new CancellationTokenSource(); + var ns = rig.ServiceProvider.GetRequiredService(); + var value = ns.CreateValue(rig.LocalSilo, rig.InitialStatistics); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + async () => await ns.ApplyValueAsync(value, cancellation.Token)); + + Assert.Empty(rig.Publisher.PeriodicStatistics); + } + + [Fact] + public async Task ApplyLoadStatistics_RequiresFullValue() + { + using var rig = CreateTestRig(TimeSpan.Zero); + var ns = rig.ServiceProvider.GetRequiredService(); + var fullValue = ns.CreateValue(rig.LocalSilo, rig.InitialStatistics); + var delta = new DisseminationValue(fullValue.Key, 1, fullValue.ToVersion, fullValue.Payload); + + Assert.Equal(DisseminationApplyResult.Rejected, + await ns.ApplyValueAsync(delta, TestContext.Current.CancellationToken)); + Assert.Empty(rig.Publisher.PeriodicStatistics); + } + [Theory] [InlineData(-5000)] [InlineData(-1)] @@ -234,7 +430,10 @@ public async Task StatisticsSubscribers_DeliverToRemainingSubscribersBeforeSurfa Assert.Same(rig.InitialStatistics, rig.Publisher.PeriodicStatistics[rig.LocalSilo]); } - private static TestRig CreateTestRig(TimeSpan refreshTime, ITimerRegistry? timerRegistry = null) + private static TestRig CreateTestRig( + TimeSpan refreshTime, + ITimerRegistry? timerRegistry = null, + bool enableDissemination = false) { var localSilo = SiloAddress.FromParsableString("127.0.0.1:100@100"); var remoteSilo = SiloAddress.FromParsableString("127.0.0.1:200@100"); @@ -258,9 +457,15 @@ private static TestRig CreateTestRig(TimeSpan refreshTime, ITimerRegistry? timer var control = Substitute.For(); control.GetRuntimeStatistics(Arg.Any()).Returns(Task.FromResult(initialStatistics)); grainFactory.GetSystemTarget(Constants.SiloControlType, Arg.Any()).Returns(control); + var dissemination = Substitute.For(); + dissemination.Publish( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(ValueTask.FromResult(true)); var services = new ServiceCollection(); + services.AddSerializer(); services.AddMetrics(); + services.AddSingleton(new FakeTimeProvider()); services.AddSingleton(NullLoggerFactory.Instance); services.AddSingleton(); services.AddSingleton(); @@ -271,11 +476,15 @@ private static TestRig CreateTestRig(TimeSpan refreshTime, ITimerRegistry? timer services.AddSingleton(localDetails); services.AddSingleton(statusOracle); services.AddSingleton(grainFactory); + services.AddSingleton(dissemination); services.AddSingleton(Substitute.For()); services.AddSingleton(environmentStatistics); services.AddSingleton>(loadSheddingOptions); - services.AddOptions().Configure( - options => options.DeploymentLoadPublisherRefreshTime = refreshTime); + services.AddOptions().Configure(options => + { + options.DeploymentLoadPublisherRefreshTime = refreshTime; + options.Dissemination.Enabled = enableDissemination; + }); services.AddSingleton(); services.AddSingleton(serviceProvider => new SystemTargetShared( runtimeClient: null!, @@ -290,8 +499,9 @@ private static TestRig CreateTestRig(TimeSpan refreshTime, ITimerRegistry? timer serviceProvider.GetRequiredService(), serviceProvider.GetRequiredService())); services.AddSingleton(); + services.AddSingleton(); var serviceProvider = services.BuildServiceProvider(); - return new(serviceProvider, serviceProvider.GetRequiredService(), directTarget, control, initialStatistics, localSilo); + return new(serviceProvider, serviceProvider.GetRequiredService(), directTarget, control, initialStatistics, localSilo, dissemination); } private sealed record TestRig( @@ -300,7 +510,8 @@ private sealed record TestRig( IDeploymentLoadPublisher DirectTarget, ISiloControl Control, SiloRuntimeStatistics InitialStatistics, - SiloAddress LocalSilo) : IDisposable + SiloAddress LocalSilo, + IDisseminationService Dissemination) : IDisposable { public void Dispose() => ServiceProvider.Dispose(); } From 7e185b94b465fbefa7a9f2ac9cc296190f2d1337 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 15:10:02 -0700 Subject: [PATCH 02/50] test(dissemination): preserve current real-process and cost evidence --- .github/scripts/dissemination-evidence.ps1 | 179 ++++++++ .github/workflows/dissemination-evidence.yml | 153 +++++++ .../Shared/Protocol.cs | 271 +++++++++++ .../Silo/ControlTarget.cs | 110 +++++ .../Silo/Dissemination.Silo.csproj | 18 + .../Silo/FileMembershipTable.cs | 238 ++++++++++ .../Silo/NewRuntime.cs | 227 ++++++++++ .../Silo/Observation.cs | 378 ++++++++++++++++ .../Silo/Program.cs | 409 +++++++++++++++++ ...semination.IntegrationHarness.Tests.csproj | 12 + .../Tests/MeasurementTests.cs | 302 +++++++++++++ .../Tests/ProcessCluster.cs | 424 ++++++++++++++++++ .../Tests/ScalingTests.cs | 312 +++++++++++++ .../Tests/VersionSkewTests.cs | 341 ++++++++++++++ .../methodology.json | 164 +++++++ 15 files changed, 3538 insertions(+) create mode 100644 .github/scripts/dissemination-evidence.ps1 create mode 100644 .github/workflows/dissemination-evidence.yml create mode 100644 test/Dissemination.IntegrationHarness/Shared/Protocol.cs create mode 100644 test/Dissemination.IntegrationHarness/Silo/ControlTarget.cs create mode 100644 test/Dissemination.IntegrationHarness/Silo/Dissemination.Silo.csproj create mode 100644 test/Dissemination.IntegrationHarness/Silo/FileMembershipTable.cs create mode 100644 test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs create mode 100644 test/Dissemination.IntegrationHarness/Silo/Observation.cs create mode 100644 test/Dissemination.IntegrationHarness/Silo/Program.cs create mode 100644 test/Dissemination.IntegrationHarness/Tests/Dissemination.IntegrationHarness.Tests.csproj create mode 100644 test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs create mode 100644 test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs create mode 100644 test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs create mode 100644 test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs create mode 100644 test/Dissemination.IntegrationHarness/methodology.json diff --git a/.github/scripts/dissemination-evidence.ps1 b/.github/scripts/dissemination-evidence.ps1 new file mode 100644 index 00000000000..b311c2e8cc9 --- /dev/null +++ b/.github/scripts/dissemination-evidence.ps1 @@ -0,0 +1,179 @@ +[CmdletBinding()] +param( + [ValidateSet('Gate', 'Cost', 'Scale')] + [string] $Mode = 'Gate', + [string] $BaselineDirectory = '.dissemination-baseline', + [string] $ArtifactsDirectory = 'Artifacts/DisseminationEvidence', + [ValidatePattern('^\d+(,\d+){0,5}$')] + [string] $Sizes = '4,8,16', + [ValidateRange(3, 200)] + [int] $Iterations = 10, + [ValidateRange(1, 5)] + [int] $Repetitions = 1, + [switch] $SkipBuild +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +if ($Mode -eq 'Cost') { + # This automatic PR profile cannot be expanded by dispatch inputs or caller overrides. + # Three rounds ensure that both the churn and partition scenarios actually inject a failure. + $Sizes = '4,8' + $Iterations = 3 + $Repetitions = 1 +} + +$baselineSha = '6739589254b746a8790cf53524e6abe372bb53d4' +$root = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..')) +$artifacts = [System.IO.Path]::GetFullPath((Join-Path $root $ArtifactsDirectory)) +$binaries = Join-Path $artifacts 'bin' +$results = Join-Path $artifacts "results/$Mode" +New-Item -ItemType Directory -Force $binaries, $results | Out-Null +Copy-Item -LiteralPath (Join-Path $root 'test/Dissemination.IntegrationHarness/methodology.json') -Destination $results +foreach ($size in $Sizes.Split(',')) { + if ([int]$size -lt 3 -or [int]$size -gt 32) { + throw 'Silo counts must be between 3 and 32.' + } +} + +function Invoke-DotNet([string[]] $Arguments) { + & dotnet @Arguments + if ($LASTEXITCODE -ne 0) { + throw "dotnet failed ($LASTEXITCODE): $($Arguments -join ' ')" + } +} + +function Get-Revision([string] $Directory) { + $revision = & git -C $Directory rev-parse HEAD + if ($LASTEXITCODE -ne 0) { + throw "Not a source checkout: $Directory" + } + return $revision.Trim() +} + +function Publish-Silo([string] $Checkout, [string] $Runtime, [string] $Revision) { + $project = Join-Path $Checkout 'test/Dissemination.IntegrationHarness/Silo/Dissemination.Silo.csproj' + $output = Join-Path $binaries $Runtime + Invoke-DotNet @( + 'publish', $project, '--configuration', 'Release', '--framework', 'net10.0', + '--output', $output, '--verbosity', 'minimal', + "-p:HarnessRuntime=$Runtime", "-p:SourceRevisionId=$Revision", + '-p:OfficialBuild=true', "-p:BUILD_SOURCEVERSION=$Revision", + "-bl:$(Join-Path $artifacts "build-$Runtime.binlog")" + ) + $assemblyPath = Join-Path $output 'Orleans.Runtime.dll' + $identity = [System.Reflection.AssemblyName]::GetAssemblyName($assemblyPath) + $assemblies = @{} + foreach ($file in Get-ChildItem -LiteralPath $output -Filter 'Orleans*.dll' -File) { + $name = [System.Reflection.AssemblyName]::GetAssemblyName($file.FullName) + $assemblies[$name.Name] = @{ + AssemblyVersion = $name.Version.ToString() + Sha256 = (Get-FileHash -LiteralPath $file.FullName -Algorithm SHA256).Hash + } + } + @{ + Runtime = $Runtime + SourceRevision = $Revision + AssemblyName = $identity.Name + AssemblyVersion = $identity.Version.ToString() + Sha256 = (Get-FileHash -LiteralPath $assemblyPath -Algorithm SHA256).Hash + Assemblies = $assemblies + } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $output 'manifest.json') +} + +if (!$SkipBuild) { + if ($env:GITHUB_ACTIONS -ne 'true') { + throw 'Building the pinned baseline is CI-only. Download the workflow binary artifact and use -SkipBuild for explicit local reproduction.' + } + + $baseline = [System.IO.Path]::GetFullPath((Join-Path $root $BaselineDirectory)) + if ($baseline -eq $root) { + throw 'The baseline must not be the feature checkout.' + } + $baselineTopLevel = (& git -C $baseline rev-parse --show-toplevel) + if ($LASTEXITCODE -ne 0 -or [System.IO.Path]::GetFullPath($baselineTopLevel.Trim()) -ne $baseline) { + throw 'The baseline must have its own isolated git checkout, not merely a directory in the feature checkout.' + } + if ((Get-Revision $baseline) -ne $baselineSha) { + throw "Baseline must be pinned to $baselineSha." + } + $sourceChanges = & git -C $baseline status --porcelain --untracked-files=all -- src + if ($LASTEXITCODE -ne 0 -or $sourceChanges) { + throw 'Pinned baseline runtime sources have local modifications.' + } + if (Test-Path -LiteralPath (Join-Path $baseline 'src/Orleans.Runtime/Dissemination')) { + throw 'The pinned baseline unexpectedly contains the new dissemination implementation.' + } + + # Only the test executable is copied. Every Orleans source project resolves inside its own checkout. + $harnessSource = Join-Path $root 'test/Dissemination.IntegrationHarness' + $harnessDestination = Join-Path $baseline 'test/Dissemination.IntegrationHarness' + if (Test-Path -LiteralPath $harnessDestination) { + throw "Refusing to overwrite an existing baseline harness: $harnessDestination" + } + New-Item -ItemType Directory -Force $harnessDestination | Out-Null + foreach ($part in @('Silo', 'Shared')) { + New-Item -ItemType Directory -Force (Join-Path $harnessDestination $part) | Out-Null + Get-ChildItem -LiteralPath (Join-Path $harnessSource $part) -File | + Where-Object { $_.Extension -in '.cs', '.csproj' } | + Copy-Item -Destination (Join-Path $harnessDestination $part) + } + + Publish-Silo $baseline 'Old' $baselineSha + Publish-Silo $root 'New' (Get-Revision $root) + Invoke-DotNet @( + 'publish', (Join-Path $root 'test/Dissemination.IntegrationHarness/Tests/Dissemination.IntegrationHarness.Tests.csproj'), + '--configuration', 'Release', '--framework', 'net10.0', + '--output', (Join-Path $binaries 'Runner'), '--verbosity', 'minimal', + "-bl:$(Join-Path $artifacts 'build-runner.binlog')" + ) +} + +$env:ORLEANS_DISSEMINATION_OLD = Join-Path $binaries 'Old' +$env:ORLEANS_DISSEMINATION_NEW = Join-Path $binaries 'New' +$env:ORLEANS_DISSEMINATION_RESULTS = $results +$env:ORLEANS_DISSEMINATION_SIZES = $Sizes +$env:ORLEANS_DISSEMINATION_ITERATIONS = $Iterations.ToString([System.Globalization.CultureInfo]::InvariantCulture) +$env:ORLEANS_DISSEMINATION_REPETITIONS = $Repetitions.ToString([System.Globalization.CultureInfo]::InvariantCulture) +$env:ORLEANS_DISSEMINATION_PROFILE = $Mode +$runner = Join-Path $binaries 'Runner/Dissemination.IntegrationHarness.Tests.dll' +if (!(Test-Path -LiteralPath $runner)) { + throw "Published runner not found: $runner" +} +$testClass = if ($Mode -eq 'Gate') { + 'Orleans.Dissemination.IntegrationHarness.VersionSkewTests' +} else { + 'Orleans.Dissemination.IntegrationHarness.ScalingTests' +} +$minimum = if ($Mode -eq 'Gate') { '7' } else { '1' } + +@{ + Mode = $Mode + PinnedBaseline = $baselineSha + CurrentCommit = Get-Revision $root + Sizes = $Sizes + Iterations = $Iterations + Repetitions = $Repetitions + StartUtc = [DateTimeOffset]::UtcNow + Command = "dotnet $runner --filter-class $testClass --minimum-expected-tests $minimum --report-trx" + Limitation = 'Loopback OS processes; controlled test membership provider; not cross-machine production throughput.' +} | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $results 'invocation.json') + +Push-Location $root +try { + if ($Mode -eq 'Gate') { + Invoke-DotNet @( + $runner, '--filter-class', 'Orleans.Dissemination.IntegrationHarness.MeasurementTests', + '--minimum-expected-tests', '9', '--report-trx', + '--results-directory', (Join-Path $results 'instrument-checks') + ) + } + + Invoke-DotNet @( + $runner, '--filter-class', $testClass, '--minimum-expected-tests', $minimum, + '--report-trx', '--results-directory', $results + ) +} +finally { + Pop-Location +} diff --git a/.github/workflows/dissemination-evidence.yml b/.github/workflows/dissemination-evidence.yml new file mode 100644 index 00000000000..37058d4d27c --- /dev/null +++ b/.github/workflows/dissemination-evidence.yml @@ -0,0 +1,153 @@ +name: Dissemination evidence + +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened] + paths: + - '.github/workflows/dissemination-evidence.yml' + - '.github/scripts/dissemination-evidence.ps1' + - 'test/Dissemination.IntegrationHarness/**' + - 'src/Orleans.Runtime/Dissemination/**' + - 'src/Orleans.Runtime/MembershipService/**' + - 'src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs' + - 'src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs' + - 'src/Orleans.Runtime/Hosting/DefaultSiloServices.cs' + - 'src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs' + - 'src/Orleans.Core/SystemTargetInterfaces/IDeploymentLoadPublisher.cs' + - 'src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs' + - 'src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs' + - 'src/Orleans.Core/Configuration/Options/DisseminationOptions.cs' + merge_group: + types: [checks_requested] + workflow_dispatch: + inputs: + scale: + description: 'Also run the larger controlled off/on scaling comparison' + type: boolean + default: false + sizes: + description: 'Comma-separated process counts (3..32, maximum six sizes)' + type: string + default: '4,8,16' + iterations: + description: 'Publication rounds per scenario (3..200)' + type: number + default: 10 + repetitions: + description: 'Paired off/on repetitions (1..5)' + type: number + default: 1 + +permissions: + contents: read + +concurrency: + group: dissemination-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + DOTNET_NOLOGO: true + +jobs: + version-skew: + name: Pinned-binary upgrade, rollback and recovery + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - name: Isolated pre-dissemination checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + repository: dotnet/orleans + ref: 6739589254b746a8790cf53524e6abe372bb53d4 + path: .dissemination-baseline + persist-credentials: false + - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 + with: + global-json-file: global.json + - name: Build isolated binaries and run bounded merge-gate checks + shell: pwsh + run: ./.github/scripts/dissemination-evidence.ps1 -Mode Gate + - name: Preserve gate results and complete process evidence + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-gate-evidence + retention-days: 14 + path: | + Artifacts/DisseminationEvidence/results + Artifacts/DisseminationEvidence/*.binlog + - name: Preserve reproducible binary pair + if: success() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-binaries + retention-days: 7 + path: Artifacts/DisseminationEvidence/bin + + bounded-cost: + name: Bounded original, default-off and enabled costs + needs: version-skew + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 + with: + global-json-file: global.json + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dissemination-binaries + path: Artifacts/DisseminationEvidence/bin + - name: Measure bounded original, default-off and supported-enabled paths + shell: pwsh + run: ./.github/scripts/dissemination-evidence.ps1 -Mode Cost -SkipBuild + - name: Preserve the automatic PR cost evidence + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-bounded-cost-evidence + retention-days: 14 + path: Artifacts/DisseminationEvidence/results/Cost + + scaling: + name: Larger controlled dissemination cost comparison + if: github.event_name == 'workflow_dispatch' && inputs.scale + needs: [version-skew, bounded-cost] + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 + with: + global-json-file: global.json + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dissemination-binaries + path: Artifacts/DisseminationEvidence/bin + - name: Measure stable, churn and partition windows on all three paths + shell: pwsh + env: + EVIDENCE_SIZES: ${{ inputs.sizes || '4,8,16' }} + EVIDENCE_ITERATIONS: ${{ inputs.iterations || 10 }} + EVIDENCE_REPETITIONS: ${{ inputs.repetitions || 1 }} + run: | + ./.github/scripts/dissemination-evidence.ps1 -Mode Scale -SkipBuild ` + -Sizes $env:EVIDENCE_SIZES ` + -Iterations ([int]$env:EVIDENCE_ITERATIONS) ` + -Repetitions ([int]$env:EVIDENCE_REPETITIONS) + - name: Preserve raw cost windows, per-node load and failure evidence + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-scale-evidence + retention-days: 30 + path: Artifacts/DisseminationEvidence/results diff --git a/test/Dissemination.IntegrationHarness/Shared/Protocol.cs b/test/Dissemination.IntegrationHarness/Shared/Protocol.cs new file mode 100644 index 00000000000..36eca6675ed --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Shared/Protocol.cs @@ -0,0 +1,271 @@ +using System.Text.Json; + +namespace Orleans.Dissemination.IntegrationHarness; + +internal sealed record NodeConfiguration( + string Name, + string ClusterId, + string MembershipDirectory, + int SiloPort, + int ParentProcessId, + bool Enabled, + bool FastRecovery = true); + +internal sealed record Command( + int Id, + string Operation, + string? Peer = null, + int Count = 1, + bool Value = false, + long Version = 0); + +internal sealed record Response(int Id, NodeSnapshot? Snapshot, string? Error); + +internal sealed record BinaryIdentity( + int ProcessId, + string Runtime, + string SourceRevision, + string AssemblyName, + string AssemblyVersion, + string InformationalVersion, + string ModuleVersionId, + string Sha256, + string AssemblyPath, + bool HasDissemination) +{ + public Dictionary Assemblies { get; init; } = []; +} + +internal sealed record AssemblyProof( + string Version, + string InformationalVersion, + string ModuleVersionId, + string Sha256, + string Path); + +internal sealed record MetricValue(long Count, double Sum); + +internal sealed record TreeGateSnapshot(bool Blocked, int InFlight, long Admitted, long Rejected); + +internal sealed record StateComparisonProbe(string Before, string After, string[] PublicFields); + +internal sealed record ControlledCallSnapshot( + string? Peer, + DateTimeOffset? StartedAtUtc, + DateTimeOffset? CancellationRequestedAtUtc, + bool CancellationRequested, + bool RawTaskCompleted, + string? RawTaskStatus, + string? RawTaskError); + +// The token source outlives the start command. Cancellation is a separate, explicit phase and +// completion means the actual RPC task completed, not a wait canceled by that same token. +internal sealed class ControlledCall : IDisposable +{ + private CancellationTokenSource? _cancellation; + private Task? _request; + private string? _peer; + private DateTimeOffset? _startedAt; + private DateTimeOffset? _cancelledAt; + + public ControlledCallSnapshot Snapshot => new( + _peer, + _startedAt, + _cancelledAt, + _cancellation?.IsCancellationRequested == true, + _request?.IsCompleted == true, + _request?.Status.ToString(), + _request?.Exception?.ToString()); + + public void Start(string peer, Func invoke) + { + if (_request is not null) + { + throw new InvalidOperationException("A controlled RPC has already been started."); + } + + _peer = peer; + _startedAt = DateTimeOffset.UtcNow; + _cancellation = new CancellationTokenSource(); + _request = invoke(_cancellation.Token); + _ = _request.ContinueWith( + static completed => _ = completed.Exception, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + } + + public async Task Cancel(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var source = _cancellation ?? throw new InvalidOperationException("No controlled RPC has been started."); + var request = _request ?? throw new InvalidOperationException("The controlled RPC was not created."); + if (request.IsCompleted) + { + throw new InvalidOperationException($"The RPC completed before explicit cancellation: {request.Status}.", request.Exception); + } + + _cancelledAt = DateTimeOffset.UtcNow; + await source.CancelAsync().WaitAsync(cancellationToken); + try + { + await request.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) when (source.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + return; + } + + throw new InvalidOperationException("The RPC completed successfully instead of observing cancellation."); + } + + public void Dispose() => _cancellation?.Dispose(); +} + +internal static class StateComparison +{ + private static readonly JsonSerializerOptions Options = new() { IncludeFields = true }; + + public static string Serialize(T value) => JsonSerializer.Serialize(value, Options); +} + +// Closing the gate is an admission barrier, not a timer or an assumption about queue expiry. +internal sealed class InFlightCallGate +{ + public const string ClosedMessage = "Harness PushBroadcast gate is closed."; + private readonly object _lock = new(); + private bool _blocked; + private int _inFlight; + private long _admitted; + private long _rejected; + private TaskCompletionSource? _drained; + + public TreeGateSnapshot Snapshot + { + get + { + lock (_lock) + { + return new(_blocked, _inFlight, _admitted, _rejected); + } + } + } + + public async Task Invoke(Func action) + { + lock (_lock) + { + if (_blocked) + { + _rejected++; + throw new InvalidOperationException(ClosedMessage); + } + + _admitted++; + _inFlight++; + } + + try + { + await action(); + } + finally + { + lock (_lock) + { + if (--_inFlight == 0) + { + _drained?.TrySetResult(); + } + } + } + } + + public async Task BlockAndDrain(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Task pending; + lock (_lock) + { + _blocked = true; + pending = _inFlight == 0 + ? Task.CompletedTask + : (_drained ??= new(TaskCreationOptions.RunContinuationsAsynchronously)).Task; + } + + await pending.WaitAsync(cancellationToken); + } + + public void Open() + { + lock (_lock) + { + if (_inFlight != 0) + { + throw new InvalidOperationException("Cannot reopen the tree gate before draining admitted calls."); + } + + _blocked = false; + _drained = null; + } + } +} + +internal sealed record ApplyEvidence( + string Namespace, + string Key, + long FromVersion, + long ToVersion, + string Result, + string? Peer); + +internal sealed record NodeSnapshot +{ + public DateTimeOffset CapturedAtUtc { get; init; } + public required BinaryIdentity Identity { get; init; } + public required string Address { get; init; } + public bool Enabled { get; init; } + public bool NamespaceEnabled { get; init; } + public long MembershipVersion { get; init; } + public required SortedDictionary Membership { get; init; } + public required SortedDictionary Load { get; init; } + public required SortedDictionary LoadVersions { get; init; } + public required string[] ActiveMembers { get; init; } + public required string[] UnconfirmedPeers { get; init; } + public required string[] OriginatorTargets { get; init; } + public required string[] ForwardingTargets { get; init; } + public string[] TopologyMembers { get; init; } = []; + public int Fanout { get; init; } + public required Dictionary Metrics { get; init; } + public required ApplyEvidence[] Applies { get; init; } + public long TransportBytesWritten { get; init; } + public long TransportBytesRead { get; init; } + public double SocketBytesSent { get; init; } + public double SocketBytesReceived { get; init; } + public long SocketCounterSamples { get; init; } + public double CpuMilliseconds { get; init; } + public long AllocatedBytes { get; init; } + public long ManagedHeapBytes { get; init; } + public long WorkingSetBytes { get; init; } + public long PrivateBytes { get; init; } + public int PendingControlCalls { get; init; } + public int StartedControlCalls { get; init; } + public int CancelledControlCalls { get; init; } + public int ControlCancellationSignals { get; init; } + public bool ControlTokenCanBeCanceled { get; init; } + public bool ControlTokenCancelledOnEntry { get; init; } + public DateTimeOffset? ControlStartedAtUtc { get; init; } + public DateTimeOffset? ControlCancellationObservedAtUtc { get; init; } + public ControlledCallSnapshot? OutboundControlCall { get; init; } + public bool Partitioned { get; init; } + public bool LegacyGossipSuppressed { get; init; } + public bool MembershipReadsFrozen { get; init; } + public int? RemoteProcessId { get; init; } + public string? ProbeError { get; init; } + public bool ProbeTimedOut { get; init; } + public long? RepairFromVersion { get; init; } + public long? HeartbeatTicks { get; init; } + public TreeGateSnapshot? TreeGate { get; init; } + public bool? TreeProbeRejected { get; init; } + public StateComparisonProbe? ComparisonProbe { get; init; } +} diff --git a/test/Dissemination.IntegrationHarness/Silo/ControlTarget.cs b/test/Dissemination.IntegrationHarness/Silo/ControlTarget.cs new file mode 100644 index 00000000000..984dab6172d --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Silo/ControlTarget.cs @@ -0,0 +1,110 @@ +using Orleans.Runtime; +using Orleans.Runtime.MembershipService; + +namespace Orleans.Dissemination.IntegrationHarness; + +[Alias("dissemination-evidence-control-v1")] +internal interface IControlTarget : ISystemTarget +{ + [Alias("echo-v1")] + Task Echo(CancellationToken cancellationToken); + + [Alias("hold-v1")] + Task Hold(CancellationToken cancellationToken); +} + +internal sealed class ControlTarget : SystemTarget, IControlTarget +{ + public static readonly GrainType TargetType = SystemTargetGrainId.CreateGrainType("test.dissemination-evidence"); + private int _pending; + private int _started; + private int _cancelled; + private int _cancellationSignals; + private int _canBeCanceled; + private int _cancelledOnEntry; + private long _startedAtTicks; + private long _cancellationObservedAtTicks; + + public ControlTarget(SystemTargetShared shared) : base(TargetType, shared) => + shared.ActivationDirectory.RecordNewTarget(this); + + public int Pending => Volatile.Read(ref _pending); + public int Started => Volatile.Read(ref _started); + public int Cancelled => Volatile.Read(ref _cancelled); + public int CancellationSignals => Volatile.Read(ref _cancellationSignals); + public bool TokenCanBeCanceled => Volatile.Read(ref _canBeCanceled) != 0; + public bool TokenCancelledOnEntry => Volatile.Read(ref _cancelledOnEntry) != 0; + public DateTimeOffset? StartedAtUtc => Timestamp(Interlocked.Read(ref _startedAtTicks)); + public DateTimeOffset? CancellationObservedAtUtc => Timestamp(Interlocked.Read(ref _cancellationObservedAtTicks)); + + public Task Echo(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(Environment.ProcessId); + } + + public async Task Hold(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _pending); + Interlocked.Increment(ref _started); + Volatile.Write(ref _canBeCanceled, cancellationToken.CanBeCanceled ? 1 : 0); + Volatile.Write(ref _cancelledOnEntry, cancellationToken.IsCancellationRequested ? 1 : 0); + Interlocked.Exchange(ref _startedAtTicks, DateTime.UtcNow.Ticks); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + Interlocked.Exchange(ref _cancellationObservedAtTicks, DateTime.UtcNow.Ticks); + Interlocked.Increment(ref _cancellationSignals); + throw; + } + finally + { + if (cancellationToken.IsCancellationRequested) + { + Interlocked.Increment(ref _cancelled); + } + + Interlocked.Decrement(ref _pending); + } + } + + private static DateTimeOffset? Timestamp(long ticks) => + ticks == 0 ? null : new DateTimeOffset(ticks, TimeSpan.Zero); +} + +internal sealed class GatedMembershipGossiper(MembershipGossiper inner) : IMembershipGossiper +{ + private int _suppressed; + + public bool Suppressed + { + get => Volatile.Read(ref _suppressed) != 0; + set => Volatile.Write(ref _suppressed, value ? 1 : 0); + } + + public Task GossipToRemoteSilos( + List gossipPartners, + MembershipTableSnapshot snapshot, + SiloAddress updatedSilo, +#if NEW_RUNTIME + SiloStatus updatedStatus, + CancellationToken cancellationToken) +#else + SiloStatus updatedStatus) +#endif + { +#if NEW_RUNTIME + cancellationToken.ThrowIfCancellationRequested(); + return Suppressed + ? Task.CompletedTask + : inner.GossipToRemoteSilos(gossipPartners, snapshot, updatedSilo, updatedStatus, cancellationToken); +#else + return Suppressed + ? Task.CompletedTask + : inner.GossipToRemoteSilos(gossipPartners, snapshot, updatedSilo, updatedStatus); +#endif + } +} diff --git a/test/Dissemination.IntegrationHarness/Silo/Dissemination.Silo.csproj b/test/Dissemination.IntegrationHarness/Silo/Dissemination.Silo.csproj new file mode 100644 index 00000000000..9a4dd8239cc --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Silo/Dissemination.Silo.csproj @@ -0,0 +1,18 @@ + + + net10.0 + Exe + + Orleans.Runtime.Tests + Orleans.Dissemination.IntegrationHarness + true + New + $(DefineConstants);NEW_RUNTIME + + + + + + + + diff --git a/test/Dissemination.IntegrationHarness/Silo/FileMembershipTable.cs b/test/Dissemination.IntegrationHarness/Silo/FileMembershipTable.cs new file mode 100644 index 00000000000..4e354dd2f77 --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Silo/FileMembershipTable.cs @@ -0,0 +1,238 @@ +using System.Text.Json; +using Orleans.Runtime; + +namespace Orleans.Dissemination.IntegrationHarness; + +// The wire schema is deliberately independent of either Orleans serializer version. +// This is a single-machine CI provider, not a production storage implementation. +internal sealed class FileMembershipTable(NodeConfiguration configuration) : IMembershipTable +{ + private readonly string _dataPath = Path.Combine(configuration.MembershipDirectory, "membership.json"); + private readonly string _lockPath = Path.Combine(configuration.MembershipDirectory, "membership.lock"); + private TableState? _frozen; + + public bool ReadsFrozen => Volatile.Read(ref _frozen) is not null; + + public async Task FreezeReads(bool freeze, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var state = freeze ? await Access(static state => state, write: false, cancellationToken) : null; + Volatile.Write(ref _frozen, state); + } + +#if NEW_RUNTIME + Task IMembershipTable.InitializeMembershipTable(bool tryInitTableVersion) => + InitializeMembershipTableAsync(tryInitTableVersion, CancellationToken.None); + + Task IMembershipTable.DeleteMembershipTableEntries(string clusterId) => + DeleteMembershipTableEntriesAsync(clusterId, CancellationToken.None); + + Task IMembershipTable.CleanupDefunctSiloEntries(DateTimeOffset beforeDate) => + CleanupDefunctSiloEntriesAsync(beforeDate, CancellationToken.None); + + Task IMembershipTable.ReadRow(SiloAddress key) => ReadRowAsync(key, CancellationToken.None); + + Task IMembershipTable.ReadAll() => ReadAllAsync(CancellationToken.None); + + Task IMembershipTable.InsertRow(MembershipEntry entry, TableVersion tableVersion) => + InsertRowAsync(entry, tableVersion, CancellationToken.None); + + Task IMembershipTable.UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion) => + UpdateRowAsync(entry, etag, tableVersion, CancellationToken.None); + + Task IMembershipTable.UpdateIAmAlive(MembershipEntry entry) => UpdateIAmAliveAsync(entry, CancellationToken.None); +#else + public Task InitializeMembershipTable(bool tryInitTableVersion) => + InitializeMembershipTableAsync(tryInitTableVersion, CancellationToken.None); + + public Task DeleteMembershipTableEntries(string clusterId) => + DeleteMembershipTableEntriesAsync(clusterId, CancellationToken.None); + + public Task CleanupDefunctSiloEntries(DateTimeOffset beforeDate) => + CleanupDefunctSiloEntriesAsync(beforeDate, CancellationToken.None); + + public Task ReadRow(SiloAddress key) => ReadRowAsync(key, CancellationToken.None); + + public Task ReadAll() => ReadAllAsync(CancellationToken.None); + + public Task InsertRow(MembershipEntry entry, TableVersion tableVersion) => + InsertRowAsync(entry, tableVersion, CancellationToken.None); + + public Task UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion) => + UpdateRowAsync(entry, etag, tableVersion, CancellationToken.None); + + public Task UpdateIAmAlive(MembershipEntry entry) => UpdateIAmAliveAsync(entry, CancellationToken.None); +#endif + + public Task InitializeMembershipTableAsync(bool tryInitTableVersion, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Directory.CreateDirectory(configuration.MembershipDirectory); + return Access(static _ => true, write: true, cancellationToken); + } + + public Task DeleteMembershipTableEntriesAsync(string clusterId, CancellationToken cancellationToken) => Access(state => + { + if (!StringComparer.Ordinal.Equals(clusterId, configuration.ClusterId)) + { + throw new InvalidOperationException("The harness provider cannot delete another cluster."); + } + + state.Rows.Clear(); + state.Version++; + return true; + }, write: true, cancellationToken); + + public Task CleanupDefunctSiloEntriesAsync(DateTimeOffset beforeDate, CancellationToken cancellationToken) => Access(state => + { + foreach (var key in state.Rows.Where(pair => + pair.Value.Entry.Status == (int)SiloStatus.Dead + && pair.Value.Entry.IAmAliveTime < beforeDate.UtcDateTime).Select(pair => pair.Key).ToArray()) + { + state.Rows.Remove(key); + } + + return true; + }, write: true, cancellationToken); + + public async Task ReadRowAsync(SiloAddress key, CancellationToken cancellationToken) + { + var all = await ReadAllAsync(cancellationToken); + return new MembershipTableData(all.Members.Where(row => row.Item1.SiloAddress.Equals(key)).ToList(), all.Version); + } + + public async Task ReadAllAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var state = Volatile.Read(ref _frozen) ?? await Access(static state => state, write: false, cancellationToken); + return new MembershipTableData( + state.Rows.Values.Select(row => Tuple.Create(row.Entry.ToEntry(), row.Etag)).ToList(), + new TableVersion(state.Version, state.Etag)); + } + + public Task InsertRowAsync(MembershipEntry entry, TableVersion tableVersion, CancellationToken cancellationToken) => Access(state => + { + var key = entry.SiloAddress.ToParsableString(); + if (state.Rows.ContainsKey(key) || !StringComparer.Ordinal.Equals(tableVersion.VersionEtag, state.Etag)) + { + return false; + } + + state.Rows.Add(key, new Row(EntryData.From(entry), Guid.NewGuid().ToString("N"))); + state.Version = tableVersion.Version; + state.Etag = Guid.NewGuid().ToString("N"); + return true; + }, write: true, cancellationToken); + + public Task UpdateRowAsync(MembershipEntry entry, string etag, TableVersion tableVersion, CancellationToken cancellationToken) => Access(state => + { + var key = entry.SiloAddress.ToParsableString(); + if (!state.Rows.TryGetValue(key, out var row) + || !StringComparer.Ordinal.Equals(row.Etag, etag) + || !StringComparer.Ordinal.Equals(tableVersion.VersionEtag, state.Etag)) + { + return false; + } + + state.Rows[key] = new Row(EntryData.From(entry), Guid.NewGuid().ToString("N")); + state.Version = tableVersion.Version; + state.Etag = Guid.NewGuid().ToString("N"); + return true; + }, write: true, cancellationToken); + + public Task UpdateIAmAliveAsync(MembershipEntry entry, CancellationToken cancellationToken) => Access(state => + { + if (state.Rows.TryGetValue(entry.SiloAddress.ToParsableString(), out var row)) + { + state.Rows[entry.SiloAddress.ToParsableString()] = row with + { + Entry = row.Entry with { IAmAliveTime = entry.IAmAliveTime }, + }; + } + + return true; + }, write: true, cancellationToken); + + private async Task Access(Func action, bool write, CancellationToken cancellationToken) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(10)); + FileStream lease; + while (true) + { + deadline.Token.ThrowIfCancellationRequested(); + try + { + lease = new FileStream(_lockPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + break; + } + catch (IOException) + { + await Task.Delay(10, deadline.Token); + } + } + + await using (lease) + { + var state = File.Exists(_dataPath) + ? JsonSerializer.Deserialize(await File.ReadAllTextAsync(_dataPath, deadline.Token))! + : new TableState(); + deadline.Token.ThrowIfCancellationRequested(); + var result = action(state); + if (write) + { + // The sibling file is in the owned artifact directory, never the system temporary directory. + var staging = _dataPath + ".next"; + await File.WriteAllTextAsync(staging, JsonSerializer.Serialize(state), deadline.Token); + deadline.Token.ThrowIfCancellationRequested(); + File.Move(staging, _dataPath, overwrite: true); + } + + return result; + } + } + + private sealed class TableState + { + public int Version { get; set; } + public string Etag { get; set; } = "0"; + public Dictionary Rows { get; set; } = []; + } + + private sealed record Row(EntryData Entry, string Etag); + + internal sealed record EntryData( + string Address, + int Status, + int ProxyPort, + string HostName, + string SiloName, + string? RoleName, + int UpdateZone, + int FaultZone, + DateTime StartTime, + DateTime IAmAliveTime, + Dictionary Suspects) + { + public static EntryData From(MembershipEntry entry) => new( + entry.SiloAddress.ToParsableString(), (int)entry.Status, entry.ProxyPort, + entry.HostName, entry.SiloName, entry.RoleName, entry.UpdateZone, entry.FaultZone, + entry.StartTime, entry.IAmAliveTime, + entry.SuspectTimes?.ToDictionary(pair => pair.Item1.ToParsableString(), pair => pair.Item2) ?? []); + + public MembershipEntry ToEntry() => new() + { + SiloAddress = SiloAddress.FromParsableString(Address), + Status = (SiloStatus)Status, + ProxyPort = ProxyPort, + HostName = HostName, + SiloName = SiloName, + RoleName = RoleName, + UpdateZone = UpdateZone, + FaultZone = FaultZone, + StartTime = StartTime, + IAmAliveTime = IAmAliveTime, + SuspectTimes = Suspects.Select(pair => Tuple.Create(SiloAddress.FromParsableString(pair.Key), pair.Value)).ToList(), + }; + } +} diff --git a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs new file mode 100644 index 00000000000..08c0335ca57 --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs @@ -0,0 +1,227 @@ +#if NEW_RUNTIME +using System.Collections.Immutable; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.MembershipService; + +namespace Orleans.Dissemination.IntegrationHarness; + +internal static class NewRuntime +{ + public static void Configure(IServiceCollection services, NodeConfiguration configuration) + { + // The disabled leg must exercise untouched production defaults, not explicit Enabled=false. + if (!configuration.Enabled) + { + return; + } + + if (configuration.FastRecovery) + { + services.AddSingleton(); + services.AddSingleton(provider => provider.GetRequiredService()); + } + + services.Configure(options => + { + options.Enabled = true; + if (configuration.FastRecovery) + { + options.Overlay.AntiEntropyInterval = TimeSpan.FromMilliseconds(250); + options.Overlay.AntiEntropyPeerCount = 2; + options.Overlay.MinFanOutFactor = 2; + options.Overlay.MaxFanOutFactor = 2; + options.MaxConcurrentSends = 4; + } + }); + services.Configure(options => Enable(options.Dissemination, configuration.FastRecovery)); + services.Configure(options => Enable(options.Dissemination, configuration.FastRecovery)); + } + + private static void Enable(DisseminationNamespaceOptions options, bool fastRecovery) + { + options.Enabled = true; + if (fastRecovery) + { + options.ExpectedUpdateCadence = TimeSpan.FromMilliseconds(100); + options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(25); + options.StaleItemTtl = TimeSpan.FromSeconds(1); + } + } + + public static NodeSnapshot Decorate(IServiceProvider services, NodeSnapshot snapshot) + { + var options = services.GetRequiredService>().Value; + if (!options.Enabled) + { + return snapshot with + { + NamespaceEnabled = services.GetRequiredService>().Value.Dissemination.Enabled, + }; + } + + var ns = services.GetRequiredService(); + var topology = services.GetRequiredService().CurrentSnapshots.ActiveMembers; + return snapshot with + { + Enabled = options.Enabled, + NamespaceEnabled = ns.Options.Enabled, + UnconfirmedPeers = services.GetRequiredService() + .GetUnconfirmedPeers(ns).Select(address => address.ToParsableString()).Order(StringComparer.Ordinal).ToArray(), + OriginatorTargets = topology.OriginatorTreeTargets.Select(address => address.ToParsableString()).ToArray(), + ForwardingTargets = topology.ForwardingTreeTargets.Select(address => address.ToParsableString()).ToArray(), + TopologyMembers = topology.Members.Select(address => address.ToParsableString()).ToArray(), + Fanout = options.Overlay.GetFanOutFactor(topology.Members.Length), + TreeGate = services.GetService()?.Snapshot, + }; + } + + public static async Task Execute(IServiceProvider services, Command command, CancellationToken cancellationToken) + { + var manager = services.GetRequiredService(); + var dissemination = services.GetRequiredService(); + var ns = services.GetRequiredService(); + var local = services.GetRequiredService().SiloAddress; + switch (command.Operation) + { + case "block-tree": + { + var gate = services.GetRequiredService(); + if (command.Value) + { + await gate.BlockAndDrain(cancellationToken); + } + else + { + gate.Open(); + } + + break; + } + case "probe-tree": + { + var peer = services.GetRequiredService() + .GetSystemTarget( + Constants.DisseminationSystemTargetType, SiloAddress.FromParsableString(command.Peer!)); + try + { + await peer.PushBroadcast(new() { Sender = local }, cancellationToken); + return Program.Snapshot(services) with { TreeProbeRejected = false }; + } + catch (InvalidOperationException exception) when (exception.Message == InFlightCallGate.ClosedMessage) + { + return Program.Snapshot(services) with { TreeProbeRejected = true }; + } + } + case "probe": + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(8)); + try + { + var peer = services.GetRequiredService() + .GetSystemTarget( + Constants.DisseminationSystemTargetType, SiloAddress.FromParsableString(command.Peer!)); + var response = await peer.ExchangeAntiEntropy(new() + { + Sender = local, + SupportedNamespaces = [ns.Name], + }, deadline.Token); + return Program.Snapshot(services) with + { + ProbeError = response.UnsupportedNamespaces.Contains(ns.Name) ? "namespace-unsupported" : null, + }; + } + catch (Exception exception) when (!cancellationToken.IsCancellationRequested) + { + return Program.Snapshot(services) with + { + ProbeError = $"{exception.GetType().FullName}: {exception.Message}", + ProbeTimedOut = deadline.IsCancellationRequested || exception is TimeoutException, + }; + } + } + case "publish-membership": + await ns.PublishAsync(dissemination, manager.CurrentSnapshot, cancellationToken); + break; + case "membership-history": + { + if (!services.GetRequiredService().Suppressed) + { + throw new InvalidOperationException("History injection requires legacy gossip isolation."); + } + + var table = services.GetRequiredService(); + await table.FreezeReads(false, cancellationToken); + for (var index = 0; index < command.Count; index++) + { + cancellationToken.ThrowIfCancellationRequested(); + var current = await table.ReadAllAsync(cancellationToken); + var row = current.Members.Single(row => row.Item1.SiloAddress.Equals(local)); + row.Item1.HostName = $"retained-history-{current.Version.Version + 1}"; + if (!await table.UpdateRowAsync(row.Item1, row.Item2, current.Version.Next(), cancellationToken)) + { + throw new InvalidOperationException("Unexpected concurrent membership writer during isolated history test."); + } + + await manager.Refresh(null, cancellationToken); + await ns.PublishAsync(dissemination, manager.CurrentSnapshot, cancellationToken); + } + + await table.FreezeReads(true, cancellationToken); + var repair = ns.CreateRepair(new( + DisseminationKey.Default, command.Version, null, 100, 1024 * 1024, 1024 * 1024)); + if (repair.Status != DisseminationRepairStatus.Produced || repair.Values.Length != 1) + { + throw new InvalidOperationException($"Expected a production membership repair, got {repair.Status}."); + } + + return Program.Snapshot(services) with { RepairFromVersion = repair.Values[0].FromVersion }; + } + case "membership-heartbeat": + { + if (!services.GetRequiredService().Suppressed + || !services.GetRequiredService().ReadsFrozen) + { + throw new InvalidOperationException("Heartbeat injection requires storage and legacy gossip isolation."); + } + + var current = manager.CurrentSnapshot; + var entries = current.Entries.ToBuilder(); + var entry = FileMembershipTable.EntryData.From(entries[local]).ToEntry(); + entry.IAmAliveTime = entry.IAmAliveTime.AddHours(1); + entries[local] = entry; + await manager.ProcessGossipSnapshot(new(current.Version, entries.ToImmutable()), cancellationToken); + // Deliberately no Publish: only same-version fingerprint anti-entropy can carry this heartbeat. + return Program.Snapshot(services) with { HeartbeatTicks = entry.IAmAliveTime.Ticks }; + } + default: + throw new NotSupportedException(command.Operation); + } + + return Program.Snapshot(services); + } +} + +// System targets use the normal incoming-call filter pipeline. This filter blocks only tree +// application; native ExchangeAntiEntropy requests/responses continue over the real connection. +internal sealed class DisseminationTreeGate : IIncomingGrainCallFilter +{ + private readonly InFlightCallGate _gate = new(); + + public TreeGateSnapshot Snapshot => _gate.Snapshot; + + public Task BlockAndDrain(CancellationToken cancellationToken) => _gate.BlockAndDrain(cancellationToken); + + public void Open() => _gate.Open(); + + public Task Invoke(IIncomingGrainCallContext context) => + context.InterfaceMethod.DeclaringType == typeof(IDisseminationSystemTarget) + && context.InterfaceMethod.Name == nameof(IDisseminationSystemTarget.PushBroadcast) + ? _gate.Invoke(context.Invoke) + : context.Invoke(); +} +#endif diff --git a/test/Dissemination.IntegrationHarness/Silo/Observation.cs b/test/Dissemination.IntegrationHarness/Silo/Observation.cs new file mode 100644 index 00000000000..95b4e10c6c2 --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Silo/Observation.cs @@ -0,0 +1,378 @@ +using System.Buffers; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Diagnostics.Tracing; +using System.IO.Pipelines; +using Microsoft.AspNetCore.Connections; + +namespace Orleans.Dissemination.IntegrationHarness; + +internal sealed class Observation : IDisposable, IObserver, IObserver> +{ + private readonly object _lock = new(); + private readonly MeterListener _meters = new(); + private readonly List _subscriptions = []; + private readonly Dictionary _metrics = []; + private readonly Queue _applies = []; + private readonly SocketCounters _sockets = new(); + private readonly object _connectionLock = new(); + private readonly Dictionary _connections = []; + private readonly HashSet _drainingConnections = []; + private Func? _blockedConnections; + private int _partitioned; + private long _bytesWritten; + private long _bytesRead; + + private readonly bool _captureProvenance; + + public Observation(bool captureProvenance) + { + _captureProvenance = captureProvenance; + _meters.InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name.StartsWith("Microsoft.Orleans", StringComparison.Ordinal) + && (instrument.Name is "orleans-messaging-sent-messages-size" or "orleans-messaging-received-messages-size" + || instrument.Name.StartsWith("orleans-dissemination-", StringComparison.Ordinal))) + { + listener.EnableMeasurementEvents(instrument); + } + }; + _meters.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _meters.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _meters.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _meters.Start(); + _subscriptions.Add(DiagnosticListener.AllListeners.Subscribe(this)); + } + + public bool Partitioned => Volatile.Read(ref _partitioned) != 0; + public long BytesWritten => Interlocked.Read(ref _bytesWritten); + public long BytesRead => Interlocked.Read(ref _bytesRead); + public (double Sent, double Received, long Samples) SocketSnapshot => _sockets.Snapshot(); + + public void Partition() + { + ConnectionLifetime[] connections; + lock (_connectionLock) + { + Volatile.Write(ref _partitioned, 1); + connections = CaptureConnectionsUnsafe(static _ => true); + } + + foreach (var connection in connections) + { + connection.Context.Abort(new ConnectionAbortedException("Harness partition")); + } + } + + public async Task DrainConnections( + Func matches, + Func close, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ConnectionLifetime[] connections; + lock (_connectionLock) + { + _blockedConnections = matches; + connections = CaptureConnectionsUnsafe(matches); + } + + var closing = new List(connections.Length * 2); + foreach (var connection in connections) + { + if (!connection.Completion.Task.IsCompleted) + { + connection.Context.Abort(new ConnectionAbortedException("Harness partition")); + } + + var closeTask = close(connection.Context); + lock (_connectionLock) + { + connection.CloseTask = closeTask; + } + + closing.Add(closeTask); + closing.Add(connection.Completion.Task); + } + + var drained = Task.WhenAll(closing); + _ = drained.ContinueWith( + static task => _ = task.Exception, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + await drained.WaitAsync(cancellationToken); + } + + public void ResumeConnections() + { + lock (_connectionLock) + { + if (_drainingConnections.Any(static connection => + !connection.Completion.Task.IsCompleted || connection.CloseTask is not { IsCompletedSuccessfully: true })) + { + throw new InvalidOperationException("Partitioned connection middleware must drain before reopening."); + } + + _drainingConnections.Clear(); + _blockedConnections = null; + Volatile.Write(ref _partitioned, 0); + } + } + + private ConnectionLifetime[] CaptureConnectionsUnsafe(Func matches) + { + foreach (var connection in _connections.Values) + { + if (matches(connection.Context)) + { + _drainingConnections.Add(connection); + } + } + + return [.. _drainingConnections]; + } + + public async Task Connection(ConnectionContext context, ConnectionDelegate next) + { + ConnectionLifetime? lifetime = null; + lock (_connectionLock) + { + if (!Partitioned && _blockedConnections?.Invoke(context) != true) + { + lifetime = new(context); + _connections.Add(context, lifetime); + } + } + + if (lifetime is null) + { + context.Abort(new ConnectionAbortedException("Harness partition")); + return; + } + + var original = context.Transport; + context.Transport = new CountingDuplexPipe(original, this); + try + { + await next(context); + } + finally + { + context.Transport = original; + lock (_connectionLock) + { + _connections.Remove(context); + lifetime.Completion.SetResult(); + } + } + } + + private sealed class ConnectionLifetime(ConnectionContext context) + { + public ConnectionContext Context { get; } = context; + public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Task? CloseTask { get; set; } + } + + public Dictionary Metrics() + { + _meters.RecordObservableInstruments(); + lock (_lock) + { + return new(_metrics); + } + } + + public ApplyEvidence[] Applies() + { + lock (_lock) + { + return _applies.ToArray(); + } + } + + private void Record(Instrument instrument, double value, ReadOnlySpan> tags) + { + var tagValues = new string[tags.Length]; + for (var index = 0; index < tags.Length; index++) + { + tagValues[index] = $"{tags[index].Key}={tags[index].Value}"; + } + + Array.Sort(tagValues, StringComparer.Ordinal); + var key = $"{instrument.Name}|{string.Join(';', tagValues)}"; + lock (_lock) + { + var previous = _metrics.GetValueOrDefault(key, new(0, 0)); + _metrics[key] = instrument.IsObservable + ? new(previous.Count + 1, value) + : new(previous.Count + 1, previous.Sum + value); + } + } + + public void OnNext(DiagnosticListener value) + { + if (_captureProvenance && value.Name == "Microsoft.Orleans.Dissemination") + { + lock (_lock) + { + _subscriptions.Add(value.Subscribe(this)); + } + } + } + + public void OnNext(KeyValuePair value) + { + if (value.Key != "Dissemination.ValueApply" || value.Value is not { } payload) + { + return; + } + + // Reflection keeps this observer identical in both binaries: the old runtime has no such type. + object? Property(string name) => payload.GetType().GetProperty(name)?.GetValue(payload); + var evidence = new ApplyEvidence( + Property("Namespace")?.ToString() ?? "", + Property("Key")?.ToString() ?? "", + Convert.ToInt64(Property("FromVersion"), System.Globalization.CultureInfo.InvariantCulture), + Convert.ToInt64(Property("ToVersion"), System.Globalization.CultureInfo.InvariantCulture), + Property("Result")?.ToString() ?? "", + Property("Peer")?.ToString()); + lock (_lock) + { + _applies.Enqueue(evidence); + while (_applies.Count > 4096) + { + _applies.Dequeue(); + } + } + } + + public void OnCompleted() { } + public void OnError(Exception error) { } + + public void Dispose() + { + _meters.Dispose(); + _sockets.Dispose(); + lock (_lock) + { + foreach (var subscription in _subscriptions) + { + subscription.Dispose(); + } + } + } + + private sealed class CountingDuplexPipe(IDuplexPipe inner, Observation owner) : IDuplexPipe + { + public PipeReader Input { get; } = new CountingReader(inner.Input, owner); + public PipeWriter Output { get; } = new CountingWriter(inner.Output, owner); + } + + private sealed class CountingWriter(PipeWriter inner, Observation owner) : PipeWriter + { + public override void Advance(int bytes) + { + inner.Advance(bytes); + Interlocked.Add(ref owner._bytesWritten, bytes); + } + + public override Memory GetMemory(int sizeHint = 0) => inner.GetMemory(sizeHint); + public override Span GetSpan(int sizeHint = 0) => inner.GetSpan(sizeHint); + public override void CancelPendingFlush() => inner.CancelPendingFlush(); + public override void Complete(Exception? exception = null) => inner.Complete(exception); + public override ValueTask FlushAsync(CancellationToken cancellationToken = default) => inner.FlushAsync(cancellationToken); + } + + private sealed class CountingReader(PipeReader inner, Observation owner) : PipeReader + { + private ReadOnlySequence _buffer; + + public override void AdvanceTo(SequencePosition consumed) => AdvanceTo(consumed, consumed); + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + Interlocked.Add(ref owner._bytesRead, _buffer.Slice(0, consumed).Length); + inner.AdvanceTo(consumed, examined); + _buffer = default; + } + + public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + var result = await inner.ReadAsync(cancellationToken); + _buffer = result.Buffer; + return result; + } + + public override bool TryRead(out ReadResult result) + { + if (!inner.TryRead(out result)) + { + return false; + } + + _buffer = result.Buffer; + return true; + } + + public override void CancelPendingRead() => inner.CancelPendingRead(); + public override void Complete(Exception? exception = null) => inner.Complete(exception); + } + + private sealed class SocketCounters : EventListener + { + private readonly object _lock = new(); + private double _sent; + private double _received; + private long _samples; + + protected override void OnEventSourceCreated(EventSource eventSource) + { + if (eventSource.Name == "System.Net.Sockets") + { + EnableEvents(eventSource, EventLevel.LogAlways, EventKeywords.All, + new Dictionary { ["EventCounterIntervalSec"] = "1" }); + } + } + + protected override void OnEventWritten(EventWrittenEventArgs eventData) + { + if (eventData.EventName != "EventCounters" + || eventData.Payload?.Count != 1 + || eventData.Payload[0] is not IDictionary counter + || !counter.TryGetValue("Name", out var name)) + { + return; + } + + var incremental = counter.TryGetValue("Increment", out var measurement); + if (!incremental && !counter.TryGetValue("Mean", out measurement)) + { + return; + } + + lock (_lock) + { + var value = Convert.ToDouble(measurement, System.Globalization.CultureInfo.InvariantCulture); + if ((string)name == "bytes-sent") + { + _sent = incremental ? _sent + value : value; + _samples++; + } + else if ((string)name == "bytes-received") + { + _received = incremental ? _received + value : value; + } + } + } + + public (double Sent, double Received, long Samples) Snapshot() + { + lock (_lock) + { + return (_sent, _received, _samples); + } + } + } +} diff --git a/test/Dissemination.IntegrationHarness/Silo/Program.cs b/test/Dissemination.IntegrationHarness/Silo/Program.cs new file mode 100644 index 00000000000..6e51e3218e2 --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Silo/Program.cs @@ -0,0 +1,409 @@ +using System.Diagnostics; +using System.Net; +using System.Reflection; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.AspNetCore.Connections; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.MembershipService; +using Orleans.Runtime.Messaging; +using Orleans.Runtime.Scheduler; +using Orleans.Statistics; +using Orleans.TestingHost; + +namespace Orleans.Dissemination.IntegrationHarness; + +internal static class Program +{ + private static readonly MethodInfo PublishMethod = typeof(DeploymentLoadPublisher).GetMethod( + "PublishStatistics", BindingFlags.Instance | BindingFlags.NonPublic)!; + + public static async Task Main(string[] args) + { + if (args is not ["--silo", var configurationPath]) + { + Console.Error.WriteLine("Expected --silo ."); + return 2; + } + + var configuration = JsonSerializer.Deserialize(await File.ReadAllTextAsync(configurationPath))!; +#if !NEW_RUNTIME + if (configuration.Enabled) + { + throw new InvalidOperationException("The pinned old runtime cannot be opted into dissemination."); + } +#endif + using var lifetime = new CancellationTokenSource(TimeSpan.FromMinutes(45)); + using var observation = new Observation(configuration.FastRecovery); + var membership = new FileMembershipTable(configuration); + using var host = TestClusterHostFactory.CreateSiloHost( + configuration.Name, + new ConfigurationBuilder().Build(), + builder => builder.ConfigureServices(services => + { + services.AddSingleton(configuration); + services.AddSingleton(observation); + services.AddSingleton(membership); + services.AddSingleton(membership); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(); + services.Configure(options => + { + options.ClusterId = configuration.ClusterId; + options.ServiceId = "dissemination-evidence"; + }); + services.Configure(options => options.SiloName = configuration.Name); + services.Configure(options => + { + options.AdvertisedIPAddress = IPAddress.Loopback; + options.SiloPort = configuration.SiloPort; + options.GatewayPort = 0; + }); + services.Configure(options => + { + // Partitions test dissemination repair, not the separately tested failure detector. + options.LivenessEnabled = false; + options.TableRefreshTimeout = TimeSpan.FromHours(1); + options.IAmAliveTablePublishTimeout = TimeSpan.FromHours(1); + options.MaxJoinAttemptTime = TimeSpan.FromSeconds(60); + }); + services.Configure(options => + options.DeploymentLoadPublisherRefreshTime = TimeSpan.FromHours(1)); + services.Configure(options => options.ResponseTimeout = TimeSpan.FromSeconds(5)); + services.Configure(options => options.ShutdownTimeout = TimeSpan.FromSeconds(15)); + services.Configure(options => + { + options.ConfigureSiloInboundConnection(connection => connection.Use(next => context => observation.Connection(context, next))); + options.ConfigureSiloOutboundConnection(connection => connection.Use(next => context => observation.Connection(context, next))); + }); + services.AddLogging(logging => + { + logging.SetMinimumLevel(LogLevel.Warning).AddConsole(options => + options.LogToStandardErrorThreshold = LogLevel.Trace); + if (configuration.ClusterId.StartsWith("bounded-shutdown-", StringComparison.Ordinal)) + { + logging.AddFilter("Orleans.Runtime.GrainCallCancellationManager", LogLevel.Debug); + } + }); +#if NEW_RUNTIME + NewRuntime.Configure(services, configuration); +#endif + })); + + _ = host.Services.GetRequiredService(); + var monitor = MonitorParent(configuration.ParentProcessId, lifetime); + try + { + using var startup = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token); + startup.CancelAfter(TimeSpan.FromSeconds(90)); + await host.StartAsync(startup.Token); + var publisher = host.Services.GetRequiredService(); + // Commands own publication cadence; lifecycle shutdown retains its disposable timer reference. + await publisher.RunOrQueueTask(() => + { + var field = typeof(DeploymentLoadPublisher).GetField("_publishTimer", BindingFlags.Instance | BindingFlags.NonPublic)!; + ((IDisposable)field.GetValue(publisher)!).Dispose(); + return Task.CompletedTask; + }); + await Write(new Response(0, Snapshot(host.Services), null)); + + while (await Console.In.ReadLineAsync(lifetime.Token) is { } line) + { + var command = JsonSerializer.Deserialize(line)!; + using var operation = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token); + operation.CancelAfter(TimeSpan.FromSeconds(60)); + try + { + var result = await Execute(host.Services, command, operation.Token); + await Write(new Response(command.Id, result, null)); + if (command.Operation == "stop") + { + break; + } + } + catch (Exception exception) + { + await Write(new Response(command.Id, null, exception.ToString())); + } + } + + using var shutdown = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + await host.StopAsync(shutdown.Token).WaitAsync(shutdown.Token); + // Drain the last one-second OS socket-counter bucket before recording shutdown cost. + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + await Write(new Response(-1, Snapshot(host.Services, detailed: true), null)); + return 0; + } + finally + { + await lifetime.CancelAsync(); + await monitor; + } + } + + private static async Task Execute(IServiceProvider services, Command command, CancellationToken cancellationToken) + { + var membership = services.GetRequiredService(); + var manager = services.GetRequiredService(); + var publisher = services.GetRequiredService(); + switch (command.Operation) + { + case "snapshot": + case "stop": + break; + case "measure": + return Snapshot(services, detailed: true); + case "verify-load-comparison": + { + var original = new EnvironmentStatistics(10, 20, 100, 100, 900, 900, 1000); + var changed = new EnvironmentStatistics(10, 35, 100, 100, 900, 900, 1000); + return Snapshot(services) with + { + ComparisonProbe = new( + StateComparison.Serialize(new { EnvironmentStatistics = original }), + StateComparison.Serialize(new { EnvironmentStatistics = changed }), + typeof(EnvironmentStatistics).GetFields(BindingFlags.Instance | BindingFlags.Public) + .Select(field => field.Name).ToArray()), + }; + } + case "refresh": + await manager.Refresh(null, cancellationToken); + break; + case "publish": + for (var index = 0; index < command.Count; index++) + { + await publisher.RunOrQueueTask(() => (Task)PublishMethod.Invoke(publisher, [cancellationToken])!) + .WaitAsync(cancellationToken); + } + + break; + case "partition": + { + var observation = services.GetRequiredService(); + if (command.Value) + { + observation.Partition(); + } + else + { + await DrainConnections(services, peer: null, cancellationToken); + observation.ResumeConnections(); + } + + break; + } + case "drain-connections": + await DrainConnections(services, command.Peer, cancellationToken); + break; + case "resume-connections": + services.GetRequiredService().ResumeConnections(); + break; + case "isolate-membership": + services.GetRequiredService().Suppressed = command.Value; + await membership.FreezeReads(command.Value, cancellationToken); + break; + case "forget-load": + await publisher.RunOrQueueTask(() => + { + cancellationToken.ThrowIfCancellationRequested(); + publisher.PeriodicStatistics.TryRemove(SiloAddress.FromParsableString(command.Peer!), out _); + return Task.CompletedTask; + }).WaitAsync(cancellationToken); + break; + case "retained-memory": + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + break; + case "echo": + { + var peer = services.GetRequiredService().GetSystemTarget( + ControlTarget.TargetType, SiloAddress.FromParsableString(command.Peer!)); + var processId = await peer.Echo(cancellationToken).WaitAsync(cancellationToken); + return Snapshot(services) with { RemoteProcessId = processId }; + } + case "probe-echo": + { + var peer = services.GetRequiredService().GetSystemTarget( + ControlTarget.TargetType, SiloAddress.FromParsableString(command.Peer!)); + using var probeCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + try + { + var request = peer.Echo(probeCancellation.Token); + request.Ignore(); + var processId = await request.WaitAsync(TimeSpan.FromSeconds(2), cancellationToken); + return Snapshot(services) with { RemoteProcessId = processId }; + } + catch (Exception exception) when (exception is ConnectionFailedException + or SiloUnavailableException or OrleansMessageRejectionException or TimeoutException) + { + return Snapshot(services) with + { + ProbeError = $"{exception.GetType().FullName}: {exception.Message}", + ProbeTimedOut = exception is TimeoutException, + }; + } + finally + { + await probeCancellation.CancelAsync(); + } + } + case "start-cancel-rpc": + { + var peer = services.GetRequiredService().GetSystemTarget( + ControlTarget.TargetType, SiloAddress.FromParsableString(command.Peer!)); + var call = services.GetRequiredService(); + // Match dissemination's silo-system-target calling context, not a hosted-client call. + await services.GetRequiredService().RunOrQueueTask(() => + { + cancellationToken.ThrowIfCancellationRequested(); + call.Start(command.Peer!, peer.Hold); + return Task.CompletedTask; + }).WaitAsync(cancellationToken); + + break; + } + case "cancel-started-rpc": + await services.GetRequiredService().Cancel(cancellationToken); + break; + default: +#if NEW_RUNTIME + return await NewRuntime.Execute(services, command, cancellationToken); +#else + throw new NotSupportedException($"The pinned old runtime does not support '{command.Operation}'."); +#endif + } + + return Snapshot(services); + } + + private static Task DrainConnections(IServiceProvider services, string? peer, CancellationToken cancellationToken) + { + var address = peer is null ? null : SiloAddress.FromParsableString(peer); + return services.GetRequiredService().DrainConnections( + context => + { + var connection = (SiloConnection)context.Features.Get()!; + // Inbound handshakes have no remote silo identity until their preamble completes. + return address is null || connection.RemoteSiloAddress is null || connection.RemoteSiloAddress.Equals(address); + }, + context => context.Features.Get()!.CloseAsync(new ConnectionAbortedException("Harness partition")), + cancellationToken); + } + + internal static NodeSnapshot Snapshot(IServiceProvider services, bool? detailed = null) + { + var includeDetails = detailed ?? services.GetRequiredService().FastRecovery; + var manager = services.GetRequiredService(); + var publisher = services.GetRequiredService(); + var observation = services.GetRequiredService(); + var membership = manager.CurrentSnapshot; + using var process = Process.GetCurrentProcess(); + var socket = observation.SocketSnapshot; + var result = new NodeSnapshot + { + CapturedAtUtc = DateTimeOffset.UtcNow, + Identity = Identity(), + Address = services.GetRequiredService().SiloAddress.ToParsableString(), + MembershipVersion = membership.Version.Value, + Membership = new(membership.Entries.ToDictionary( + pair => pair.Key.ToParsableString(), + pair => JsonSerializer.Serialize(FileMembershipTable.EntryData.From(pair.Value))), StringComparer.Ordinal), + ActiveMembers = membership.Entries.Values.Where(entry => entry.Status == SiloStatus.Active) + .Select(entry => entry.SiloAddress.ToParsableString()).Order(StringComparer.Ordinal).ToArray(), + Load = new(publisher.PeriodicStatistics.ToDictionary( + pair => pair.Key.ToParsableString(), pair => StateComparison.Serialize(pair.Value)), StringComparer.Ordinal), + LoadVersions = new(publisher.PeriodicStatistics.ToDictionary( + pair => pair.Key.ToParsableString(), pair => pair.Value.DateTime.Ticks), StringComparer.Ordinal), + UnconfirmedPeers = [], + OriginatorTargets = [], + ForwardingTargets = [], + Metrics = includeDetails ? observation.Metrics() : [], + Applies = includeDetails ? observation.Applies() : [], + TransportBytesWritten = observation.BytesWritten, + TransportBytesRead = observation.BytesRead, + SocketBytesSent = socket.Sent, + SocketBytesReceived = socket.Received, + SocketCounterSamples = socket.Samples, + CpuMilliseconds = process.TotalProcessorTime.TotalMilliseconds, + AllocatedBytes = GC.GetTotalAllocatedBytes(precise: true), + ManagedHeapBytes = GC.GetTotalMemory(forceFullCollection: false), + WorkingSetBytes = process.WorkingSet64, + PrivateBytes = process.PrivateMemorySize64, + PendingControlCalls = services.GetRequiredService().Pending, + StartedControlCalls = services.GetRequiredService().Started, + CancelledControlCalls = services.GetRequiredService().Cancelled, + ControlCancellationSignals = services.GetRequiredService().CancellationSignals, + ControlTokenCanBeCanceled = services.GetRequiredService().TokenCanBeCanceled, + ControlTokenCancelledOnEntry = services.GetRequiredService().TokenCancelledOnEntry, + ControlStartedAtUtc = services.GetRequiredService().StartedAtUtc, + ControlCancellationObservedAtUtc = services.GetRequiredService().CancellationObservedAtUtc, + OutboundControlCall = services.GetRequiredService().Snapshot, + Partitioned = observation.Partitioned, + LegacyGossipSuppressed = services.GetRequiredService().Suppressed, + MembershipReadsFrozen = services.GetRequiredService().ReadsFrozen, + }; +#if NEW_RUNTIME + result = NewRuntime.Decorate(services, result); +#endif + return result; + } + + private static BinaryIdentity Identity() + { + // Hash once: periodic snapshots must not read the runtime image repeatedly. + return CachedIdentity.Value; + } + + private static readonly Lazy CachedIdentity = new(() => + { + var assembly = typeof(Silo).Assembly; + var metadata = typeof(Program).Assembly.GetCustomAttributes() + .ToDictionary(attribute => attribute.Key, attribute => attribute.Value); + return new( + Environment.ProcessId, metadata["HarnessRuntime"]!, metadata["HarnessSourceRevision"]!, + assembly.GetName().Name!, assembly.GetName().Version!.ToString(), + assembly.GetCustomAttribute()!.InformationalVersion, + assembly.ManifestModule.ModuleVersionId.ToString(), + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(assembly.Location))), + assembly.Location, + assembly.GetType("Orleans.Runtime.Dissemination.DisseminationSystemTarget") is not null) + { + Assemblies = AppDomain.CurrentDomain.GetAssemblies() + .Where(candidate => !candidate.IsDynamic && candidate.GetName().Name!.StartsWith("Orleans", StringComparison.Ordinal)) + .ToDictionary(candidate => candidate.GetName().Name!, candidate => new AssemblyProof( + candidate.GetName().Version!.ToString(), + candidate.GetCustomAttribute()?.InformationalVersion ?? "", + candidate.ManifestModule.ModuleVersionId.ToString(), + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(candidate.Location))), + candidate.Location)), + }; + }); + + private static async Task MonitorParent(int parentProcessId, CancellationTokenSource lifetime) + { + try + { + using var parent = Process.GetProcessById(parentProcessId); + await parent.WaitForExitAsync(lifetime.Token); + await lifetime.CancelAsync(); + } + catch (ArgumentException) + { + await lifetime.CancelAsync(); + } + catch (OperationCanceledException) when (lifetime.IsCancellationRequested) + { + } + } + + private static Task Write(Response response) => Console.Out.WriteLineAsync("HARNESS " + JsonSerializer.Serialize(response)); +} diff --git a/test/Dissemination.IntegrationHarness/Tests/Dissemination.IntegrationHarness.Tests.csproj b/test/Dissemination.IntegrationHarness/Tests/Dissemination.IntegrationHarness.Tests.csproj new file mode 100644 index 00000000000..6ce835f241a --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Tests/Dissemination.IntegrationHarness.Tests.csproj @@ -0,0 +1,12 @@ + + + net10.0 + enable + + + + + + + + diff --git a/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs b/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs new file mode 100644 index 00000000000..a1184d3036a --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs @@ -0,0 +1,302 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.IO.Pipelines; +using System.Net; +using System.Net.Sockets; +using System.Text.Json; +using Microsoft.AspNetCore.Connections; +using Xunit; + +namespace Orleans.Dissemination.IntegrationHarness; + +public sealed class MeasurementTests +{ + [Fact] + public async Task ConnectionDrain_WaitsForInitializationAndCloseBeforeReopening() + { + using var observation = new Observation(captureProvenance: false); + await using var context = CreateConnectionContext("initializing"); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var closed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var middleware = observation.Connection(context, async _ => + { + entered.SetResult(); + await release.Task; + }); + await entered.Task.WaitAsync(TestContext.Current.CancellationToken); + observation.Partition(); + var closeCalls = 0; + var drain = observation.DrainConnections( + static _ => true, + connection => + { + Assert.Same(context, connection); + closeCalls++; + return closed.Task; + }, + TestContext.Current.CancellationToken); + try + { + Assert.Equal(1, closeCalls); + Assert.False(drain.IsCompleted); + Assert.Throws(observation.ResumeConnections); + await using var rejected = CreateConnectionContext("rejected"); + await observation.Connection(rejected, _ => throw new InvalidOperationException("A partitioned connection entered.")); + release.SetResult(); + await middleware; + Assert.False(drain.IsCompleted); + Assert.Throws(observation.ResumeConnections); + closed.SetResult(); + await drain.WaitAsync(TestContext.Current.CancellationToken); + observation.ResumeConnections(); + Assert.False(observation.Partitioned); + var resumed = false; + await using var reopened = CreateConnectionContext("reopened"); + await observation.Connection(reopened, _ => + { + resumed = true; + return Task.CompletedTask; + }); + Assert.True(resumed); + } + finally + { + release.TrySetResult(); + closed.TrySetResult(); + await middleware; + await drain; + } + } + + [Fact] + public async Task ConnectionDrain_CapturesUnidentifiedPeersAndPreservesHealthyConnections() + { + using var observation = new Observation(captureProvenance: false); + await using var unknown = CreateConnectionContext("unknown"); + await using var healthy = CreateConnectionContext("healthy"); + healthy.Items["peer"] = "healthy"; + var releaseUnknown = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseHealthy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var unknownMiddleware = observation.Connection(unknown, _ => releaseUnknown.Task); + var healthyMiddleware = observation.Connection(healthy, _ => releaseHealthy.Task); + using var cancellation = new CancellationTokenSource(); + var closed = new List(); + var drain = observation.DrainConnections( + context => !context.Items.TryGetValue("peer", out var peer) || Equals(peer, "partitioned"), + context => + { + closed.Add(context.ConnectionId); + return Task.CompletedTask; + }, + cancellation.Token); + try + { + Assert.Equal(new[] { "unknown" }, closed); + Assert.False(healthyMiddleware.IsCompleted); + await cancellation.CancelAsync(); + await Assert.ThrowsAnyAsync(() => drain); + await using var rejected = CreateConnectionContext("new-unknown"); + await observation.Connection(rejected, _ => throw new InvalidOperationException("An unidentified connection entered.")); + releaseUnknown.SetResult(); + await unknownMiddleware; + observation.ResumeConnections(); + Assert.False(healthyMiddleware.IsCompleted); + } + finally + { + releaseUnknown.TrySetResult(); + releaseHealthy.TrySetResult(); + await Task.WhenAll(unknownMiddleware, healthyMiddleware); + } + } + + [Fact] + public async Task ControlledCall_WaitsForRawCancellationAfterExplicitSignal() + { + var cancellationToken = TestContext.Current.CancellationToken; + using var call = new ControlledCall(); + var raw = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cancellationObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + CancellationToken actualToken = default; + call.Start("receiver", token => + { + actualToken = token; + return raw.Task; + }); + using var registration = actualToken.UnsafeRegister( + static state => ((TaskCompletionSource)state!).SetResult(), cancellationObserved); + Assert.True(actualToken.CanBeCanceled); + Assert.False(actualToken.IsCancellationRequested); + Assert.False(call.Snapshot.CancellationRequested); + Assert.False(call.Snapshot.RawTaskCompleted); + var cancelling = call.Cancel(cancellationToken); + await cancellationObserved.Task.WaitAsync(cancellationToken); + Assert.True(actualToken.IsCancellationRequested); + Assert.False(cancelling.IsCompleted); + Assert.False(call.Snapshot.RawTaskCompleted); + raw.SetCanceled(actualToken); + await cancelling.WaitAsync(cancellationToken); + Assert.True(call.Snapshot.CancellationRequested); + Assert.True(call.Snapshot.RawTaskCompleted); + Assert.Equal("Canceled", call.Snapshot.RawTaskStatus); + } + + [Fact] + public async Task ControlledCall_DoesNotTreatRawTimeoutAsCancellation() + { + using var call = new ControlledCall(); + var raw = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + call.Start("receiver", _ => raw.Task); + var cancelling = call.Cancel(TestContext.Current.CancellationToken); + raw.SetException(new TimeoutException("Raw RPC expired without observing cancellation.")); + var exception = await Assert.ThrowsAsync(() => cancelling); + Assert.Equal("Raw RPC expired without observing cancellation.", exception.Message); + Assert.True(call.Snapshot.CancellationRequested); + Assert.True(call.Snapshot.RawTaskCompleted); + Assert.Equal("Faulted", call.Snapshot.RawTaskStatus); + } + + [Fact] + public void StateComparison_ObservesNestedPublicReadonlyFieldChanges() + { + var original = new { Version = 7, EnvironmentStatistics = new ReadonlyStatistics(20) }; + var changed = new { Version = 7, EnvironmentStatistics = new ReadonlyStatistics(35) }; + // This is the previous comparison's blind spot: properties alone produce identical JSON. + Assert.Equal(JsonSerializer.Serialize(original), JsonSerializer.Serialize(changed)); + var before = StateComparison.Serialize(original); + var after = StateComparison.Serialize(changed); + Assert.NotEqual(before, after); + using var beforeDocument = JsonDocument.Parse(before); + using var afterDocument = JsonDocument.Parse(after); + Assert.Equal(20, beforeDocument.RootElement.GetProperty("EnvironmentStatistics").GetProperty("RawCpuUsagePercentage").GetSingle()); + Assert.Equal(35, afterDocument.RootElement.GetProperty("EnvironmentStatistics").GetProperty("RawCpuUsagePercentage").GetSingle()); + } + + [Fact] + public async Task InFlightCallGate_DrainsPriorCallsAndRejectsLaterCalls() + { + var cancellationToken = TestContext.Current.CancellationToken; + var gate = new InFlightCallGate(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var prior = gate.Invoke(async () => + { + entered.SetResult(); + await release.Task.WaitAsync(cancellationToken); + }); + try + { + await entered.Task.WaitAsync(cancellationToken); + var drained = gate.BlockAndDrain(cancellationToken); + Assert.False(drained.IsCompleted); + var rejected = await Assert.ThrowsAsync(() => gate.Invoke( + () => throw new InvalidOperationException("A blocked call must never be invoked."))); + Assert.Equal(InFlightCallGate.ClosedMessage, rejected.Message); + Assert.Equal(new TreeGateSnapshot(true, 1, 1, 1), gate.Snapshot); + release.SetResult(); + await prior.WaitAsync(cancellationToken); + await drained.WaitAsync(cancellationToken); + Assert.Equal(new TreeGateSnapshot(true, 0, 1, 1), gate.Snapshot); + await Assert.ThrowsAsync(() => gate.Invoke( + () => throw new InvalidOperationException("A call must remain blocked after the drain."))); + Assert.Equal(new TreeGateSnapshot(true, 0, 1, 2), gate.Snapshot); + gate.Open(); + await gate.Invoke(() => Task.CompletedTask); + Assert.Equal(new TreeGateSnapshot(false, 0, 2, 2), gate.Snapshot); + } + finally + { + release.TrySetResult(); + await prior.WaitAsync(cancellationToken); + } + } + + [Fact] + public void MessageObservation_KeepsRequestOneWayAndBytesDistinct() + { + using var observation = new Observation(captureProvenance: false); + using var meter = new Meter("Microsoft.Orleans.HarnessMeasurementTest"); + var histogram = meter.CreateHistogram("orleans-messaging-sent-messages-size"); + histogram.Record(97, new KeyValuePair("MessageDirection", "Request")); + histogram.Record(31, new KeyValuePair("MessageDirection", "OneWay")); + histogram.Record(17, new KeyValuePair("MessageDirection", "Response")); + var metrics = observation.Metrics(); + Assert.Equal(new MetricValue(1, 97), metrics["orleans-messaging-sent-messages-size|MessageDirection=Request"]); + Assert.Equal(new MetricValue(1, 31), metrics["orleans-messaging-sent-messages-size|MessageDirection=OneWay"]); + Assert.Equal(new MetricValue(1, 17), metrics["orleans-messaging-sent-messages-size|MessageDirection=Response"]); + } + + [Fact] + public async Task TransportObservation_CountsConsumedAndSubmittedBytes() + { + var cancellationToken = TestContext.Current.CancellationToken; + using var observation = new Observation(captureProvenance: false); + var input = new Pipe(); + var output = new Pipe(); + await using var context = new DefaultConnectionContext("instrumentation-test") + { + Transport = new TestPipe(input.Reader, output.Writer), + }; + var payload = new byte[] { 1, 2, 3, 4, 5 }; + await observation.Connection(context, async connection => + { + await input.Writer.WriteAsync(payload, cancellationToken); + var received = await connection.Transport.Input.ReadAsync(cancellationToken); + connection.Transport.Input.AdvanceTo(received.Buffer.End); + await connection.Transport.Output.WriteAsync(payload, cancellationToken); + }); + Assert.Equal(payload.Length, observation.BytesRead); + Assert.Equal(payload.Length, observation.BytesWritten); + await input.Reader.CompleteAsync(); + await input.Writer.CompleteAsync(); + await output.Reader.CompleteAsync(); + await output.Writer.CompleteAsync(); + } + + [Fact] + public async Task SocketObservation_ReportsActualLoopbackTransfer() + { + var cancellationToken = TestContext.Current.CancellationToken; + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + using var observation = new Observation(captureProvenance: false); + using var client = new TcpClient(); + var accepted = listener.AcceptTcpClientAsync(cancellationToken); + await client.ConnectAsync(IPAddress.Loopback, ((IPEndPoint)listener.LocalEndpoint).Port, cancellationToken); + using var server = await accepted; + await WaitFor(() => observation.SocketSnapshot.Samples > 0, cancellationToken); + var before = observation.SocketSnapshot; + var payload = new byte[4096]; + await client.GetStream().WriteAsync(payload, cancellationToken); + await server.GetStream().ReadExactlyAsync(payload, cancellationToken); + await WaitFor(() => + { + var snapshot = observation.SocketSnapshot; + return snapshot.Sent - before.Sent >= payload.Length && snapshot.Received - before.Received >= payload.Length; + }, cancellationToken); + } + + private static async Task WaitFor(Func predicate, CancellationToken cancellationToken) + { + var clock = Stopwatch.StartNew(); + while (!predicate()) + { + Assert.True(clock.Elapsed < TimeSpan.FromSeconds(8), "The OS socket EventCounters did not report the measured transfer."); + await Task.Delay(50, cancellationToken); + } + } + + private sealed record TestPipe(PipeReader Input, PipeWriter Output) : IDuplexPipe; + + private static DefaultConnectionContext CreateConnectionContext(string id) + { + var pipe = new Pipe(); + return new(id) { Transport = new TestPipe(pipe.Reader, pipe.Writer) }; + } + + private readonly struct ReadonlyStatistics(float rawCpuUsagePercentage) + { + public readonly float RawCpuUsagePercentage = rawCpuUsagePercentage; + } +} diff --git a/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs b/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs new file mode 100644 index 00000000000..bcca9a1d1f9 --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs @@ -0,0 +1,424 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using System.Text.Json; +using Xunit; + +namespace Orleans.Dissemination.IntegrationHarness; + +internal sealed record AssemblyManifest(string AssemblyVersion, string Sha256); +internal sealed record BuildManifest( + string Runtime, + string SourceRevision, + string AssemblyName, + string AssemblyVersion, + string Sha256, + Dictionary Assemblies); + +internal sealed class ProcessCluster : IAsyncDisposable +{ + public const string Baseline = "6739589254b746a8790cf53524e6abe372bb53d4"; + private readonly List _all = []; + private readonly string _membershipDirectory; + private int _sequence; + + public ProcessCluster(string scenario, bool fastRecovery = true) + { + var artifactRoot = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RESULTS") + ?? throw new InvalidOperationException("Run .github/scripts/dissemination-evidence.ps1 first, or set ORLEANS_DISSEMINATION_{OLD,NEW,RESULTS}."); + Directory = Path.Combine(Path.GetFullPath(artifactRoot), $"{scenario}-{Guid.NewGuid():N}"); + System.IO.Directory.CreateDirectory(Directory); + _membershipDirectory = Path.Combine(Directory, "membership"); + System.IO.Directory.CreateDirectory(_membershipDirectory); + FastRecovery = fastRecovery; + } + + public string Directory { get; } + public bool FastRecovery { get; } + public IReadOnlyList All => _all; + public SiloProcess[] Active => _all.Where(node => !node.Stopped).ToArray(); + + public async Task Start(string runtime, bool enabled, int? port = null) + { + var binaryDirectory = Path.GetFullPath(Environment.GetEnvironmentVariable($"ORLEANS_DISSEMINATION_{runtime.ToUpperInvariant()}") + ?? throw new InvalidOperationException($"Missing {runtime} published binary directory.")); + var manifest = JsonSerializer.Deserialize(await File.ReadAllTextAsync(Path.Combine(binaryDirectory, "manifest.json")))!; + Assert.Equal(runtime, manifest.Runtime); + if (runtime == "Old") + { + Assert.Equal(Baseline, manifest.SourceRevision); + } + else + { + Assert.NotEqual(Baseline, manifest.SourceRevision); + } + + var name = $"{runtime}-{_sequence++}-{(enabled ? "enabled" : "default")}"; + var node = new SiloProcess( + binaryDirectory, + new(name, Path.GetFileName(Directory), _membershipDirectory, port ?? AvailablePort(), Environment.ProcessId, enabled, FastRecovery), + Directory, + manifest); + _all.Add(node); + await node.Start(); + Assert.DoesNotContain(_all.Where(other => !ReferenceEquals(other, node) && !other.Stopped), + other => other.Last.Identity.ProcessId == node.Last.Identity.ProcessId); + return node; + } + + public async Task Stabilize() + { + var expected = Active.Select(node => node.Last.Address).Order(StringComparer.Ordinal).ToArray(); + await Eventually("every process has the exact active membership", async () => + { + var snapshots = await Task.WhenAll(Active.Select(node => node.Send("refresh"))); + return snapshots.All(snapshot => snapshot.ActiveMembers.SequenceEqual(expected)); + }); + } + + public async Task AssertControlRpcs(bool allPairs = true) + { + var nodes = Active; + for (var index = 0; index < nodes.Length; index++) + { + var peers = allPairs ? nodes.Where(peer => !ReferenceEquals(peer, nodes[index])) : [nodes[(index + 1) % nodes.Length]]; + foreach (var peer in peers) + { + var result = await nodes[index].Send("echo", peer: peer.Last.Address); + Assert.Equal(peer.Last.Identity.ProcessId, result.RemoteProcessId); + Assert.NotEqual(nodes[index].Last.Identity.ProcessId, result.RemoteProcessId); + } + } + } + + public async Task PublishAndConverge() + { + var clock = Stopwatch.StartNew(); + var expected = new Dictionary(StringComparer.Ordinal); + foreach (var node in Active) + { + var previous = node.Last.LoadVersions.GetValueOrDefault(node.Last.Address); + var snapshot = await node.Send("publish"); + Assert.True(snapshot.LoadVersions[snapshot.Address] > previous, "The production publisher did not create a new sample."); + expected.Add(snapshot.Address, snapshot.Load[snapshot.Address]); + } + + await Eventually("exact production load state at every active process", async () => + { + var snapshots = await Task.WhenAll(Active.Select(node => node.Send("snapshot"))); + return snapshots.All(snapshot => expected.All(pair => + snapshot.Load.TryGetValue(pair.Key, out var actual) && StringComparer.Ordinal.Equals(pair.Value, actual))); + }, expected: expected); + return clock.Elapsed.TotalMilliseconds; + } + + public async Task HealPartition(SiloProcess partitioned) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(60)); + var cancellationToken = deadline.Token; + var phase = "retiring partitioned connections at both endpoints"; + try + { + var nodes = Active; + // Drain admitted middleware, including unfinished handshakes, before reopening any affected endpoint. + await Task.WhenAll(nodes.Select(node => node.Send( + cancellationToken, + "drain-connections", + peer: ReferenceEquals(node, partitioned) ? null : partitioned.Last.Address))); + await Task.WhenAll(nodes.Select(node => node.Send(cancellationToken, "resume-connections"))); + phase = "bidirectional control RPCs after transport healing"; + await Eventually(phase, async () => + { + foreach (var node in nodes) + { + foreach (var peer in nodes.Where(peer => !ReferenceEquals(peer, node))) + { + var result = await node.Send(cancellationToken, "probe-echo", peer: peer.Last.Address); + if (result.RemoteProcessId is not { } processId) + { + Assert.NotNull(result.ProbeError); + return false; + } + + Assert.Equal(peer.Last.Identity.ProcessId, processId); + Assert.NotEqual(node.Last.Identity.ProcessId, processId); + } + } + + return true; + }); + } + catch (OperationCanceledException) when (deadline.IsCancellationRequested && !TestContext.Current.CancellationToken.IsCancellationRequested) + { + await Save("failure-state.json", new { Phase = phase, Nodes = _all.Select(node => node.Last) }); + Assert.Fail($"Timed out during {phase}. Exact snapshots/logs: {Directory}"); + } + } + + public async Task Eventually(string phase, Func> assertion, TimeSpan? timeout = null, object? expected = null) + { + var clock = Stopwatch.StartNew(); + while (true) + { + if (await assertion()) + { + return; + } + + if (clock.Elapsed > (timeout ?? TimeSpan.FromSeconds(60))) + { + await Save("failure-state.json", new { Phase = phase, Expected = expected, Nodes = _all.Select(node => node.Last) }); + Assert.Fail($"Timed out during {phase}. Exact snapshots/logs: {Directory}"); + } + + await Task.Delay(100, TestContext.Current.CancellationToken); + } + } + + public Task Save(string filename, object value) => + File.WriteAllTextAsync(Path.Combine(Directory, filename), JsonSerializer.Serialize(value, new JsonSerializerOptions { WriteIndented = true })); + + public async ValueTask DisposeAsync() + { + List failures = []; + foreach (var node in _all.AsEnumerable().Reverse()) + { + try + { + await node.DisposeAsync(); + } + catch (Exception exception) + { + failures.Add(exception); + } + } + + await Save("final-processes.json", _all.Select(node => new { node.Configuration, node.Last, node.ShutdownMilliseconds, node.ForcedKill })); + if (failures.Count > 0) + { + throw new AggregateException("One or more silos exceeded shutdown bounds.", failures); + } + } + + private static int AvailablePort() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + try + { + return ((IPEndPoint)listener.LocalEndpoint).Port; + } + finally + { + listener.Stop(); + } + } +} + +internal sealed class SiloProcess : IAsyncDisposable +{ + private readonly Process _process; + private readonly string _configurationPath; + private readonly string _logPath; + private readonly string _errorPath; + private readonly BuildManifest _manifest; + private readonly ConcurrentDictionary> _pending = new(); + private readonly SemaphoreSlim _commandLock = new(1); + private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); + private Task _output = Task.CompletedTask; + private Task _error = Task.CompletedTask; + private int _commandId; + private bool _started; + private bool _disposed; + + public SiloProcess(string binaryDirectory, NodeConfiguration configuration, string artifacts, BuildManifest manifest) + { + Configuration = configuration; + _manifest = manifest; + _configurationPath = Path.Combine(artifacts, configuration.Name + ".configuration.json"); + _logPath = Path.Combine(artifacts, configuration.Name + ".stdout.jsonl"); + _errorPath = Path.Combine(artifacts, configuration.Name + ".stderr.log"); + _process = new Process + { + StartInfo = new ProcessStartInfo(Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet") + { + WorkingDirectory = binaryDirectory, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + ArgumentList = { Path.Combine(binaryDirectory, "Orleans.Runtime.Tests.dll"), "--silo", _configurationPath }, + }, + }; + _pending[0] = _ready; + } + + public NodeConfiguration Configuration { get; } + public NodeSnapshot Last { get; private set; } = null!; + public NodeSnapshot Initial { get; private set; } = null!; + public bool Stopped { get; private set; } + public bool ForcedKill { get; private set; } + public double ShutdownMilliseconds { get; private set; } + + public async Task Start() + { + await File.WriteAllTextAsync(_configurationPath, JsonSerializer.Serialize(Configuration)); + Assert.True(_process.Start(), "Failed to launch silo OS process."); + _started = true; + _output = PumpOutput(); + _error = PumpErrors(); + var ready = await _ready.Task.WaitAsync(TimeSpan.FromSeconds(100), TestContext.Current.CancellationToken); + Assert.Null(ready.Error); + Last = ready.Snapshot!; + Initial = Last; + Assert.Equal(_process.Id, Last.Identity.ProcessId); + Assert.Equal(_manifest.Runtime, Last.Identity.Runtime); + Assert.Equal(_manifest.SourceRevision, Last.Identity.SourceRevision); + Assert.Equal(_manifest.AssemblyName, Last.Identity.AssemblyName); + Assert.Equal(_manifest.AssemblyVersion, Last.Identity.AssemblyVersion); + Assert.Equal(_manifest.Sha256, Last.Identity.Sha256, ignoreCase: true); + Assert.Contains(_manifest.SourceRevision, Last.Identity.InformationalVersion, StringComparison.OrdinalIgnoreCase); + Assert.Equal(_manifest.Runtime == "New", Last.Identity.HasDissemination); + Assert.Equal(Configuration.Enabled, Last.Enabled); + Assert.Equal(Configuration.Enabled, Last.NamespaceEnabled); + var actualDirectory = Path.GetDirectoryName(Path.GetFullPath(Last.Identity.AssemblyPath)); + Assert.Equal(Path.GetFullPath(_process.StartInfo.WorkingDirectory), actualDirectory); + foreach (var name in new[] { "Orleans.Runtime", "Orleans.Core", "Orleans.Core.Abstractions", "Orleans.Serialization" }) + { + Assert.Contains(name, Last.Identity.Assemblies.Keys); + } + + foreach (var (name, proof) in Last.Identity.Assemblies) + { + Assert.True(_manifest.Assemblies.TryGetValue(name, out var expected), $"Loaded unexpected assembly {name}."); + Assert.Equal(expected!.AssemblyVersion, proof.Version); + Assert.Equal(expected.Sha256, proof.Sha256, ignoreCase: true); + Assert.Equal(Path.GetFullPath(_process.StartInfo.WorkingDirectory), Path.GetDirectoryName(Path.GetFullPath(proof.Path))); + } + } + + public Task Send(string operation, string? peer = null, int count = 1, bool value = false, long version = 0) => + Send(TestContext.Current.CancellationToken, operation, peer, count, value, version); + + public async Task Send(CancellationToken cancellationToken, string operation, string? peer = null, int count = 1, bool value = false, long version = 0) + { + await _commandLock.WaitAsync(cancellationToken); + try + { + var command = new Command(Interlocked.Increment(ref _commandId), operation, peer, count, value, version); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _pending[command.Id] = completed; + await _process.StandardInput.WriteLineAsync(JsonSerializer.Serialize(command).AsMemory(), cancellationToken); + await _process.StandardInput.FlushAsync(cancellationToken); + var response = await completed.Task.WaitAsync(TimeSpan.FromSeconds(70), cancellationToken); + Assert.True(response.Error is null, $"{Configuration.Name}/{operation}: {response.Error}; {_errorPath}"); + Last = response.Snapshot!; + return Last; + } + finally + { + _commandLock.Release(); + } + } + + public async Task Stop() + { + if (Stopped || !_started) + { + return; + } + + var clock = Stopwatch.StartNew(); + try + { + if (!_process.HasExited) + { + // Cleanup has its own bound, independent of a cancelled xUnit test token. + await _process.StandardInput.WriteLineAsync(JsonSerializer.Serialize(new Command(-2, "stop"))); + await _process.StandardInput.FlushAsync(); + await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(35)); + } + + await Task.WhenAll(_output, _error).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, _process.ExitCode); + } + catch + { + if (!_process.HasExited) + { + ForcedKill = true; + _process.Kill(entireProcessTree: true); + await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10)); + } + + throw; + } + finally + { + ShutdownMilliseconds = clock.Elapsed.TotalMilliseconds; + Stopped = true; + } + } + + private async Task PumpOutput() + { + await using var log = File.CreateText(_logPath); + while (await _process.StandardOutput.ReadLineAsync() is { } line) + { + await log.WriteLineAsync(line); + await log.FlushAsync(); + if (!line.StartsWith("HARNESS ", StringComparison.Ordinal)) + { + continue; + } + + var response = JsonSerializer.Deserialize(line["HARNESS ".Length..])!; + if (response.Snapshot is not null) + { + Last = response.Snapshot; + } + + if (_pending.TryRemove(response.Id, out var completed)) + { + completed.TrySetResult(response); + } + } + + foreach (var completion in _pending.Values) + { + completion.TrySetException(new InvalidOperationException($"Silo stdout closed; inspect {_errorPath}.")); + } + } + + private async Task PumpErrors() + { + await using var log = File.CreateText(_errorPath); + while (await _process.StandardError.ReadLineAsync() is { } line) + { + await log.WriteLineAsync(line); + await log.FlushAsync(); + } + } + + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + try + { + await Stop(); + } + finally + { + _process.Dispose(); + _commandLock.Dispose(); + } + } +} diff --git a/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs b/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs new file mode 100644 index 00000000000..d31b5b18644 --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs @@ -0,0 +1,312 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Text.Json; +using Xunit; + +namespace Orleans.Dissemination.IntegrationHarness; + +public sealed class ScalingTests +{ + [Fact] + [Trait("Category", "DisseminationScale")] + public async Task Scaling_OriginalLegacy_DefaultOff_EnabledSupported() + { + var sizes = (Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_SIZES") ?? "4,8,16") + .Split(',').Select(value => int.Parse(value, System.Globalization.CultureInfo.InvariantCulture)).ToArray(); + var iterations = Setting("ITERATIONS", 10, 3, 200); + var repetitions = Setting("REPETITIONS", 1, 1, 5); + Assert.All(sizes, size => Assert.InRange(size, 3, 32)); + Assert.InRange(sizes.Length, 1, 6); + + var records = new List(); + var output = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RESULTS")!; + RuntimeVariant[] variants = + [ + new("OriginalLegacy", "Old", Enabled: false), + new("CurrentDefaultOff", "New", Enabled: false), + new("CurrentEnabledSupported", "New", Enabled: true), + ]; + foreach (var size in sizes) + { + foreach (var scenario in new[] { "stable", "churn", "partition" }) + { + for (var repetition = 0; repetition < repetitions; repetition++) + { + // Rotate all three paths to expose thermal/cache/order effects across repetitions. + for (var order = 0; order < variants.Length; order++) + { + var variant = variants[(repetition + order) % variants.Length]; + var record = await Measure(size, scenario, variant, iterations, repetition, order); + records.Add(record); + await File.WriteAllTextAsync(Path.Combine(output, "cost-results.json"), + JsonSerializer.Serialize(records, new JsonSerializerOptions { WriteIndented = true }), + TestContext.Current.CancellationToken); + } + } + } + } + } + + private static async Task Measure(int size, string scenario, RuntimeVariant variant, int iterations, int repetition, int order) + { + var enabled = variant.Enabled; + await using var cluster = new ProcessCluster($"scale-{size}-{scenario}-{variant.Name}-{repetition}", fastRecovery: false); + for (var index = 0; index < size; index++) + { + await cluster.Start(variant.Runtime, enabled); + } + + await cluster.Stabilize(); + await cluster.AssertControlRpcs(allPairs: false); + await cluster.PublishAndConverge(); + await cluster.PublishAndConverge(); + if (enabled) + { + await ConfirmSupport(cluster); + } + + await cluster.Save("warmup.json", await Task.WhenAll(cluster.Active.Select(node => node.Send("measure")))); + // Socket EventCounters arrive in one-second buckets. Every path has the same two-second margins. + await Task.Delay(TimeSpan.FromSeconds(2), TestContext.Current.CancellationToken); + var start = (await Task.WhenAll(cluster.Active.Select(node => node.Send("measure")))) + .ToDictionary(snapshot => snapshot.Identity.ProcessId); + var startedAt = DateTimeOffset.UtcNow; + var clock = Stopwatch.StartNew(); + var latencies = new List(); + var recoveryLatencies = new List(); + var offeredPublications = 0; + var restarts = 0; + var partitions = 0; + for (var iteration = 0; iteration < iterations; iteration++) + { + if (scenario == "churn" && iteration > 0 && iteration % 2 == 0) + { + var victim = cluster.Active[iteration % size]; + await victim.Stop(); + // Reuse the endpoint to exercise a real new silo incarnation and stale-generation cleanup. + await cluster.Start(variant.Runtime, enabled, victim.Configuration.SiloPort); + await cluster.Stabilize(); + if (enabled) + { + // Bootstrap/fallback work stays inside the churn window, but offered rounds use supported peers. + await ConfirmSupport(cluster); + } + + restarts++; + } + else if (scenario == "partition" && iteration % 3 == 1) + { + var victim = cluster.Active[iteration % size]; + await victim.Send("partition", value: true); + partitions++; + foreach (var source in cluster.Active.Where(node => !ReferenceEquals(node, victim))) + { + await source.Send("publish"); + offeredPublications++; + } + + var recovery = Stopwatch.StartNew(); + // Establish delivery paths before offering the next sample, including on the one-way legacy path. + await cluster.HealPartition(victim); + latencies.Add(await cluster.PublishAndConverge()); + offeredPublications += size; + recoveryLatencies.Add(recovery.Elapsed.TotalMilliseconds); + continue; + } + + latencies.Add(await cluster.PublishAndConverge()); + offeredPublications += size; + } + + await Task.Delay(TimeSpan.FromSeconds(2), TestContext.Current.CancellationToken); + await Task.WhenAll(cluster.Active.Select(node => node.Send("measure"))); + var elapsed = clock.Elapsed.TotalMilliseconds; + var finishedAt = DateTimeOffset.UtcNow; + var finish = cluster.All.Select(node => node.Last).ToArray(); + var retained = await Task.WhenAll(cluster.Active.Select(node => node.Send("retained-memory"))); + var retainedByPid = retained.ToDictionary(snapshot => snapshot.Identity.ProcessId, snapshot => snapshot.ManagedHeapBytes); + Assert.Equal(scenario == "churn" ? (iterations - 1) / 2 : 0, restarts); + Assert.Equal(scenario == "partition" ? (iterations + 1) / 3 : 0, partitions); + if (enabled) + { + Assert.All(start.Values, snapshot => Assert.Empty(snapshot.UnconfirmedPeers)); + Assert.All(cluster.Active, node => Assert.Empty(node.Last.UnconfirmedPeers)); + } + + var nodes = finish.Select(snapshot => + { + start.TryGetValue(snapshot.Identity.ProcessId, out var baseline); + var allMessages = Delta(snapshot, baseline, "orleans-messaging-sent-messages-size|"); + var requestMessages = Delta(snapshot, baseline, "orleans-messaging-sent-messages-size|", "MessageDirection=Request"); + var oneWayMessages = Delta(snapshot, baseline, "orleans-messaging-sent-messages-size|", "MessageDirection=OneWay"); + var received = Delta(snapshot, baseline, "orleans-messaging-received-messages-size|"); + var topology = snapshot.TopologyMembers.Contains(snapshot.Address) + ? snapshot + : baseline ?? cluster.All.Single(node => node.Last.Identity.ProcessId == snapshot.Identity.ProcessId).Initial; + var index = Array.IndexOf(topology.TopologyMembers, snapshot.Address); + return new NodeCost( + snapshot.Identity.ProcessId, + snapshot.Address, + index >= 0 ? index : null, + index >= 0 && index < topology.Fanout, + topology.OriginatorTargets, + topology.ForwardingTargets, + requestMessages.Count + oneWayMessages.Count, + allMessages.Count, + allMessages.Sum, + received.Count, + received.Sum, + snapshot.TransportBytesWritten - (baseline?.TransportBytesWritten ?? 0), + snapshot.TransportBytesRead - (baseline?.TransportBytesRead ?? 0), + snapshot.SocketBytesSent - (baseline?.SocketBytesSent ?? 0), + snapshot.SocketBytesReceived - (baseline?.SocketBytesReceived ?? 0), + snapshot.SocketCounterSamples - (baseline?.SocketCounterSamples ?? 0), + snapshot.CpuMilliseconds - (baseline?.CpuMilliseconds ?? 0), + snapshot.AllocatedBytes - (baseline?.AllocatedBytes ?? 0), + retainedByPid.GetValueOrDefault(snapshot.Identity.ProcessId), + snapshot.WorkingSetBytes, + snapshot.PrivateBytes, + Delta(snapshot, baseline, "orleans-dissemination-bytes-sent|").Sum, + Delta(snapshot, baseline, "orleans-dissemination-broadcast-sent|").Sum, + Delta(snapshot, baseline, "orleans-dissemination-anti-entropy-exchanges|", "direction=out").Sum); + }).ToArray(); + Assert.All(nodes, node => + { + Assert.True(node.SentMessages > 0, "Orleans message histogram evidence is missing."); + Assert.True(node.SerializedBytesSent > 0, "Serialized byte evidence is missing."); + Assert.True(node.SocketSamples > 0 && node.SocketBytesSent > 0, "OS socket byte evidence is missing."); + }); + if (!enabled) + { + Assert.All(nodes, node => + { + Assert.Equal(0, node.DisseminationBroadcasts); + Assert.Equal(0, node.AntiEntropyExchanges); + }); + } + else + { + Assert.True(nodes.Sum(node => node.DisseminationBroadcasts) > 0, + "The supported-enabled cost window did not exercise any dissemination broadcasts."); + } + + var record = new + { + Schema = 1, + Profile = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_PROFILE") ?? "Standalone", + RuntimePath = variant.Name, + Runtime = finish[0].Identity, + Size = size, + Scenario = scenario, + Enabled = enabled, + Iterations = iterations, + Repetition = repetition, + ExecutionOrder = order, + OfferedPublications = offeredPublications, + RestartedProcesses = restarts, + InjectedPartitions = partitions, + WindowMilliseconds = elapsed, + WindowStartedAtUtc = startedAt, + WindowFinishedAtUtc = finishedAt, + ConvergenceMilliseconds = latencies, + PartitionRecoveryMilliseconds = recoveryLatencies, + TotalRpcs = nodes.Sum(node => node.Rpcs), + TotalMessages = nodes.Sum(node => node.SentMessages), + SerializedBytesSent = nodes.Sum(node => node.SerializedBytesSent), + SocketBytesSent = nodes.Sum(node => node.SocketBytesSent), + TransportBytesSubmitted = nodes.Sum(node => node.TransportBytesSubmitted), + CpuMilliseconds = nodes.Sum(node => node.CpuMilliseconds), + AllocatedBytes = nodes.Sum(node => node.AllocatedBytes), + RetainedManagedBytes = nodes.Sum(node => node.RetainedManagedBytes), + Nodes = nodes, + Environment = new { RuntimeInformation.OSDescription, RuntimeInformation.FrameworkDescription, System.Environment.ProcessorCount }, + Methodology = new + { + WarmupRounds = 2, + SettlingMarginSeconds = 2, + AutomaticLoadPublisherPaused = true, + ProductionDisseminationDefaults = variant.Runtime == "New", + LoadNamespaceSupportConfirmedBeforeOfferedRounds = enabled, + IncludesUnconfirmedBootstrapTraffic = enabled && scenario == "churn", + LivenessFailureDetectionEnabled = false, + StorageProvider = "locked local JSON file; no external-service network", + Window = "after warmup/capability discovery, through offered load, churn/restarts and final 2-second drain; GC retained-size probe is outside CPU/allocation window", + RpcDefinition = "sent Orleans message-size observations tagged MessageDirection=Request OR MessageDirection=OneWay; not response messages", + SerializedBytes = "complete serialized Orleans messages (headers and bodies), not dissemination payload counters", + SocketBytes = "System.Net.Sockets bytes-sent/bytes-received EventCounters, 1-second sampling; cumulative values differenced across the window (incremental counters also supported); excludes TCP/IP headers and retransmits", + TransportBytes = "bytes advanced at the real connection transport pipe, including handshake; may include buffered bytes lost when a partition aborts a connection", + RetainedMemory = "full GC live managed bytes on surviving nodes only; not a peak; RSS/private bytes and retired-node snapshots also retained", + Includes = "all silo work during the window, including control/snapshot/instrumentation overhead and legacy fallback; restart startup/shutdown costs included for churn", + Observation = "state polling on all three paths; detailed counters only at window boundaries; per-value DiagnosticListener capture disabled in scale runs", + Topology = "per-node actual tree at end, or start for a retired node; raw command snapshots preserve intervening topology changes; role is not time-weighted", + Excludes = "controller process CPU/allocation, storage disk byte accounting, cross-machine latency, NIC/TCP wire overhead, production storage/placement/grain work", + PartitionComparison = "retire partitioned connections at both endpoints, then confirm healed connections with bidirectional acknowledged control RPCs before the same single publication round on all paths; teardown and readiness remain inside partition recovery time and total cost; silent anti-entropy recovery is asserted separately in VersionSkewTests", + OriginalBaselineAttribution = "original-to-current default-off deltas include all intervening source changes, including lifecycle prerequisites and load notification/tombstone/dedup changes; only current off/on isolates enabling the subsystem on the same binary", + Interpretation = "separate original-legacy, current-default-off and current-enabled-supported measurements; no asserted improvement or threshold", + }, + }; + await cluster.Save("window-start.json", start); + await cluster.Save("window-finish.json", finish); + await cluster.Save("cost.json", record); + return record; + } + + private static Task ConfirmSupport(ProcessCluster cluster) => + cluster.Eventually("all enabled peers confirm support before offered publication rounds", async () => + { + var snapshots = await Task.WhenAll(cluster.Active.Select(node => node.Send("snapshot"))); + return snapshots.All(snapshot => snapshot.UnconfirmedPeers.Length == 0); + }, TimeSpan.FromSeconds(90)); + + private sealed record RuntimeVariant(string Name, string Runtime, bool Enabled); + + private static MetricValue Delta(NodeSnapshot value, NodeSnapshot? baseline, string prefix, string? tag = null) + { + var entries = value.Metrics.Where(pair => pair.Key.StartsWith(prefix, StringComparison.Ordinal) + && (tag is null || pair.Key.Contains(tag, StringComparison.Ordinal))); + long count = 0; + double sum = 0; + foreach (var (key, metric) in entries) + { + var previous = baseline?.Metrics.GetValueOrDefault(key); + count += metric.Count - (previous?.Count ?? 0); + sum += metric.Sum - (previous?.Sum ?? 0); + } + + return new(count, sum); + } + + private static int Setting(string name, int defaultValue, int minimum, int maximum) + { + var value = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_" + name); + var result = value is null ? defaultValue : int.Parse(value, System.Globalization.CultureInfo.InvariantCulture); + Assert.InRange(result, minimum, maximum); + return result; + } + + private sealed record NodeCost( + int ProcessId, + string Address, + int? TreeRank, + bool UpperTree, + string[] OriginatorTargets, + string[] ForwardingTargets, + long Rpcs, + long SentMessages, + double SerializedBytesSent, + long ReceivedMessages, + double SerializedBytesReceived, + long TransportBytesSubmitted, + long TransportBytesRead, + double SocketBytesSent, + double SocketBytesReceived, + long SocketSamples, + double CpuMilliseconds, + long AllocatedBytes, + long RetainedManagedBytes, + long WorkingSetBytes, + long PrivateBytes, + double DisseminationPayloadBytes, + double DisseminationBroadcasts, + double AntiEntropyExchanges); +} diff --git a/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs new file mode 100644 index 00000000000..d28f4d348cd --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs @@ -0,0 +1,341 @@ +using System.Text.Json; +using Xunit; + +[assembly: CollectionBehavior(DisableTestParallelization = true)] + +namespace Orleans.Dissemination.IntegrationHarness; + +public sealed class VersionSkewTests +{ + [Theory] + [InlineData("Old", false)] + [InlineData("New", false)] + [InlineData("New", true)] + [Trait("Category", "DisseminationProcess")] + public async Task PartitionHealing_ConfirmsTransportBeforeSinglePublicationRound(string runtime, bool enabled) + { + await using var cluster = new ProcessCluster($"partition-publication-{runtime}-{enabled}", fastRecovery: false); + var sender = await cluster.Start(runtime, enabled); + var receiver = await cluster.Start(runtime, enabled); + await cluster.Stabilize(); + await cluster.AssertControlRpcs(); + await cluster.PublishAndConverge(); + + for (var iteration = 0; iteration < 3; iteration++) + { + await receiver.Send("partition", value: true); + if (iteration == 0) + { + var probe = await sender.Send("probe-echo", peer: receiver.Last.Address); + Assert.Null(probe.RemoteProcessId); + Assert.NotNull(probe.ProbeError); + } + + await sender.Send("publish"); + await cluster.HealPartition(receiver); + await cluster.PublishAndConverge(); + Assert.All(cluster.Active, node => Assert.False(node.Last.Partitioned)); + await cluster.Save($"healed-publication-{iteration}.json", cluster.Active.Select(node => node.Last)); + } + } + + [Fact] + [Trait("Category", "DisseminationProcess")] + public async Task PinnedBinaries_RollForward_MixedEnablement_AndRollback() + { + await using var cluster = new ProcessCluster("rolling-upgrade"); + var old = new[] + { + await cluster.Start("Old", enabled: false), + await cluster.Start("Old", enabled: false), + await cluster.Start("Old", enabled: false), + }; + await cluster.Stabilize(); + await cluster.AssertControlRpcs(); + await cluster.PublishAndConverge(); + + var replacements = new List(); + for (var index = 0; index < old.Length; index++) + { + var replacement = await cluster.Start("New", enabled: index != 0); + replacements.Add(replacement); + Assert.NotEqual(old[index].Last.Identity.Sha256, replacement.Last.Identity.Sha256); + Assert.NotEqual(old[index].Last.Identity.ModuleVersionId, replacement.Last.Identity.ModuleVersionId); + await cluster.Stabilize(); + await cluster.AssertControlRpcs(); + await cluster.PublishAndConverge(); + if (index != 0) + { + // A working stable RPC on this exact connection brackets the unsupported new RPC. + var probe = await replacement.Send("probe", peer: old[index].Last.Address); + Assert.NotNull(probe.ProbeError); + Assert.False(probe.ProbeTimedOut, $"Unknown-RPC compatibility must not be inferred from a timeout: {probe.ProbeError}"); + Assert.Equal(old[index].Last.Identity.ProcessId, + (await replacement.Send("echo", peer: old[index].Last.Address)).RemoteProcessId); + Assert.Contains(old[index].Last.Address, replacement.Last.UnconfirmedPeers); + var beforeFallback = old[index].Last.LoadVersions[replacement.Last.Address]; + await cluster.PublishAndConverge(); + Assert.True(old[index].Last.LoadVersions[replacement.Last.Address] > beforeFallback, + "The old process must receive a fresh exact load sample after the unsupported dissemination RPC."); + Assert.Contains(old[index].Last.Address, replacement.Last.UnconfirmedPeers); + Assert.Empty(old[index].Last.Applies); + await cluster.Save($"old-rpc-probe-{index}.json", probe); + await cluster.Save($"old-rpc-legacy-fallback-{index}.json", new + { + BeforeVersion = beforeFallback, + Sender = replacement.Last, + LegacyReceiver = old[index].Last, + }); + } + + await cluster.Save($"upgrade-{index}-before-stop.json", cluster.Active.Select(node => node.Last)); + await old[index].Stop(); + await cluster.Stabilize(); + await cluster.PublishAndConverge(); + } + + var disabled = replacements[0]; + var enabled = replacements.Skip(1).ToArray(); + await cluster.Eventually("enabled peers confirm support, default-disabled peers remain legacy", async () => + { + foreach (var node in enabled) + { + await node.Send("snapshot"); + } + + return enabled.All(node => + node.Last.UnconfirmedPeers.Contains(disabled.Last.Address) + && enabled.Where(peer => !ReferenceEquals(peer, node)).All(peer => + !node.Last.UnconfirmedPeers.Contains(peer.Last.Address))); + }); + var disabledProbe = await enabled[0].Send("probe", peer: disabled.Last.Address); + Assert.Equal("namespace-unsupported", disabledProbe.ProbeError); + await disabled.Send("snapshot"); + Assert.False(disabled.Last.Enabled); + Assert.False(disabled.Last.NamespaceEnabled); + Assert.Equal(0, Sum(disabled.Last, "orleans-dissemination-broadcast-sent|")); + Assert.Equal(0, disabled.Last.Metrics.Where(pair => + pair.Key.StartsWith("orleans-dissemination-anti-entropy-exchanges|", StringComparison.Ordinal) + && pair.Key.Contains("direction=out", StringComparison.Ordinal)).Sum(pair => pair.Value.Sum)); + Assert.Empty(disabled.Last.Applies); + + // The same original binary is reintroduced, not a flag on the new binary. + for (var index = replacements.Count - 1; index >= 0; index--) + { + var rolledBack = await cluster.Start("Old", enabled: false, port: old[index].Configuration.SiloPort); + Assert.Equal(old[index].Last.Identity.Sha256, rolledBack.Last.Identity.Sha256); + Assert.NotEqual(replacements[index].Last.Identity.ProcessId, rolledBack.Last.Identity.ProcessId); + await cluster.Stabilize(); + await cluster.AssertControlRpcs(); + await cluster.PublishAndConverge(); + await replacements[index].Stop(); + await cluster.Stabilize(); + await cluster.PublishAndConverge(); + await cluster.Save($"rollback-{index}.json", cluster.Active.Select(node => node.Last)); + } + + Assert.All(cluster.Active, node => Assert.Equal("Old", node.Last.Identity.Runtime)); + } + + [Fact] + [Trait("Category", "DisseminationProcess")] + public async Task PartitionHistoryAndHeartbeat_RequireDisseminationProvenance() + { + await using var cluster = new ProcessCluster("partition-history-heartbeat"); + var origin = await cluster.Start("New", enabled: true); + var relay = await cluster.Start("New", enabled: true); + var receiver = await cluster.Start("New", enabled: true); + await cluster.Stabilize(); + await cluster.AssertControlRpcs(); + await cluster.PublishAndConverge(); + await cluster.Eventually("identical membership before isolation", async () => + { + var snapshots = await Task.WhenAll(cluster.Active.Select(node => node.Send("refresh"))); + return snapshots.All(snapshot => SameMembership(snapshots[0], snapshot)); + }); + foreach (var node in cluster.Active) + { + var isolated = await node.Send("isolate-membership", value: true); + Assert.True(isolated.LegacyGossipSuppressed); + Assert.True(isolated.MembershipReadsFrozen); + } + + Assert.False((await origin.Send("probe-tree", peer: receiver.Last.Address)).TreeProbeRejected); + foreach (var node in cluster.Active) + { + var quiesced = await node.Send("block-tree", value: true); + Assert.NotNull(quiesced.TreeGate); + Assert.True(quiesced.TreeGate.Blocked); + Assert.Equal(0, quiesced.TreeGate.InFlight); + } + + // Verify the native PushBroadcast RPC is intercepted, not just that a local flag was set. + Assert.True((await origin.Send("probe-tree", peer: receiver.Last.Address)).TreeProbeRejected); + await Task.WhenAll(cluster.Active.Select(node => node.Send("snapshot"))); + var before = cluster.Active.ToDictionary(node => node.Last.Identity.ProcessId, node => node.Last); + var oldVersion = receiver.Last.MembershipVersion; + await receiver.Send("partition", value: true); + var source = await origin.Send("membership-history", count: 40, version: oldVersion); + Assert.Equal(oldVersion + 40, source.MembershipVersion); + Assert.Equal(0, source.RepairFromVersion); // The actual production namespace evicted the old baseline. + var blocked = await receiver.Send("snapshot"); + Assert.Equal(oldVersion, blocked.MembershipVersion); + Assert.False(SameMembership(source, blocked)); + await receiver.Send("partition", value: false); + + await cluster.Eventually("anti-entropy-only full repair after a real transport partition and retained-history miss", async () => + { + await receiver.Send("snapshot"); + await relay.Send("snapshot"); + return SameMembership(source, receiver.Last) && SameMembership(source, relay.Last) + && receiver.Last.Applies.Any(evidence => evidence.Namespace == "membership" + && evidence.FromVersion == 0 && evidence.ToVersion == source.MembershipVersion + && evidence.Result == "Applied" && evidence.Peer is not null); + }); + await AssertNoTreeAdmissions(cluster, before); + Assert.True(receiver.Last.MembershipReadsFrozen); + Assert.True(receiver.Last.LegacyGossipSuppressed); + await cluster.Save("history-recovery.json", new { Before = before, After = cluster.Active.Select(node => node.Last) }); + + var priorHeartbeatApplications = receiver.Last.Applies.Count(evidence => + evidence.Namespace == "membership" && evidence.Result == "Applied" && evidence.ToVersion == source.MembershipVersion); + var heartbeat = await origin.Send("membership-heartbeat"); + Assert.Equal(source.MembershipVersion, heartbeat.MembershipVersion); + Assert.NotNull(heartbeat.HeartbeatTicks); + Assert.False(SameMembership(source, heartbeat)); + await cluster.Eventually("same-version heartbeat fingerprint repair without Publish, gossip, or table reads", async () => + { + await receiver.Send("snapshot"); + await relay.Send("snapshot"); + return SameMembership(heartbeat, receiver.Last) && SameMembership(heartbeat, relay.Last) + && receiver.Last.Applies.Count(evidence => evidence.Namespace == "membership" + && evidence.FromVersion == 0 && evidence.ToVersion == heartbeat.MembershipVersion + && evidence.Result == "Applied") > priorHeartbeatApplications; + }); + await AssertNoTreeAdmissions(cluster, before); + await cluster.Save("same-version-heartbeat-recovery.json", cluster.Active.Select(node => node.Last)); + + var expectedLoad = receiver.Last.Load[origin.Last.Address]; + var expectedVersion = receiver.Last.LoadVersions[origin.Last.Address]; + var priorLoadApplications = receiver.Last.Applies.Count(evidence => + evidence.Namespace == "load" && evidence.Result == "Applied" && evidence.ToVersion == expectedVersion); + await receiver.Send("forget-load", peer: origin.Last.Address); + Assert.DoesNotContain(origin.Last.Address, receiver.Last.Load.Keys); + await cluster.Eventually("load repair reaches exact state with no new publication or direct refresh", async () => + { + await receiver.Send("snapshot"); + return receiver.Last.Load.TryGetValue(origin.Last.Address, out var actual) + && actual == expectedLoad + && receiver.Last.Applies.Count(evidence => evidence.Namespace == "load" + && evidence.FromVersion == 0 && evidence.ToVersion == expectedVersion + && evidence.Result == "Applied") > priorLoadApplications; + }); + await AssertNoTreeAdmissions(cluster, before); + await cluster.Save("load-full-repair.json", receiver.Last); + foreach (var node in cluster.Active) + { + await node.Send("isolate-membership", value: false); + await node.Send("block-tree", value: false); + } + } + + [Fact] + [Trait("Category", "DisseminationProcess")] + public async Task RuntimeStatisticsComparison_ObservesPublicReadonlyFieldMutation() + { + foreach (var runtime in new[] { "Old", "New" }) + { + await using var cluster = new ProcessCluster($"statistics-projection-{runtime}"); + var node = await cluster.Start(runtime, enabled: false); + var result = await node.Send("verify-load-comparison"); + var probe = Assert.IsType(result.ComparisonProbe); + Assert.NotEqual(probe.Before, probe.After); + using var original = JsonDocument.Parse(probe.Before); + using var changed = JsonDocument.Parse(probe.After); + var originalEnvironment = original.RootElement.GetProperty("EnvironmentStatistics"); + var changedEnvironment = changed.RootElement.GetProperty("EnvironmentStatistics"); + Assert.Contains("RawCpuUsagePercentage", probe.PublicFields); + Assert.All(probe.PublicFields, field => + { + Assert.True(originalEnvironment.TryGetProperty(field, out _), $"Missing original field {field}"); + Assert.True(changedEnvironment.TryGetProperty(field, out _), $"Missing changed field {field}"); + }); + Assert.Equal(20, originalEnvironment.GetProperty("RawCpuUsagePercentage").GetSingle()); + Assert.Equal(35, changedEnvironment.GetProperty("RawCpuUsagePercentage").GetSingle()); + await cluster.Save("statistics-field-mutation.json", probe); + } + } + + [Fact] + [Trait("Category", "DisseminationProcess")] + public async Task RequiredCancellationAndPartitionedShutdown_AreBounded() + { + await using var cluster = new ProcessCluster("bounded-shutdown"); + var sender = await cluster.Start("New", enabled: true); + var receiver = await cluster.Start("New", enabled: true); + await cluster.Stabilize(); + await cluster.AssertControlRpcs(); + var previousStarted = receiver.Last.StartedControlCalls; + var previousCancelled = receiver.Last.CancelledControlCalls; + var previousSignals = receiver.Last.ControlCancellationSignals; + await sender.Send("start-cancel-rpc", peer: receiver.Last.Address); + await cluster.Eventually("the remote Hold has entered with a live token before cancellation is issued", async () => + { + await receiver.Send("snapshot"); + return receiver.Last.PendingControlCalls == 1 + && receiver.Last.StartedControlCalls == previousStarted + 1 + && receiver.Last.CancelledControlCalls == previousCancelled + && receiver.Last.ControlCancellationSignals == previousSignals + && receiver.Last.ControlTokenCanBeCanceled + && !receiver.Last.ControlTokenCancelledOnEntry + && receiver.Last.ControlStartedAtUtc.HasValue; + }); + await sender.Send("snapshot"); + Assert.NotNull(sender.Last.OutboundControlCall); + Assert.False(sender.Last.OutboundControlCall.CancellationRequested); + Assert.False(sender.Last.OutboundControlCall.RawTaskCompleted); + await cluster.Save("cancellation-started.json", new { Sender = sender.Last, Receiver = receiver.Last }); + await sender.Send("cancel-started-rpc"); + Assert.True(sender.Last.OutboundControlCall!.CancellationRequested); + Assert.True(sender.Last.OutboundControlCall.RawTaskCompleted); + await cluster.Eventually("the remote cancellable test RPC releases its server-side request", async () => + { + await receiver.Send("snapshot"); + return receiver.Last.PendingControlCalls == 0 + && receiver.Last.StartedControlCalls == previousStarted + 1 + && receiver.Last.CancelledControlCalls == previousCancelled + 1 + && receiver.Last.ControlCancellationSignals == previousSignals + 1 + && receiver.Last.ControlCancellationObservedAtUtc.HasValue; + }); + await cluster.Save("cancellation-observed.json", new { Sender = sender.Last, Receiver = receiver.Last }); + await receiver.Send("partition", value: true); + await sender.Send("publish", count: 4); + await sender.Stop(); + Assert.False(sender.ForcedKill); + Assert.InRange(sender.ShutdownMilliseconds, 0, 35_000); + await receiver.Send("partition", value: false); + await receiver.Stop(); + Assert.False(receiver.ForcedKill); + Assert.InRange(receiver.ShutdownMilliseconds, 0, 35_000); + } + + internal static bool SameMembership(NodeSnapshot left, NodeSnapshot right) => + left.MembershipVersion == right.MembershipVersion && left.Membership.SequenceEqual(right.Membership); + + private static async Task AssertNoTreeAdmissions(ProcessCluster cluster, Dictionary before) + { + foreach (var node in cluster.Active) + { + var snapshot = await node.Send("snapshot"); + Assert.NotNull(snapshot.TreeGate); + Assert.True(snapshot.TreeGate.Blocked); + Assert.Equal(0, snapshot.TreeGate.InFlight); + Assert.Equal(before[snapshot.Identity.ProcessId].TreeGate!.Admitted, snapshot.TreeGate.Admitted); + Assert.True(snapshot.LegacyGossipSuppressed); + Assert.True(snapshot.MembershipReadsFrozen); + } + } + + internal static double Sum(NodeSnapshot snapshot, string prefix) => + snapshot.Metrics.Where(pair => pair.Key.StartsWith(prefix, StringComparison.Ordinal)).Sum(pair => pair.Value.Sum); +} diff --git a/test/Dissemination.IntegrationHarness/methodology.json b/test/Dissemination.IntegrationHarness/methodology.json new file mode 100644 index 00000000000..66f45aa52f7 --- /dev/null +++ b/test/Dissemination.IntegrationHarness/methodology.json @@ -0,0 +1,164 @@ +{ + "schema": 1, + "purpose": "Executable real-binary compatibility/recovery gates and controlled load evidence, not a microbenchmark or a performance claim.", + "entrypoint": ".github/scripts/dissemination-evidence.ps1", + "workflow": ".github/workflows/dissemination-evidence.yml", + "automaticPrTrigger": "pull_request and merge_group run both the pinned-binary version-skew gate and measured bounded original-legacy, current-default-off and current-enabled-supported costs. The bounded job reuses published binaries, needs no label or workflow_dispatch registration, and has a 25-minute hard limit.", + "controlledScaleTrigger": "Larger runs require workflow_dispatch with scale=true after the workflow is registered on the default branch. No pull_request_target or write permission.", + "sliceInventory": [ + ".github/scripts/dissemination-evidence.ps1", + ".github/workflows/dissemination-evidence.yml", + "test/Dissemination.IntegrationHarness/methodology.json", + "test/Dissemination.IntegrationHarness/Shared/Protocol.cs", + "test/Dissemination.IntegrationHarness/Silo/ControlTarget.cs", + "test/Dissemination.IntegrationHarness/Silo/Dissemination.Silo.csproj", + "test/Dissemination.IntegrationHarness/Silo/FileMembershipTable.cs", + "test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs", + "test/Dissemination.IntegrationHarness/Silo/Observation.cs", + "test/Dissemination.IntegrationHarness/Silo/Program.cs", + "test/Dissemination.IntegrationHarness/Tests/Dissemination.IntegrationHarness.Tests.csproj", + "test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs", + "test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs", + "test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs", + "test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs" + ], + "runtimeDependencies": { + "build": "Existing Orleans.TestingHost project, repository code generation, net10.0 SDK, existing MTP/xUnit packages; controller uses the installed ASP.NET Core shared framework for instrumentation tests.", + "bothBinaries": "TestClusterHostFactory; IMembershipTable; IMembershipManager.CurrentSnapshot/Refresh; DeploymentLoadPublisher.PeriodicStatistics; IInternalGrainFactory/SystemTarget/SystemTargetShared; stable test-only aliased control RPCs.", + "reflectionSeams": "DeploymentLoadPublisher.PublishStatistics(CancellationToken) and its IDisposable _publishTimer field, verified against the pinned source and current source. Publication, timer pause and deliberate cache-loss injection run on the publisher scheduler. Pausing disposes the timer and retains its reference for normal lifecycle cleanup.", + "lifecycleSignature": "GatedMembershipGossiper compiles against the original no-token IMembershipGossiper method for Old, and the required CancellationToken method on the current dissemination top for New.", + "newOnly": "DisseminationOptions and namespace options; IDisseminationService/IDisseminationSystemTarget; MembershipDisseminationNamespace and DeploymentLoadStatisticsDisseminationNamespace; DisseminationMembership topology and bounded repair contracts. Enabled recovery hosts register a test-only incoming-call filter which gates only PushBroadcast, never ExchangeAntiEntropy.", + "diagnostics": "Existing Microsoft.Orleans.Dissemination ValueApply event, Microsoft.Orleans messaging/dissemination meters, and System.Net.Sockets counters. No additional production diagnostic API is required.", + "notRequired": "No cluster-manifest hash RPC, and no direct access to notification locks, tombstone bookkeeping or dedup internals.", + "pendingProviderApi": "The candidate FileMembershipTable still targets the confirmed pinned/current interface. Token-bearing provider implementations and the conditional legacy bridge will be updated after the provider owner confirms the new API shape; no prospective overload signatures are assumed." + }, + "baseline": { + "repository": "dotnet/orleans", + "commit": "6739589254b746a8790cf53524e6abe372bb53d4", + "build": "Separate CI checkout. Only Silo and Shared test sources are copied into it. Its runtime source tree must be pristine and must lack dissemination.", + "identity": "Independent publish directory and dotnet OS process per node. Assert PID, runtime SHA256, assembly name/version, informational commit, MVID difference, and presence/absence of the dissemination target. Every loaded Orleans assembly must match its published SHA/version and live in that process's binary directory; Runtime, Core, Core.Abstractions and Serialization must all be present.", + "assemblyVersion": "Numeric assembly versions can legitimately be equal. They are checked against build manifests, not falsely required to differ.", + "controller": "MTP/xUnit controller references no Orleans assembly. The separate silo executable reuses the existing Orleans.Runtime.Tests friend identity." + }, + "featureBoundary": { + "initialLifecycleBaseCommit": "789e8cc27394b78e9a04887409a4fcec965b1b4f", + "new": "Build the current dissemination top, including its main-based lifecycle-prerequisite commits.", + "old": "Always build the exact pinned pre-feature main commit; do not apply the lifecycle prerequisite or dissemination patches to it.", + "manifestHashRpc": "No test uses a cluster-manifest hash RPC. Binary manifest.json files and local DLL SHA256 checks only attest which assemblies each OS process loaded; they are unrelated to the excluded manifest-hash feature.", + "compatibility": "Probe the actual IDisseminationSystemTarget RPC on an old process, then require a fresh exact production load sample through legacy delivery while that process remains unconfirmed.", + "sliceOwnership": "All harness and CI files belong to the dissemination/evidence slice. The current dissemination top inherits the lifecycle and scoped-RPC prerequisites and applies load updates on the publisher scheduler while preserving default-off notification behavior. The pinned old source retains its original runtime implementation." + }, + "infrastructure": { + "hosting": "Existing TestClusterHostFactory, SystemTarget and code generation; OS process orchestration follows StandaloneSiloHandle without its single-binary configuration assumption.", + "membership": "Small JSON provider with inter-process exclusive file locking, atomic replacement, version/row ETags, heartbeat-only updates and per-node frozen read snapshots.", + "control": "Stable aliased Orleans system-target Echo and Hold RPCs with required CancellationToken parameters. Local test commands use redirected stdin/stdout, not an exposed control server.", + "partition": "Abort existing real silo connections and reject new inbound/outbound connections at connection middleware. Cost recovery drains captured middleware and connection-close tasks at both endpoints, including unidentified inbound handshakes, before reopening and confirming acknowledged control RPCs.", + "scope": "One machine, loopback TCP, real serializer/transport, real membership/load namespaces. Liveness failure detection is disabled to keep partition repair distinct from membership eviction." + }, + "gates": [ + { + "test": "PartitionHealing_ConfirmsTransportBeforeSinglePublicationRound", + "assertions": "Repeat three partition/heal cycles on pinned old, current default-off and current enabled binaries. The first cycle proves an acknowledged control probe fails during the partition; remaining cycles publish and heal immediately. Healing retires old connections at both endpoints while the isolated endpoint rejects new handshakes, then bidirectional control RPCs confirm each exact remote PID. Full load snapshots must converge after one production publication per node with automatic publishers paused." + }, + { + "test": "PinnedBinaries_RollForward_MixedEnablement_AndRollback", + "assertions": "Three pinned old processes are replaced one at a time by one default-disabled and two enabled new processes, then rolled back to pinned old processes. Exact load state equality and stable cross-process RPC PID identity are checked at each stage. Unsupported old IDisseminationSystemTarget RPC failures cannot be inferred from timeouts and must be followed by fresh exact load-state delivery to the still-unconfirmed old process. Disabled new nodes return explicit namespace-unsupported and retain legacy delivery. Enabled peers confirm support. Disabled nodes have no dissemination sends, applies or outgoing anti-entropy." + }, + { + "test": "PartitionHistoryAndHeartbeat_RequireDisseminationProvenance", + "assertions": "Freeze central-table reads and suppress legacy membership gossip on all nodes. Close the test-only native PushBroadcast incoming-call gate on every node and await all previously admitted PushBroadcast invocations before creating the history gap. Probe the actual RPC to verify rejection. Keep ExchangeAntiEntropy enabled. Isolate one real process while the production membership namespace sees 40 actual table versions. Its 32-entry history must miss the receiver baseline and produce FromVersion=0. Heal transport and require exact full membership state plus Applied provenance while every tree gate stays closed with unchanged admitted-call counts. Keep those gates closed for same-version heartbeat and missing-load full repair. Disabling anti-entropy cannot be rescued by pending/retried/rematerialized tree sends.", + "nonClaims": "TTL expiry is not queue expiry and is not used as a tree-drain proof. FromVersion=0 alone does not prove anti-entropy. Legacy gossip, central-table reads and all newly admitted PushBroadcast calls are excluded throughout these assertions." + }, + { + "test": "RuntimeStatisticsComparison_ObservesPublicReadonlyFieldMutation", + "assertions": "Execute a mutation probe inside both the pinned old and current OS processes using the real EnvironmentStatistics type. Changing only RawCpuUsagePercentage must change the shared comparison representation; every reflected public instance field must be present." + }, + { + "test": "RequiredCancellationAndPartitionedShutdown_AreBounded", + "assertions": "Start a required-token RPC from the sender's system-target scheduler, retain its CTS across commands, and wait for an explicit receiver snapshot proving one pending started call with a live uncanceled token before issuing cancellation. Invoke CancelAsync, await the actual raw RPC task using an independent command deadline, then require exactly one receiver-observed cancellation and no pending server call. Save start/cancel timestamps and raw-task status; do not accept only cancellation of a local WaitAsync. A partitioned dissemination sender must shut down without controller kill inside 35 seconds." + } + ], + "focusedRegressions": [ + { + "test": "ConnectionDrain_WaitsForInitializationAndCloseBeforeReopening", + "scope": "The partition captures middleware before initialization completes, rejects new admissions, and waits for both middleware exit and connection close before reopening." + }, + { + "test": "ConnectionDrain_CapturesUnidentifiedPeersAndPreservesHealthyConnections", + "scope": "Peer-specific drain includes unidentified handshakes, preserves healthy connections, and retains admission blocking when the drain caller is canceled." + }, + { + "test": "ControlledCall_WaitsForRawCancellationAfterExplicitSignal", + "scope": "The durable controller call has no automatic timer, retains a live token until explicit cancellation, and cannot complete merely because cancellation was requested while its raw task is still pending." + }, + { + "test": "ControlledCall_DoesNotTreatRawTimeoutAsCancellation", + "scope": "A raw RPC timeout after cancellation is requested remains a failure, rather than being mistaken for successful cooperative cancellation." + }, + { + "test": "StateComparison_ObservesNestedPublicReadonlyFieldChanges", + "scope": "Lightweight controller regression for the exact cached IncludeFields serializer used by the host load-state comparison. Demonstrates that the previous properties-only serializer misses the mutation." + }, + { + "test": "InFlightCallGate_DrainsPriorCallsAndRejectsLaterCalls", + "scope": "Lightweight regression for the admission/drain primitive used by the native PushBroadcast filter. A prior active call delays the barrier; subsequent calls cannot enter before or after drain until explicitly reopened." + } + ], + "load": { + "test": "Scaling_OriginalLegacy_DefaultOff_EnabledSupported", + "comparison": "Three separately labeled paths: pinned original legacy binary; actual current binary with untouched dissemination-off defaults; the same current binary enabled using production dissemination defaults after peers have confirmed support.", + "runtimePaths": [ + { "name": "OriginalLegacy", "runtime": "Old", "enabled": false }, + { "name": "CurrentDefaultOff", "runtime": "New", "enabled": false }, + { "name": "CurrentEnabledSupported", "runtime": "New", "enabled": true } + ], + "automaticPrProfile": { "mode": "Cost", "sizes": [4, 8], "iterations": 3, "pairedRepetitions": 1, "overrideable": false }, + "manualDefaults": { "mode": "Scale", "sizes": [4, 8, 16], "iterations": 10, "pairedRepetitions": 1 }, + "controls": { "sizes": "1..6 comma-separated sizes, each 3..32", "iterations": "3..200", "pairedRepetitions": "1..5" }, + "scenarios": "Stable; graceful stop/restart at the same endpoint every other round; real connection partition and heal with old pooled connections retired at both endpoints before reopening and confirming transport readiness via bidirectional acknowledged control RPCs. Teardown, readiness probes and retries share a 60-second deadline and remain inside recovery latency and total cost. Each path then offers the same single publication round. Gate artifacts separately cover silent repair, retained-history misses and same-version heartbeat recovery.", + "cadence": "Pause automatic deployment-load timers after startup. Invoke the actual production PublishStatistics method once per active node per offered round, then compare state using cached JsonSerializerOptions with IncludeFields=true, including every public readonly EnvironmentStatistics field. Do not replace it with a formula or synthetic protocol implementation.", + "window": "Two warmup publication rounds and complete enabled-peer capability discovery before measuring. Replacements also finish capability discovery before offered rounds; their bootstrap/fallback/discovery costs remain inside the churn window. Two-second OS-counter sampling margins on all paths. Full-GC retained-memory probes are after the CPU/allocation window.", + "metrics": [ + "All transmitted request AND one-way RPC messages; responses reported in the separate total-message count", + "Complete serialized Orleans message bytes, including headers/body, from message-size observations", + "Actual process-wide System.Net.Sockets sent/received bytes, sampled each second; transport-pipe submitted/read bytes separately", + "Process CPU time, GC total allocated bytes, post-full-GC retained managed bytes, working set and private bytes", + "Per-round exact state-convergence latency and partition recovery latency", + "Per-node counters, actual originator/forwarding targets, tree rank and upper-tree role; raw snapshots preserve topology changes", + "Raw protocol payload, namespace-batch and anti-entropy counters, never substituted for total RPC or byte counts" + ], + "limitations": [ + "Socket counters exclude TCP/IP headers and retransmissions; transport submissions can include bytes buffered then lost on an aborted connection.", + "One-second socket-counter sampling and non-atomic cross-process snapshots impose boundary uncertainty. Raw captured UTC timestamps are included.", + "Instrumentation and stdin/state-snapshot overhead are included. Detailed counters are serialized only at cost boundaries and per-value diagnostic events are disabled for scale runs.", + "Controller CPU/allocation, disk byte costs, production providers, application grains, cross-machine latency and NIC-level wire costs are not measured. This is a closed-loop control-plane workload with a convergence barrier each round, not a saturation-throughput benchmark.", + "Retained managed memory is a post-GC live-size sample on surviving nodes, not peak memory. Retired process snapshots and all-node RSS/private-memory samples remain in artifacts.", + "Normal partition cost comparisons include a transport-readiness barrier followed by the same single publication round. The separate anti-entropy-only gate measures silent repair.", + "Original-to-current default-off deltas include the lifecycle and scoped-RPC prerequisites and feature registration costs. The current off/on pair uses the identical current binary and isolates enabling the subsystem.", + "Raw samples and repetitions are evidence inputs, not predeclared improvement thresholds or broad-enable approval." + ] + }, + "bounds": { + "startupSeconds": 90, + "controllerStartupSeconds": 100, + "siloCommandSeconds": 60, + "controllerCommandSeconds": 70, + "convergenceSeconds": 60, + "scaleCapabilityDiscoverySeconds": 90, + "membershipLockSeconds": 10, + "shutdownSeconds": 35, + "individualProcessLifetimeMinutes": 45, + "gateJobMinutes": 35, + "automaticCostJobMinutes": 25, + "manualScaleJobMinutes": 90, + "rpcLifetimeScope": "ResponseTimeout bounds locally observable raw RPC completion, not remote execution. The harness does not require the fourth-slice per-call acknowledgment-scope API. Cooperative Hold cancellation is checked separately through the remote Started/Cancelled/Pending counters, not inferred from caller cancellation or RPC completion.", + "failure": "Fail, preserve exact state/raw stdout/stderr/binlogs/TRX, and kill only the specific owned child process after its shutdown bound. A forced kill is not a successful bounded shutdown.", + "largeDispatch": "The job lifetime is a hard budget, including extreme user-selected size/iteration combinations." + }, + "artifacts": { + "gate": "dissemination-gate-evidence: TRX, identity/probe snapshots, isolated repair checkpoints, bounded shutdown outcomes, raw logs, methodology and binlogs", + "binaries": "dissemination-binaries: pinned old/current/runner publish directories plus SHA manifests for explicit replay using -SkipBuild", + "automaticCost": "dissemination-bounded-cost-evidence: cost-results.json and per-window/raw artifacts for the fixed automatic PR profile, including actual restart and partition counts", + "scale": "dissemination-scale-evidence: cost-results.json, per-window cost.json, raw start/finish/warmup/node snapshots and partial results on failure" + } +} From 67a6f928436a286799796b4652719eb630762853 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 15:11:35 -0700 Subject: [PATCH 03/50] test(dissemination): isolate current convergence regressions --- .../Membership/MembershipGossiperTests.cs | 358 +- .../DisseminationCancellationTests.cs | 70 + .../DisseminationClusterTests.cs | 129 + .../DisseminationMembershipSnapshotTests.cs | 685 ++ .../DisseminationProtocolTests.cs | 8952 +++++++++++++++++ .../Dissemination/WakeTimerTests.cs | 237 + .../Orleans.Runtime.Internal.Tests.csproj | 1 + 7 files changed, 10430 insertions(+), 2 deletions(-) create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationCancellationTests.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs diff --git a/test/Orleans.Core.Tests/Membership/MembershipGossiperTests.cs b/test/Orleans.Core.Tests/Membership/MembershipGossiperTests.cs index e945bf8d73f..801276a6344 100644 --- a/test/Orleans.Core.Tests/Membership/MembershipGossiperTests.cs +++ b/test/Orleans.Core.Tests/Membership/MembershipGossiperTests.cs @@ -8,6 +8,7 @@ using Orleans.Core.Diagnostics; using Orleans.Internal; using Orleans.Runtime; +using Orleans.Runtime.Dissemination; using Orleans.Runtime.MembershipService; using Orleans.Runtime.Scheduler; using TestExtensions; @@ -18,9 +19,249 @@ namespace NonSilo.Tests.Membership; [TestCategory("BVT"), TestCategory("Membership")] [TestSuite("BVT")] [TestProvider("None")] -[TestArea("Runtime")] +[TestArea("Dissemination")] public class MembershipGossiperTests { + [Fact] + public async Task GossipToRemoteSilos_EligibleLocalStatus_StartsDirectAndDisseminationBeforeEitherCompletes() + { + var directStarted = NewBarrier(); + var releaseDirect = NewBarrier(); + var directCompleted = NewBarrier(); + var disseminationStarted = NewBarrier(); + var releaseDissemination = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + DisseminationKey? publishedKey = null; + long? publishedVersion = null; + CancellationToken publishedToken = default; + + using var rig = CreateDisseminationTestRig( + async (_, cancellationToken) => + { + directStarted.TrySetResult(); + await releaseDirect.Task.WaitAsync(cancellationToken); + directCompleted.TrySetResult(); + }, + async (key, version, cancellationToken) => + { + publishedKey = key; + publishedVersion = version; + publishedToken = cancellationToken; + disseminationStarted.TrySetResult(); + return await releaseDissemination.Task.WaitAsync(cancellationToken); + }); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Active); + var gossipTask = rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.ShuttingDown, TestContext.Current.CancellationToken); + + try + { + await Task.WhenAll(directStarted.Task, disseminationStarted.Task).WaitAsync(TestContext.Current.CancellationToken); + Assert.False(gossipTask.IsCompleted); + Assert.False(directCompleted.Task.IsCompleted); + Assert.Equal(1, rig.DisseminationServiceResolutionCount); + } + finally + { + releaseDirect.TrySetResult(); + releaseDissemination.TrySetResult(true); + } + + await gossipTask; + await directCompleted.Task.WaitAsync(TestContext.Current.CancellationToken); + + await rig.RemoteMembershipService.Received(1).MembershipChangeNotification(snapshot, TestContext.Current.CancellationToken); + await rig.DisseminationService.Received(1).Publish( + Arg.Any(), DisseminationKey.Default, snapshot.Version.Value, TestContext.Current.CancellationToken); + Assert.Equal(DisseminationKey.Default, publishedKey); + Assert.Equal(snapshot.Version.Value, publishedVersion); + Assert.Equal(TestContext.Current.CancellationToken, publishedToken); + } + + [Fact] + public async Task GossipToRemoteSilos_IneligibleLocalStatus_SkipsDisseminationAndCompletesDirectGossip() + { + MembershipTableSnapshot? deliveredSnapshot = null; + using var rig = CreateDisseminationTestRig( + (snapshot, _) => + { + deliveredSnapshot = snapshot; + return Task.CompletedTask; + }, + (_, _, _) => throw new Xunit.Sdk.XunitException("Ineligible membership must not publish via dissemination.")); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Dead); + + await rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.Dead, TestContext.Current.CancellationToken); + + Assert.Same(snapshot, deliveredSnapshot); + await rig.RemoteMembershipService.Received(1).MembershipChangeNotification(snapshot, TestContext.Current.CancellationToken); + Assert.Equal(0, rig.DisseminationServiceResolutionCount); + Assert.Empty(rig.DisseminationService.ReceivedCalls()); + } + + [Fact] + public async Task GossipToRemoteSilos_CallerCancellation_CancelsDirectGossipAndDisseminationPublish() + { + var directStarted = NewBarrier(); + var disseminationStarted = NewBarrier(); + CancellationToken directToken = default; + CancellationToken publishedToken = default; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + using var rig = CreateDisseminationTestRig( + async (_, cancellationToken) => + { + directToken = cancellationToken; + directStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + }, + async (_, _, cancellationToken) => + { + publishedToken = cancellationToken; + disseminationStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return true; + }); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Active); + var gossipTask = rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.Stopping, cancellation.Token); + + try + { + await Task.WhenAll(directStarted.Task, disseminationStarted.Task).WaitAsync(TestContext.Current.CancellationToken); + await rig.RemoteMembershipService.Received(1).MembershipChangeNotification(snapshot, cancellation.Token); + } + finally + { + cancellation.Cancel(); + } + + await Assert.ThrowsAnyAsync(() => gossipTask); + Assert.True(gossipTask.IsCanceled); + Assert.Equal(cancellation.Token, directToken); + Assert.Equal(cancellation.Token, publishedToken); + Assert.True(directToken.IsCancellationRequested); + Assert.True(publishedToken.IsCancellationRequested); + } + + [Fact] + public async Task GossipToRemoteSilos_PreCanceled_DoesNotSendOrPublish() + { + using var cancellation = new CancellationTokenSource(); + using var rig = CreateDisseminationTestRig( + (_, _) => Task.CompletedTask, + (_, _, _) => ValueTask.FromResult(true)); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Active); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.Active, cancellation.Token)); + + Assert.Empty(rig.RemoteMembershipService.ReceivedCalls()); + Assert.Empty(rig.DisseminationService.ReceivedCalls()); + } + + [Fact] + public async Task MembershipGossiperStartsDirectGossipBeforeDissemination() + { + var directStarted = NewBarrier(); + var releaseDirect = NewBarrier(); + var disseminationStarted = NewBarrier(); + var releaseDissemination = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var resolutionOrder = new List(); + using var rig = CreateDisseminationTestRig( + async (_, cancellationToken) => + { + directStarted.TrySetResult(); + await releaseDirect.Task.WaitAsync(cancellationToken); + }, + async (_, _, cancellationToken) => + { + disseminationStarted.TrySetResult(); + return await releaseDissemination.Task.WaitAsync(cancellationToken); + }, + resolutionOrder); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Active); + var gossipTask = rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.ShuttingDown, TestContext.Current.CancellationToken); + + try + { + await Task.WhenAll(directStarted.Task, disseminationStarted.Task).WaitAsync(TestContext.Current.CancellationToken); + Assert.Equal(["MembershipSystemTargetResolved", "DisseminationServiceResolved"], resolutionOrder); + } + finally + { + releaseDirect.TrySetResult(); + releaseDissemination.TrySetResult(true); + } + + await gossipTask; + await rig.RemoteMembershipService.Received(1).MembershipChangeNotification(snapshot, TestContext.Current.CancellationToken); + await rig.DisseminationService.Received(1).Publish( + Arg.Any(), DisseminationKey.Default, snapshot.Version.Value, TestContext.Current.CancellationToken); + } + + [Fact] + public async Task MembershipGossiperDirectDeliveryRemainsAuthoritativeWhenDisseminationFails() + { + var disseminationFailure = new InvalidOperationException("Simulated dissemination failure."); + DisseminationKey? publishedKey = null; + long? publishedVersion = null; + using var rig = CreateDisseminationTestRig( + (_, _) => Task.CompletedTask, + (key, version, _) => + { + publishedKey = key; + publishedVersion = version; + throw disseminationFailure; + }); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Active); + + await rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.ShuttingDown, TestContext.Current.CancellationToken); + + Assert.Equal(DisseminationKey.Default, publishedKey); + Assert.Equal(snapshot.Version.Value, publishedVersion); + await rig.RemoteMembershipService.Received(1).MembershipChangeNotification(snapshot, TestContext.Current.CancellationToken); + await rig.DisseminationService.Received(1).Publish( + Arg.Any(), DisseminationKey.Default, snapshot.Version.Value, TestContext.Current.CancellationToken); + } + + [Fact] + public async Task MembershipGossiperPreservesEligibilityAndCallerCancellation() + { + var directStarted = NewBarrier(); + CancellationToken directToken = default; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + using var rig = CreateDisseminationTestRig( + async (_, cancellationToken) => + { + directToken = cancellationToken; + directStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + }, + (_, _, _) => throw new Xunit.Sdk.XunitException("Ineligible membership must not publish via dissemination.")); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Dead); + var gossipTask = rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.Dead, cancellation.Token); + + try + { + await directStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + } + finally + { + cancellation.Cancel(); + } + + var exception = await Assert.ThrowsAnyAsync(() => gossipTask); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(cancellation.Token, directToken); + Assert.Equal(0, rig.DisseminationServiceResolutionCount); + Assert.Empty(rig.DisseminationService.ReceivedCalls()); + await rig.RemoteMembershipService.Received(1).MembershipChangeNotification(snapshot, cancellation.Token); + } + [Theory] [InlineData("Gossip")] [InlineData("Probe")] @@ -158,10 +399,123 @@ private static TestRig CreateTestRig() }); var serviceProvider = services.BuildServiceProvider(); return new(serviceProvider, - new MembershipGossiper(serviceProvider, NullLogger.Instance), + new MembershipGossiper(serviceProvider, localDetails, NullLogger.Instance), new RemoteSiloProber(serviceProvider), remote, localSilo, remoteSilo); } + private static DisseminationTestRig CreateDisseminationTestRig( + Func directGossip, + Func> disseminationPublish, + List? resolutionOrder = null) + { + var localSilo = SiloAddress.FromParsableString("127.0.0.1:100@100"); + var remoteSilo = SiloAddress.FromParsableString("127.0.0.1:200@100"); + var localDetails = Substitute.For(); + localDetails.SiloAddress.Returns(localSilo); + var remote = Substitute.For(); + remote.MembershipChangeNotification(Arg.Any(), Arg.Any()) + .Returns(call => directGossip(call.ArgAt(0), call.ArgAt(1))); + var grainFactory = Substitute.For(); + grainFactory.GetSystemTarget(Constants.MembershipServiceType, Arg.Any()).Returns(remote); + var membershipManager = Substitute.For(); + var options = Substitute.For>(); + options.CurrentValue.Returns(new ClusterMembershipOptions + { + Dissemination = new DisseminationNamespaceOptions { Enabled = true }, + }); + var disseminationNamespace = new MembershipDisseminationNamespace(membershipManager, options, serializer: null!); + var disseminationService = Substitute.For(); + disseminationService.Publish( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => disseminationPublish( + call.ArgAt(1), call.ArgAt(2), call.ArgAt(3))); + var disseminationServiceResolutionCount = 0; + var services = new ServiceCollection(); + services.AddMetrics(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(localDetails); + services.AddSingleton(disseminationNamespace); + services.AddSingleton(_ => + { + disseminationServiceResolutionCount++; + resolutionOrder?.Add("DisseminationServiceResolved"); + return disseminationService; + }); + services.AddSingleton(provider => + { + resolutionOrder?.Add("MembershipSystemTargetResolved"); + var shared = new SystemTargetShared( + runtimeClient: null!, + localDetails, + NullLoggerFactory.Instance, + Options.Create(new SchedulingOptions()), + grainReferenceActivator: null!, + timerRegistry: null!, + new ActivationDirectory(provider.GetRequiredService()), + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService()); + return new MembershipSystemTarget( + membershipManager, + NullLogger.Instance, + grainFactory, + provider.GetRequiredService(), + shared, + TimeProvider.System); + }); + var serviceProvider = services.BuildServiceProvider(); + return new( + serviceProvider, + new MembershipGossiper(serviceProvider, localDetails, NullLogger.Instance), + remote, + disseminationService, + localSilo, + remoteSilo, + () => disseminationServiceResolutionCount); + } + + private static MembershipTableSnapshot CreateSnapshot(SiloAddress localSilo, SiloAddress remoteSilo, SiloStatus localStatus) + { + var startTime = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + return new( + new MembershipVersion(7), + ImmutableDictionary.Empty + .Add(localSilo, CreateEntry(localSilo, localStatus, startTime)) + .Add(remoteSilo, CreateEntry(remoteSilo, SiloStatus.Active, startTime))); + } + + private static MembershipEntry CreateEntry(SiloAddress silo, SiloStatus status, DateTime startTime) => new() + { + SiloAddress = silo, + Status = status, + HostName = "localhost", + SiloName = silo.ToParsableString(), + StartTime = startTime, + IAmAliveTime = startTime, + }; + + private static TaskCompletionSource NewBarrier() => new(TaskCreationOptions.RunContinuationsAsynchronously); + + private sealed record DisseminationTestRig( + ServiceProvider ServiceProvider, + MembershipGossiper Gossiper, + IMembershipService RemoteMembershipService, + IDisseminationService DisseminationService, + SiloAddress LocalSilo, + SiloAddress RemoteSilo, + Func DisseminationServiceResolutions) : IDisposable + { + public int DisseminationServiceResolutionCount => DisseminationServiceResolutions(); + + public void Dispose() => ServiceProvider.Dispose(); + } + private sealed record TestRig( ServiceProvider ServiceProvider, MembershipGossiper Gossiper, diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationCancellationTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationCancellationTests.cs new file mode 100644 index 00000000000..1a1bb763bbd --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationCancellationTests.cs @@ -0,0 +1,70 @@ +using System; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +[TestCategory("BVT"), TestCategory("Dissemination")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Dissemination")] +public sealed class DisseminationCancellationTests +{ + [Theory] + [InlineData(typeof(IDisseminationSystemTarget))] + public void DisseminationRpcMethodsExposeRequiredCancellationToken(Type interfaceType) + { + foreach (var method in interfaceType.GetMethods()) + { + Assert.Equal(typeof(CancellationToken), method.GetParameters().LastOrDefault()?.ParameterType); + Assert.False(method.GetParameters()[^1].IsOptional); + } + } + + [Theory] + [InlineData(typeof(DisseminationProtocol))] + [InlineData(typeof(DisseminationBroadcastQueue))] + [InlineData(typeof(DisseminationSendGate))] + [InlineData(typeof(DisseminationSystemTarget))] + [InlineData(typeof(DisseminationMembership))] + [InlineData(typeof(MembershipDisseminationNamespace))] + [InlineData(typeof(DeploymentLoadStatisticsDisseminationNamespace))] + [InlineData(typeof(WakeTimer))] + public void DisseminationAsyncMethodsExposeRequiredCancellationToken(Type componentType) + { + var types = componentType.GetNestedTypes(BindingFlags.Public | BindingFlags.NonPublic).Prepend(componentType); + foreach (var type in types) + { + foreach (var method in type.GetMethods( + BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly)) + { + var returnType = method.ReturnType; + var isAsync = typeof(Task).IsAssignableFrom(returnType) + || returnType == typeof(ValueTask) + || returnType.IsGenericType && returnType.GetGenericTypeDefinition() == typeof(ValueTask<>); + if (!isAsync || method.IsSpecialName || method.Name.StartsWith('<')) + { + continue; + } + + // IAsyncDisposable defines the parameterless cleanup boundary. + if (method.Name == nameof(IAsyncDisposable.DisposeAsync) && typeof(IAsyncDisposable).IsAssignableFrom(type)) + { + continue; + } + + Assert.True( + method.GetParameters().Any(static parameter => parameter.ParameterType == typeof(CancellationToken)), + $"{type.FullName}.{method.Name} must expose a CancellationToken parameter."); + Assert.All( + method.GetParameters().Where(static parameter => parameter.ParameterType == typeof(CancellationToken)), + static parameter => Assert.False(parameter.IsOptional)); + } + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs new file mode 100644 index 00000000000..27f30168059 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs @@ -0,0 +1,129 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Configuration; +using Orleans.Hosting; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.MembershipService; +using Orleans.TestingHost; +using Xunit; + +namespace UnitTests.Dissemination; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class DisseminationDiagnosticCollection +{ + public const string Name = "Dissemination diagnostics"; +} + +/// +/// Exercises dissemination across a real in-process cluster so that membership updates flow through the +/// full serialization pipeline (the in-memory transport is a byte pipe, not object pass-through). This +/// guards against wire types that lack serialization codecs, which unit tests using a fake transport +/// cannot catch. +/// +[TestCategory("Functional"), TestCategory("Dissemination")] +[TestSuite("Functional")] +[TestProvider("None")] +[TestArea("Dissemination")] +[Collection(DisseminationDiagnosticCollection.Name)] +public sealed class DisseminationClusterTests +{ + [Fact] + public async Task MembershipUpdatesAreDisseminatedAcrossRealCluster() + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cancellation.CancelAfter(TimeSpan.FromSeconds(120)); + var cancellationToken = cancellation.Token; + + var builder = new InProcessTestClusterBuilder(3); + builder.ConfigureSilo((_, siloBuilder) => + { + // Dissemination and the membership namespace are disabled by default; enable both. + siloBuilder.Configure(options => options.Enabled = true); + siloBuilder.Configure(options => options.Dissemination.Enabled = true); + }); + + await using var cluster = builder.Build(); + await cluster.DeployAsync(cancellationToken); + await cluster.WaitForLivenessToStabilizeAsync().WaitAsync(cancellationToken); + + var existingSilos = cluster.GetActiveSilos().Select(static silo => silo.SiloAddress).ToHashSet(); + var baselineVersion = cluster.Silos[0].ServiceProvider + .GetRequiredService() + .CurrentSnapshot.Version.Value; + + // Arm the observer after initial stabilization and accept only a later membership version applied by + // pre-existing silos, so cluster startup or unrelated parallel tests cannot satisfy the assertion. + var observer = new ValueApplyObserver(targetDistinctSilos: 2, existingSilos, baselineVersion); + using var subscription = DisseminationEvents.Listener.Subscribe( + observer, + static name => name == "Dissemination.ValueApply"); + + // Adding a silo produces new membership updates that must propagate to the existing silos. + await cluster.StartAdditionalSiloAsync().WaitAsync(cancellationToken); + await cluster.WaitForLivenessToStabilizeAsync().WaitAsync(cancellationToken); + + await observer.Reached.WaitAsync(cancellationToken); + + Assert.True( + observer.AppliedSilos.Count >= 2, + "Expected membership updates to be applied on at least 2 distinct silos, but saw: " + + string.Join(", ", observer.AppliedSilos.Select(static silo => silo.ToString()))); + } + + private sealed class ValueApplyObserver( + int targetDistinctSilos, + IReadOnlySet expectedSilos, + long baselineVersion) : IObserver> + { + private readonly object _lock = new(); + private readonly HashSet _appliedSilos = []; + private readonly TaskCompletionSource _reached = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task Reached => _reached.Task; + + public IReadOnlyCollection AppliedSilos + { + get + { + lock (_lock) + { + return _appliedSilos.ToArray(); + } + } + } + + public void OnNext(KeyValuePair value) + { + if (value.Value is not DisseminationValueEvent evt + || evt.Namespace != DisseminationNamespaceNames.Membership + || evt.Key != DisseminationKey.Default + || evt.ToVersion <= baselineVersion + || !expectedSilos.Contains(evt.LocalSilo) + || evt.Result is not (nameof(DisseminationApplyResult.Applied) or nameof(DisseminationApplyResult.Duplicate)) + || evt.PayloadBytes <= 0) + { + return; + } + + lock (_lock) + { + _appliedSilos.Add(evt.LocalSilo); + if (_appliedSilos.Count >= targetDistinctSilos) + { + _reached.TrySetResult(); + } + } + } + + public void OnCompleted() { } + + public void OnError(Exception error) { } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs new file mode 100644 index 00000000000..a002a986794 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs @@ -0,0 +1,685 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Net; +using CsCheck; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +[TestCategory("BVT"), TestCategory("Dissemination")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Dissemination")] +public class DisseminationMembershipSnapshotTests +{ + [Fact] + public void CollectionsSatisfyMembershipAndRoutingInvariants() + { + Gen.Select(Gen.Int[1, 64], Gen.Int[0, 127], Gen.Int[0, 1], Gen.Int[0, 144], static (count, localSeed, localMode, fanoutSeed) => + { + return new ValidSnapshotTestCase( + count, + localSeed, + LocalIsMember: localMode == 0, + RawFanout: fanoutSeed - 16); + }).Sample(testCase => + { + var members = CreateSilos(testCase.Count).ToImmutableArray(); + var local = testCase.LocalIsMember + ? members[testCase.LocalSeed % members.Length] + : CreateSilo(20000 + testCase.LocalSeed); + var snapshot = CreateSnapshot(local, members, testCase.RawFanout); + + Assert.Equal(members, snapshot.Members); + AssertNoDuplicates(nameof(snapshot.Members), snapshot.Members); + + foreach (var member in members) + { + Assert.True(snapshot.ContainsMember(member)); + } + + var outsideMember = CreateSilo(25000 + testCase.LocalSeed); + Assert.False(snapshot.ContainsMember(outsideMember)); + + var originatorTargets = snapshot.OriginatorTreeTargets; + var forwardingTargets = snapshot.ForwardingTreeTargets; + + AssertNoDuplicates(nameof(originatorTargets), originatorTargets); + AssertNoDuplicates(nameof(forwardingTargets), forwardingTargets); + AssertSubset(nameof(originatorTargets), originatorTargets, members); + AssertSubset(nameof(forwardingTargets), forwardingTargets, members); + Assert.DoesNotContain(local, originatorTargets); + Assert.DoesNotContain(local, forwardingTargets); + Assert.Equal(originatorTargets, snapshot.OriginatorTreeTargets); + Assert.Equal(forwardingTargets, snapshot.ForwardingTreeTargets); + + if (!members.Contains(local)) + { + Assert.Empty(originatorTargets); + Assert.Empty(forwardingTargets); + return; + } + + var effectiveFanout = GetEffectiveFanout(members.Length, testCase.RawFanout); + var maxTargets = Math.Max(0, members.Length - 1); + Assert.True( + originatorTargets.Length <= Math.Min(effectiveFanout * 2, maxTargets), + "Originator target count exceeded the expected bound."); + Assert.True( + forwardingTargets.Length <= Math.Min(effectiveFanout, maxTargets), + "Forwarding target count exceeded the expected bound."); + }); + } + + [Fact] + public void AntiEntropyPeerSelectionSatisfiesMembershipInvariants() + { + Gen.Select(Gen.Int[1, 64], Gen.Int[0, 127], Gen.Int[0, 1], Gen.Int[0, 64], static (count, localSeed, localMode, requestedCount) => + { + return new AntiEntropyTestCase( + count, + localSeed, + LocalIsMember: localMode == 0, + RequestedCount: requestedCount); + }).Sample(testCase => + { + var members = CreateSilos(testCase.Count).ToImmutableArray(); + var local = testCase.LocalIsMember + ? members[testCase.LocalSeed % members.Length] + : CreateSilo(20000 + testCase.LocalSeed); + var snapshot = CreateSnapshot(local, members, fanout: 4); + + var selectedPeers = snapshot.SelectAntiEntropyPeers(testCase.RequestedCount); + + AssertNoDuplicates("Anti-entropy peers", selectedPeers); + AssertSubset("Anti-entropy peers", selectedPeers, members); + Assert.DoesNotContain(local, selectedPeers); + + var expectedCount = members.Contains(local) + ? Math.Min(testCase.RequestedCount, Math.Max(0, members.Length - 1)) + : 0; + Assert.Equal(expectedCount, selectedPeers.Length); + }); + } + + [Fact] + public void ConstructorRejectsDuplicateMembers() + { + Gen.Select(Gen.Int[1, 64], Gen.Int[0, 63], static (count, duplicateSeed) => (Count: count, DuplicateSeed: duplicateSeed)) + .Sample(testCase => + { + var members = CreateSilos(testCase.Count); + var duplicate = members[testCase.DuplicateSeed % members.Length]; + var invalidMembers = members.Append(duplicate).ToImmutableArray(); + + var exception = Assert.Throws(() => CreateSnapshot( + members[0], + invalidMembers, + fanout: 4)); + Assert.Equal("members", exception.ParamName); + }); + } + + private static DisseminationMembershipSnapshot CreateSnapshot( + SiloAddress localSilo, + ImmutableArray members, + int fanout) => new( + new MembershipVersion(1), + localSilo, + members, + CreateOverlayOptions(fanout)); + + private static DisseminationOverlayOptions CreateOverlayOptions(int fanout) => new() + { + FanOutFactor = _ => fanout, + }; + + private static int GetEffectiveFanout(int memberCount, int rawFanout) => + memberCount <= 1 ? 1 : Math.Clamp(rawFanout, 1, memberCount); + + private static void AssertNoDuplicates(string name, IEnumerable values) + { + var array = values.ToArray(); + Assert.True(array.Length == array.Distinct().Count(), $"{name} contains duplicates."); + } + + private static void AssertSubset(string name, IEnumerable values, IEnumerable expectedValues) + { + var expected = expectedValues.ToHashSet(); + foreach (var value in values) + { + Assert.True(expected.Contains(value), $"{name} contains {value} outside the expected membership set."); + } + } + + private static SiloAddress CreateSilo(int port) => SiloAddress.New(new IPEndPoint(IPAddress.Loopback, port), port); + + private static SiloAddress[] CreateSilos(int count) => + Enumerable.Range(11111, count).Select(CreateSilo).OrderBy(static silo => silo).ToArray(); + + private readonly record struct ValidSnapshotTestCase( + int Count, + int LocalSeed, + bool LocalIsMember, + int RawFanout); + + private readonly record struct AntiEntropyTestCase( + int Count, + int LocalSeed, + bool LocalIsMember, + int RequestedCount); + + [Fact] + public void ActiveScopeProjectionIncludesOnlyActiveSilos() + { + var (snapshots, manager, silos) = CreateScopeProjections(); + + Assert.Equal(new[] { silos.Local, silos.Active, silos.ActiveTwo }, snapshots.ActiveMembers.Members); + Assert.Equal(new MembershipVersion(42), snapshots.ActiveMembers.MembershipVersion); + Assert.DoesNotContain(silos.Joining, snapshots.ActiveMembers.Members); + Assert.DoesNotContain(silos.ShuttingDown, snapshots.ActiveMembers.Members); + Assert.DoesNotContain(silos.Stopping, snapshots.ActiveMembers.Members); + Assert.DoesNotContain(silos.Dead, snapshots.ActiveMembers.Members); + Assert.Equal(1, manager.SnapshotReadCount); + } + + [Fact] + public void AllEligibleScopeProjectionIncludesJoiningActiveShuttingDownAndStoppingSilos() + { + var (snapshots, manager, silos) = CreateScopeProjections(); + + Assert.Equal( + new[] { silos.Local, silos.Active, silos.ActiveTwo, silos.Joining, silos.ShuttingDown, silos.Stopping }, + snapshots.AllMembers.Members); + Assert.Equal(new MembershipVersion(42), snapshots.AllMembers.MembershipVersion); + Assert.DoesNotContain(silos.Dead, snapshots.AllMembers.Members); + Assert.Equal(1, manager.SnapshotReadCount); + } + + [Fact] + public void ScopeProjectionsShareTheSameSourceMembershipVersion() + { + var (snapshots, manager, _) = CreateScopeProjections(); + + Assert.Equal(new MembershipVersion(42), snapshots.MembershipVersion); + Assert.Equal(snapshots.MembershipVersion, snapshots.ActiveMembers.MembershipVersion); + Assert.Equal(snapshots.MembershipVersion, snapshots.AllMembers.MembershipVersion); + Assert.Equal(1, manager.SnapshotReadCount); + Assert.NotSame(snapshots.ActiveMembers, snapshots.AllMembers); + } + + [Fact] + public void ScopeProjectionTreeIsDeterministicForSelectedMemberArray() + { + var first = CreateScopeProjections(reverseSourceEntries: false).Snapshots; + var second = CreateScopeProjections(reverseSourceEntries: true).Snapshots; + var active = first.ActiveMembers.Members; + var all = first.AllMembers.Members; + + Assert.Equal(active, second.ActiveMembers.Members); + Assert.Equal(all, second.AllMembers.Members); + Assert.Equal(new[] { active[1], active[2] }, first.ActiveMembers.OriginatorTreeTargets); + Assert.Equal(new[] { active[2] }, first.ActiveMembers.ForwardingTreeTargets); + Assert.Equal( + new[] { all[1], all[2], all[3] }, + first.AllMembers.OriginatorTreeTargets); + Assert.Equal(new[] { all[2], all[3] }, first.AllMembers.ForwardingTreeTargets); + Assert.Equal(first.ActiveMembers.OriginatorTreeTargets, second.ActiveMembers.OriginatorTreeTargets); + Assert.Equal(first.AllMembers.ForwardingTreeTargets, second.AllMembers.ForwardingTreeTargets); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ScopeProjectionOrderingDoesNotDependOnStartTimePrecision(bool activeMembers) + { + var members = CreateSilos(6).Select(static silo => SiloAddress.New(silo.Endpoint, 42)).ToArray(); + var entries = members.Select((member, index) => + { + var entry = CreateScopeMembershipEntry(member, activeMembers ? SiloStatus.Active : SiloStatus.Joining, 0); + // A joining silo can publish its own entry before reading back the storage-rounded timestamp. + entry.StartTime = DateTime.UnixEpoch.AddTicks(index == 0 ? 1 : 0); + return entry; + }).ToArray(); + var persistedEntries = entries.Select(entry => + { + var persisted = entry.Copy(); + persisted.StartTime = LogFormatter.ParseDate(LogFormatter.PrintDate(entry.StartTime)); + return persisted; + }).ToArray(); + + foreach (var local in members) + { + var original = CreateProjections(entries, local); + var persisted = CreateProjections(persistedEntries, local); + + Assert.Equal(activeMembers ? members : [], original.ActiveMembers.Members); + Assert.Equal(activeMembers ? members : [], persisted.ActiveMembers.Members); + Assert.Equal(members, original.AllMembers.Members); + Assert.Equal(members, persisted.AllMembers.Members); + Assert.Equal(original.ActiveMembers.OriginatorTreeTargets, persisted.ActiveMembers.OriginatorTreeTargets); + Assert.Equal(original.ActiveMembers.ForwardingTreeTargets, persisted.ActiveMembers.ForwardingTreeTargets); + Assert.Equal(original.AllMembers.OriginatorTreeTargets, persisted.AllMembers.OriginatorTreeTargets); + Assert.Equal(original.AllMembers.ForwardingTreeTargets, persisted.AllMembers.ForwardingTreeTargets); + } + + static DisseminationMembershipSnapshots CreateProjections(MembershipEntry[] entries, SiloAddress local) => + new DisseminationMembership( + new CountingMembershipManager(new( + new MembershipVersion(42), + entries.ToImmutableDictionary(static entry => entry.SiloAddress))), + new ScopeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions + { + Overlay = CreateOverlayOptions(2), + })).CurrentSnapshots; + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MembershipVersionChangesPreserveAntiEntropyPeerRotation(bool activeScope) + { + var members = CreateSilos(5); + var local = members[0]; + var peers = members[1..]; + var manager = new MutableMembershipManager( + CreateSourceSnapshot(1, local, peers), + TestContext.Current.CancellationToken); + var membership = new DisseminationMembership( + manager, + new ScopeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + var scope = activeScope ? DisseminationMembershipScope.ActiveMembers : DisseminationMembershipScope.AllMembers; + + for (var round = 0; round < peers.Length * 2; round++) + { + manager.SetSnapshot(CreateSourceSnapshot(round + 1, local, peers)); + var snapshot = membership.GetSnapshot(scope); + + Assert.Equal(new MembershipVersion(round + 1), snapshot.MembershipVersion); + Assert.Equal(new[] { peers[round % peers.Length] }, snapshot.SelectAntiEntropyPeers(1)); + } + } + + [Fact] + public void AntiEntropyPeerRotationWrapsWhenMembershipShrinks() + { + var members = CreateSilos(5); + var local = members[0]; + var peers = members[1..]; + var manager = new MutableMembershipManager( + CreateSourceSnapshot(1, local, peers), + TestContext.Current.CancellationToken); + var membership = new DisseminationMembership( + manager, + new ScopeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + var previous = membership.CurrentSnapshot; + Assert.Equal(peers[..3], previous.SelectAntiEntropyPeers(3)); + + manager.SetSnapshot(CreateSourceSnapshot(2, local, peers[..2])); + var current = membership.CurrentSnapshot; + + Assert.Equal(new[] { peers[1], peers[0] }, current.SelectAntiEntropyPeers(2)); + Assert.Equal(new[] { peers[1] }, current.SelectAntiEntropyPeers(1)); + } + + [Fact] + public void ScopeProjectionAntiEntropySelectionIsDeterministic() + { + var snapshots = CreateScopeProjections().Snapshots; + var activePeers = snapshots.ActiveMembers.Members[1..]; + var allPeers = snapshots.AllMembers.Members[1..]; + + Assert.Equal(new[] { activePeers[0] }, snapshots.ActiveMembers.SelectAntiEntropyPeers(1)); + Assert.Equal(new[] { activePeers[1] }, snapshots.ActiveMembers.SelectAntiEntropyPeers(1)); + Assert.Equal(new[] { activePeers[0] }, snapshots.ActiveMembers.SelectAntiEntropyPeers(1)); + + for (var i = 0; i < allPeers.Length; i++) + { + Assert.Equal(new[] { allPeers[i] }, snapshots.AllMembers.SelectAntiEntropyPeers(1)); + } + + Assert.Equal(new[] { allPeers[0] }, snapshots.AllMembers.SelectAntiEntropyPeers(1)); + Assert.Equal(new MembershipVersion(42), snapshots.ActiveMembers.MembershipVersion); + Assert.Equal(new MembershipVersion(42), snapshots.AllMembers.MembershipVersion); + } + + [Fact] + public async Task ConcurrentRefreshDoesNotReturnOlderProjection() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(32001); + var firstPeer = CreateSilo(32002); + var secondPeer = CreateSilo(32003); + var manager = new MutableMembershipManager(CreateSourceSnapshot(40, local), cancellationToken); + var options = new BlockingDisseminationOptions(cancellationToken); + var membership = new DisseminationMembership( + manager, + new ScopeLocalSiloDetails(local), + options); + Assert.Equal(new MembershipVersion(40), membership.CurrentSnapshots.MembershipVersion); + + manager.SetSnapshot(CreateSourceSnapshot(41, local, firstPeer)); + var firstRefresh = Task.Run(() => membership.CurrentSnapshots, cancellationToken); + await options.RefreshBlocked.Task.WaitAsync(cancellationToken); + + manager.SetSnapshot(CreateSourceSnapshot(42, local, firstPeer, secondPeer)); + var secondRefresh = Task.Run(() => membership.CurrentSnapshots, cancellationToken); + await manager.ThirdRead.Task.WaitAsync(cancellationToken); + options.ReleaseRefresh(); + + Assert.Equal(new MembershipVersion(41), (await firstRefresh).MembershipVersion); + var latest = await secondRefresh; + Assert.Equal(new MembershipVersion(42), latest.MembershipVersion); + Assert.Equal(new[] { local, firstPeer, secondPeer }, latest.ActiveMembers.Members); + Assert.Equal(3, manager.SnapshotReadCount); + } + + [Fact] + public async Task StaleConcurrentRefreshDoesNotRegressNewerProjection() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(32101); + var firstPeer = CreateSilo(32102); + var secondPeer = CreateSilo(32103); + var manager = new MutableMembershipManager(CreateSourceSnapshot(40, local), cancellationToken) + { + BlockSecondSnapshotRead = true, + }; + var membership = new DisseminationMembership( + manager, + new ScopeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + Assert.Equal(new MembershipVersion(40), membership.CurrentSnapshots.MembershipVersion); + + manager.SetSnapshot(CreateSourceSnapshot(41, local, firstPeer)); + var staleRefresh = Task.Run(() => membership.CurrentSnapshots, cancellationToken); + await manager.SecondReadCaptured.Task.WaitAsync(cancellationToken); + + manager.SetSnapshot(CreateSourceSnapshot(42, local, firstPeer, secondPeer)); + var newer = await Task.Run(() => membership.CurrentSnapshots, cancellationToken); + manager.ReleaseSecondRead(); + var stale = await staleRefresh; + + Assert.Equal(new MembershipVersion(42), newer.MembershipVersion); + Assert.Equal(new MembershipVersion(42), stale.MembershipVersion); + Assert.Equal(new[] { local, firstPeer, secondPeer }, membership.CurrentSnapshots.ActiveMembers.Members); + Assert.Equal(4, manager.SnapshotReadCount); + } + + private static ( + DisseminationMembershipSnapshots Snapshots, + CountingMembershipManager Manager, + ScopeSilos Silos) CreateScopeProjections(bool reverseSourceEntries = false) + { + var silos = new ScopeSilos( + CreateSilo(31001), + CreateSilo(31002), + CreateSilo(31003), + CreateSilo(31004), + CreateSilo(31005), + CreateSilo(31006), + CreateSilo(31007)); + var entries = new[] + { + CreateScopeMembershipEntry(silos.Dead, SiloStatus.Dead, 5), + CreateScopeMembershipEntry(silos.Stopping, SiloStatus.Stopping, 4), + CreateScopeMembershipEntry(silos.ShuttingDown, SiloStatus.ShuttingDown, 3), + CreateScopeMembershipEntry(silos.Joining, SiloStatus.Joining, 2), + CreateScopeMembershipEntry(silos.ActiveTwo, SiloStatus.Active, 2), + CreateScopeMembershipEntry(silos.Active, SiloStatus.Active, 1), + CreateScopeMembershipEntry(silos.Local, SiloStatus.Active, 0), + }; + if (reverseSourceEntries) + { + Array.Reverse(entries); + } + + var source = new MembershipTableSnapshot( + new MembershipVersion(42), + entries.ToImmutableDictionary(static entry => entry.SiloAddress)); + var manager = new CountingMembershipManager(source); + var membership = new DisseminationMembership( + manager, + new ScopeLocalSiloDetails(silos.Local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions + { + Overlay = new DisseminationOverlayOptions + { + FanOutFactor = static _ => 2, + }, + })); + + return (membership.CurrentSnapshots, manager, silos); + } + + private static MembershipEntry CreateScopeMembershipEntry( + SiloAddress silo, + SiloStatus status, + int startSeconds) => new() + { + SiloAddress = silo, + Status = status, + ProxyPort = silo.Endpoint.Port, + HostName = "localhost", + SiloName = silo.ToParsableString(), + RoleName = "test", + StartTime = DateTime.UnixEpoch.AddSeconds(startSeconds), + IAmAliveTime = DateTime.UnixEpoch.AddSeconds(startSeconds), + }; + + private static MembershipTableSnapshot CreateSourceSnapshot( + long version, + SiloAddress local, + params SiloAddress[] peers) + { + var entries = peers + .Prepend(local) + .Select((silo, index) => CreateScopeMembershipEntry(silo, SiloStatus.Active, index)) + .ToImmutableDictionary(static entry => entry.SiloAddress); + return new(new MembershipVersion(version), entries); + } + + private readonly record struct ScopeSilos( + SiloAddress Local, + SiloAddress Active, + SiloAddress ActiveTwo, + SiloAddress Joining, + SiloAddress ShuttingDown, + SiloAddress Stopping, + SiloAddress Dead); + + private sealed class ScopeLocalSiloDetails(SiloAddress siloAddress) : ILocalSiloDetails + { + public string Name => "test"; + + public string ClusterId => "test"; + + public string DnsHostName => "localhost"; + + public SiloAddress SiloAddress => siloAddress; + + public SiloAddress GatewayAddress => siloAddress; + } + + private sealed class CountingMembershipManager( + MembershipTableSnapshot snapshot) + : Orleans.Runtime.MembershipService.IMembershipManager + { + public int SnapshotReadCount { get; private set; } + + public MembershipTableSnapshot CurrentSnapshot + { + get + { + SnapshotReadCount++; + return snapshot; + } + } + + public IAsyncEnumerable MembershipUpdates => + EmptyUpdates(TestContext.Current.CancellationToken); + + public SiloStatus LocalSiloStatus => SiloStatus.Active; + + public Task UpdateLocalStatus(SiloStatus status, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task TryKillSilo(SiloAddress silo, CancellationToken cancellationToken) => Task.FromResult(false); + + public Task TrySuspectSilo( + SiloAddress silo, + SiloAddress? indirectProbingSilo, + CancellationToken cancellationToken) => Task.FromResult(false); + + public Task Refresh(MembershipVersion? targetVersion, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ProcessGossipSnapshot( + MembershipTableSnapshot value, + CancellationToken cancellationToken) => Task.CompletedTask; + + public Task UpdateIAmAlive(CancellationToken cancellationToken) => Task.CompletedTask; + + public bool CheckHealth(DateTime lastCheckTime, out string reason) + { + reason = string.Empty; + return true; + } + + public void Participate(ISiloLifecycle lifecycle) + { + } + + private static async IAsyncEnumerable EmptyUpdates( + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.CompletedTask; + yield break; + } + } + + private sealed class MutableMembershipManager( + MembershipTableSnapshot snapshot, + CancellationToken cancellationToken) + : Orleans.Runtime.MembershipService.IMembershipManager + { + private readonly ManualResetEventSlim _releaseSecondRead = new(); + private MembershipTableSnapshot _snapshot = snapshot; + private int _snapshotReadCount; + + public int SnapshotReadCount => Volatile.Read(ref _snapshotReadCount); + + public bool BlockSecondSnapshotRead { get; init; } + + public TaskCompletionSource SecondReadCaptured { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource ThirdRead { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public MembershipTableSnapshot CurrentSnapshot + { + get + { + var result = Volatile.Read(ref _snapshot); + var readCount = Interlocked.Increment(ref _snapshotReadCount); + if (readCount == 2 && BlockSecondSnapshotRead) + { + SecondReadCaptured.TrySetResult(); + _releaseSecondRead.Wait(cancellationToken); + } + + if (readCount == 3) + { + ThirdRead.TrySetResult(); + } + + return result; + } + } + + public IAsyncEnumerable MembershipUpdates => EmptyUpdates(TestContext.Current.CancellationToken); + + public SiloStatus LocalSiloStatus => SiloStatus.Active; + + public void ReleaseSecondRead() => _releaseSecondRead.Set(); + + public void SetSnapshot(MembershipTableSnapshot value) => Volatile.Write(ref _snapshot, value); + + public Task UpdateLocalStatus(SiloStatus status, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task TryKillSilo(SiloAddress silo, CancellationToken cancellationToken) => Task.FromResult(false); + + public Task TrySuspectSilo( + SiloAddress silo, + SiloAddress? indirectProbingSilo, + CancellationToken cancellationToken) => Task.FromResult(false); + + public Task Refresh(MembershipVersion? targetVersion, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ProcessGossipSnapshot( + MembershipTableSnapshot value, + CancellationToken cancellationToken) => Task.CompletedTask; + + public Task UpdateIAmAlive(CancellationToken cancellationToken) => Task.CompletedTask; + + public bool CheckHealth(DateTime lastCheckTime, out string reason) + { + reason = string.Empty; + return true; + } + + public void Participate(ISiloLifecycle lifecycle) + { + } + + private static async IAsyncEnumerable EmptyUpdates( + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.CompletedTask; + yield break; + } + } + + private sealed class BlockingDisseminationOptions(CancellationToken cancellationToken) + : Microsoft.Extensions.Options.IOptions + { + private readonly ManualResetEventSlim _releaseRefresh = new(); + private int _readCount; + + public TaskCompletionSource RefreshBlocked { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public DisseminationOptions Value + { + get + { + if (Interlocked.Increment(ref _readCount) == 2) + { + RefreshBlocked.TrySetResult(); + _releaseRefresh.Wait(cancellationToken); + } + + return new() + { + Overlay = new() + { + FanOutFactor = static _ => 2, + }, + }; + } + } + + public void ReleaseRefresh() => _releaseRefresh.Set(); + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs new file mode 100644 index 00000000000..60d499127ec --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -0,0 +1,8952 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Collections.Concurrent; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Net; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using CsCheck; +#if NET10_0_OR_GREATER +using Microsoft.Accordant; +#endif +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.MembershipService; +using Orleans.Runtime.Scheduler; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +[TestCategory("BVT"), TestCategory("Dissemination")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Dissemination")] +public class DisseminationProtocolTests +{ + [Fact] + public async Task PublishPropagatesCancellationBeforeQueueing() + { + var local = CreateSilo(39101); + var peer = CreateSilo(39102); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var protocol = CreateProtocol(transport, ns); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + try + { + var exception = await Assert.ThrowsAnyAsync( + async () => await protocol.Publish(ns, FakeNamespace.DefaultKey, 1, cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Empty(transport.BroadcastBatches); + Assert.Equal(0, transport.GetTargetResolutionCount(peer)); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public void PushBroadcastUsesRequestResponseSemantics() + { + var method = typeof(IDisseminationSystemTarget).GetMethod(nameof(IDisseminationSystemTarget.PushBroadcast)); + + Assert.NotNull(method); + Assert.False(method.IsDefined(typeof(Orleans.Concurrency.OneWayAttribute), inherit: false)); + Assert.Equal("017AA907", method.GetCustomAttribute()?.Alias); + Assert.Equal(typeof(Task), method.ReturnType); + + method = typeof(IDisseminationSystemTarget).GetMethod(nameof(IDisseminationSystemTarget.ExchangeAntiEntropy)); + Assert.NotNull(method); + Assert.Equal("EF58CB3D", method.GetCustomAttribute()?.Alias); + } + + [Fact] + public async Task PublishSendsBroadcastToDeterministicTreeChildren() + { + var local = CreateSilo(11111); + var peers = Enumerable.Range(11112, 6).Select(CreateSilo).ToArray(); + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 2; + }); + var item = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1); + + var result = await PublishValue(protocol, ns, item, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.True(result); + var expectedChildren = GetOriginatorTreeTargets(local, peers, fanout: 2); + Assert.Equal(expectedChildren.OrderBy(static peer => peer), transport.BroadcastBatches.Select(batch => batch.Peer).OrderBy(static peer => peer)); + Assert.All(transport.BroadcastBatches, batch => Assert.Equal(item, GetBroadcastValues(batch.Batch).Single().Value)); + } + + [Fact] + public async Task PublishQueuesTreeBroadcastWithoutCapabilityProbing() + { + var local = CreateSilo(11111); + var peers = Enumerable.Range(11112, 6).Select(CreateSilo).ToArray(); + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 2; + }); + var item = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1); + + var result = await PublishValue(protocol, ns, item, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.True(result); + Assert.Equal( + GetOriginatorTreeTargets(local, peers, fanout: 2).OrderBy(static peer => peer), + transport.BroadcastBatches.Select(batch => batch.Peer).OrderBy(static peer => peer)); + } + + [Fact] + public async Task BroadcastSendsHonorMaxConcurrentSends() + { + const int maxConcurrentSends = 2; + var local = CreateSilo(11111); + var peers = Enumerable.Range(11112, 6).Select(CreateSilo).ToArray(); + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + var gate = new object(); + var limitReached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSends = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var inFlight = 0; + var started = 0; + var observedMax = 0; + var sentPeers = new List(); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + lock (gate) + { + inFlight++; + started++; + observedMax = Math.Max(observedMax, inFlight); + if (started == maxConcurrentSends) + { + limitReached.TrySetResult(true); + } + } + + try + { + await releaseSends.Task.WaitAsync(cancellationToken); + lock (gate) + { + sentPeers.Add(target); + } + } + finally + { + lock (gate) + { + inFlight--; + } + } + }; + + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = maxConcurrentSends; + options.Overlay.FanOutFactor = static _ => 10; + }); + var item = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1); + + Assert.True(await PublishValue(protocol, ns, item, TestContext.Current.CancellationToken)); + var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + try + { + await limitReached.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await Task.Delay(TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + lock (gate) + { + Assert.Equal(maxConcurrentSends, started); + Assert.Equal(maxConcurrentSends, inFlight); + Assert.Equal(maxConcurrentSends, observedMax); + } + } + finally + { + releaseSends.TrySetResult(true); + } + + await flushTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + lock (gate) + { + Assert.Equal(peers.OrderBy(static peer => peer), sentPeers.OrderBy(static peer => peer)); + Assert.True(observedMax <= maxConcurrentSends); + } + } + + [Fact] + public async Task PublishRejectsInvalidValuesBeforeQueueing() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + ns.SetValue("obsolete", version: 10); + + ns.Options.MaxPayloadBytes = sizeof(long); + var oversized = new DisseminationValue( + "oversized", + fromVersion: 0, + toVersion: 1, + new byte[sizeof(long) + 1]); + var obsolete = ns.CreateValue("obsolete", sequence: 5); + + Assert.False(await PublishValue(protocol, ns, oversized, TestContext.Current.CancellationToken)); + Assert.False(await protocol.Publish(ns, "obsolete", version: 11, TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task PublishReturnsFalseWhenRootIsMissingFromMembership() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + transport.PeerStatuses[local] = SiloStatus.Dead; + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + var item = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1); + + var result = await PublishValue(protocol, ns, item, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.False(result); + Assert.Equal(1, transport.RefreshMembershipCallCount); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task PublishContinuesAfterPeerSendFailure() + { + var local = CreateSilo(11111); + var joining = CreateSilo(11112); + var active = CreateSilo(11113); + var transport = new FakeTransport(local, joining, active); + transport.PeerStatuses[joining] = SiloStatus.Joining; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + if (Equals(target, joining)) + { + throw new InvalidOperationException("joining peer is not yet reachable"); + } + + transport.BroadcastBatches.Add((target, batch)); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 2; + }); + var value = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1); + + var result = await PublishValue(protocol, ns, value, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.True(result); + Assert.Equal(new[] { active }, transport.BroadcastBatches.Select(batch => batch.Peer)); + } + + [Fact] + public async Task NewPublicationWakesFailedPeerWithoutWaitingForBackoff() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sendCount = 0; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + if (Interlocked.Increment(ref sendCount) == 1) + { + throw new InvalidOperationException("transient send failure"); + } + + transport.BroadcastBatches.Add((target, batch)); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + var firstResult = await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var secondResult = await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.True(firstResult); + Assert.True(secondResult); + Assert.Equal(2, sendCount); + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(2, GetBroadcastValues(batch.Batch).Single().Value.ToVersion); + } + + [Fact] + public async Task SendFailureRetriesAutomaticallyWithBoundedBackoff() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var firstAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + if (Interlocked.Increment(ref sendCount) == 1) + { + firstAttempt.TrySetResult(); + throw new InvalidOperationException("transient send failure"); + } + + transport.BroadcastBatches.Add((target, batch)); + secondAttempt.TrySetResult(); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(4), + timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + using var schedule = new BroadcastScheduleObserver(); + timeProvider.Advance(TimeSpan.FromSeconds(1)); + await firstAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await schedule.WaitAsync( + e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + timeProvider.Advance(TimeSpan.FromSeconds(1)); + await secondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(2, sendCount); + Assert.Single(transport.BroadcastBatches); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task NewNotificationResetsRetryBackoff() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var firstAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var thirdAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + switch (Interlocked.Increment(ref sendCount)) + { + case 1: + firstAttempt.TrySetResult(); + throw new InvalidOperationException("first transient send failure"); + case 2: + secondAttempt.TrySetResult(); + throw new InvalidOperationException("second transient send failure"); + default: + transport.BroadcastBatches.Add((target, batch)); + thirdAttempt.TrySetResult(); + return Task.CompletedTask; + } + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(4), + timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + using var schedule = new BroadcastScheduleObserver(); + timeProvider.Advance(TimeSpan.FromSeconds(1)); + await firstAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await schedule.WaitAsync( + e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry && e.Attempt == 1, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + timeProvider.Advance(TimeSpan.FromSeconds(1)); + await secondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await schedule.WaitAsync( + e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry && e.Attempt == 2, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), + TestContext.Current.CancellationToken)); + timeProvider.Advance(TimeSpan.FromMilliseconds(999)); + Assert.False(thirdAttempt.Task.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + await thirdAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(3, sendCount); + Assert.Equal(2, Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)).Value.ToVersion); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task HighPriorityNamespaceFlushesWithoutWaitingForCoalescingWindow() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + sent.TrySetResult(batch); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + ns.Options.Priority = DisseminationPriority.High; + // A long window would delay a normal namespace; a high-priority namespace must ignore it. + ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + using var schedule = new BroadcastScheduleObserver(); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + + var scheduled = await schedule.WaitAsync( + e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Priority, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(TimeSpan.Zero, scheduled.DueTime); + + // Advancing far less than the coalescing window still delivers the update immediately. + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + var batch = await sent.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task HighPriorityRetryIsCappedByAntiEntropyInterval() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var firstAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + if (Interlocked.Increment(ref sendCount) == 1) + { + firstAttempt.TrySetResult(); + throw new InvalidOperationException("transient send failure"); + } + + transport.BroadcastBatches.Add((target, batch)); + secondAttempt.TrySetResult(); + return Task.CompletedTask; + }; + + var normalNamespace = new FakeNamespace(local, new DisseminationNamespace("normal")); + normalNamespace.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + var highNamespace = new FakeNamespace(local, new DisseminationNamespace("high")); + highNamespace.Options.Priority = DisseminationPriority.High; + var protocol = CreateProtocol( + transport, + new IDisseminationNamespace[] { normalNamespace, highNamespace }, + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(1), + timeProvider); + using var schedule = new BroadcastScheduleObserver(); + + Assert.True(await PublishValue( + protocol, + highNamespace, + highNamespace.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + await firstAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var retry = await schedule.WaitAsync( + e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(TimeSpan.FromSeconds(1), retry.DueTime); + timeProvider.Advance(TimeSpan.FromMilliseconds(999)); + Assert.False(secondAttempt.Task.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + await secondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task HighPriorityNotificationPullsPendingCoalescedFlushForward() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + sent.TrySetResult(); + return Task.CompletedTask; + }; + + var normalNamespace = new FakeNamespace(local, new DisseminationNamespace("normal")); + normalNamespace.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + var highNamespace = new FakeNamespace(local, new DisseminationNamespace("high")); + highNamespace.Options.Priority = DisseminationPriority.High; + var protocol = CreateProtocol( + transport, + new IDisseminationNamespace[] { normalNamespace, highNamespace }, + timeProvider: timeProvider); + using var schedule = new BroadcastScheduleObserver(); + + // A normal update on the peer pump arms a distant coalescing timer. + Assert.True(await PublishValue( + protocol, + normalNamespace, + normalNamespace.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + var coalesced = await schedule.WaitAsync( + e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Coalesce, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(TimeSpan.FromMinutes(1), coalesced.DueTime); + Assert.False(sent.Task.IsCompleted); + + // A high-priority update on the same pump must pull that flush forward to now. + Assert.True(await PublishValue( + protocol, + highNamespace, + highNamespace.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + var pulled = await schedule.WaitAsync( + e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Priority, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(TimeSpan.Zero, pulled.DueTime); + + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + await sent.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task NormalPriorityNamespaceStillWaitsForCoalescingWindow() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + sent.TrySetResult(); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + Assert.Equal(DisseminationPriority.Normal, ns.Options.Priority); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + using var schedule = new BroadcastScheduleObserver(); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + var scheduled = await schedule.WaitAsync( + e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Coalesce, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(TimeSpan.FromSeconds(1), scheduled.DueTime); + + // A sub-window advance must not release the coalesced batch. + timeProvider.Advance(TimeSpan.FromMilliseconds(999)); + Assert.False(sent.Task.IsCompleted); + + // Crossing the window flushes it. + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + await sent.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task HighPriorityNamespaceValuesAreSentAheadOfNormalPriority() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + transport.SendBroadcastResponseHandler = (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + var acknowledgments = new Dictionary>(); + foreach (var (namespaceName, values) in batch.Values) + { + acknowledgments[namespaceName] = + [.. values.Select(static value => new DigestEntry(value.Value.Key, value.Value.ToVersion))]; + } + + return Task.FromResult(new DisseminationBroadcastResponse { Acknowledgments = acknowledgments }); + }; + + var normalNamespace = new FakeNamespace(local, new DisseminationNamespace("normal")); + normalNamespace.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + var highNamespace = new FakeNamespace(local, new DisseminationNamespace("high")); + highNamespace.Options.Priority = DisseminationPriority.High; + DisseminationOptions? optionsRef = null; + var protocol = CreateProtocol( + transport, + new IDisseminationNamespace[] { normalNamespace, highNamespace }, + options => + { + optionsRef = options; + options.MaxBatchItems = 10; + }, + timeProvider); + + Assert.True(await PublishValue( + protocol, + normalNamespace, + normalNamespace.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + Assert.True(await PublishValue( + protocol, + highNamespace, + highNamespace.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + + // One value per batch, so the recorded send sequence reflects the drain order. + optionsRef!.MaxBatchItems = 1; + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal( + new[] { highNamespace.Name, normalNamespace.Name }, + transport.BroadcastBatches.Select(batch => Assert.Single(batch.Batch.Values.Keys))); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerVersionAdvancesOnlyFromExplicitAcknowledgment() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var secondAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastResponseHandler = (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + var acknowledgedVersion = Interlocked.Increment(ref sendCount) == 1 ? 0 : 1; + if (acknowledgedVersion == 1) + { + secondAttempt.TrySetResult(); + } + + return Task.FromResult(new DisseminationBroadcastResponse + { + Acknowledgments = new() + { + [FakeNamespace.DefaultName] = + [ + new DigestEntry(FakeNamespace.DefaultKey, acknowledgedVersion), + ], + }, + }); + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + using var schedule = new BroadcastScheduleObserver(); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var retry = await schedule.WaitAsync( + e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(1, sendCount); + + timeProvider.Advance(retry.DueTime); + await secondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(2, sendCount); + Assert.Equal( + new[] { 1L, 1L }, + transport.BroadcastBatches.Select(batch => Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion)); + } + + [Fact] + public async Task UnsupportedNamespaceResponseStopsAutomaticRetries() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sendCount = 0; + transport.SendBroadcastResponseHandler = (target, batch, cancellationToken) => + { + Interlocked.Increment(ref sendCount); + return Task.FromResult(new DisseminationBroadcastResponse + { + UnsupportedNamespaces = [FakeNamespace.DefaultName], + }); + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + timeProvider.Advance(TimeSpan.FromMinutes(1)); + await Task.Delay(TimeSpan.FromMilliseconds(50), TestContext.Current.CancellationToken); + + Assert.Equal(1, sendCount); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task UnsupportedNamespaceResponseCompletesQueuedFlushWaiter() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = async (target, batch, cancellationToken) => + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task.WaitAsync(cancellationToken); + return new DisseminationBroadcastResponse + { + UnsupportedNamespaces = [FakeNamespace.DefaultName], + }; + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + timeProvider.Advance(ns.Options.MaxCoalescingDelay); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), + TestContext.Current.CancellationToken)); + var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.False(flushTask.IsCompleted); + + releaseFirstSend.TrySetResult(); + await flushTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task RemovedPendingKeyIsDroppedWithoutRetry() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sendCount = 0; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + Interlocked.Increment(ref sendCount); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue("removed", sequence: 1), + TestContext.Current.CancellationToken)); + ns.RemoveValue("removed"); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var repairRequestCount = ns.RepairRequestCount; + timeProvider.Advance(TimeSpan.FromMinutes(1)); + await Task.Delay(TimeSpan.FromMilliseconds(50), TestContext.Current.CancellationToken); + + Assert.Equal(0, sendCount); + Assert.Equal(repairRequestCount, ns.RepairRequestCount); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task BroadcastBatchingStopsPeerFlushAfterSendFailure() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new AutoAdvancingTimeProvider(TimeSpan.FromSeconds(2)); + var sendCount = 0; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + if (Interlocked.Increment(ref sendCount) == 1) + { + throw new InvalidOperationException("transient send failure"); + } + + transport.BroadcastBatches.Add((target, batch)); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + DisseminationOptions? optionsRef = null; + var protocol = CreateProtocol(transport, ns, options => + { + optionsRef = options; + options.MaxBatchItems = 10; + }, timeProvider); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("first", sequence: 1), TestContext.Current.CancellationToken)); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("second", sequence: 1), TestContext.Current.CancellationToken)); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("third", sequence: 1), TestContext.Current.CancellationToken)); + optionsRef!.MaxBatchItems = 1; + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(1, sendCount); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task OrderedBroadcastBatchingRetriesFailedChainBeforeNewerValues() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var attempts = new List(); + var failed = false; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + var value = Assert.Single(GetBroadcastValues(batch)); + attempts.Add(value.Value.ToVersion); + if (value.Value.ToVersion == 2 && !failed) + { + failed = true; + throw new InvalidOperationException("transient send failure"); + } + + transport.BroadcastBatches.Add((target, batch)); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local) { ReturnRepairChain = true }; + ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + DisseminationOptions? optionsRef = null; + var protocol = CreateProtocol(transport, ns, options => + { + optionsRef = options; + options.MaxBatchItems = 10; + }); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken)); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2, fromVersion: 1), TestContext.Current.CancellationToken)); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 3, fromVersion: 2), TestContext.Current.CancellationToken)); + optionsRef!.MaxBatchItems = 1; + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(new long[] { 1, 2 }, attempts); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 4, fromVersion: 3), TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(new long[] { 1, 2, 2, 3, 4 }, attempts); + } + + [Fact] + public async Task BroadcastPeerFailureDoesNotBlockOtherPeerPumps() + { + var local = CreateSilo(11111); + var failedPeer = CreateSilo(11112); + var healthyPeer = CreateSilo(11113); + var transport = new FakeTransport(local, failedPeer, healthyPeer); + var timeProvider = new FakeTimeProvider(); + var failedPeerAttempts = 0; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + if (Equals(target, failedPeer) && Interlocked.Increment(ref failedPeerAttempts) == 1) + { + throw new InvalidOperationException("transient peer failure"); + } + + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Add((target, batch)); + } + + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.Overlay.FanOutFactor = static _ => 10; + }, timeProvider); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("first", sequence: 1), TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(1, failedPeerAttempts); + Assert.Equal(new[] { healthyPeer }, GetSentBroadcastPeers(transport)); + + ClearBroadcastBatches(transport); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("second", sequence: 2), TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(2, failedPeerAttempts); + Assert.Equal(new[] { failedPeer, healthyPeer }.OrderBy(static peer => peer), GetSentBroadcastPeers(transport).OrderBy(static peer => peer)); + } + + [Fact] + public async Task MembershipRefreshDropsPendingBroadcastForRemovedPeers() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("before-removal", sequence: 1), TestContext.Current.CancellationToken)); + + transport.Peers.Remove(peer); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("during-removal", sequence: 2), TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task StopDrainsPendingBroadcastPumps() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("before-stop", sequence: 1), TestContext.Current.CancellationToken)); + await protocol.StopAsync(TestContext.Current.CancellationToken); + + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(peer, batch.Peer); + Assert.Equal(new DisseminationKey("before-stop"), Assert.Single(GetBroadcastValues(batch.Batch)).Value.Key); + } + + [Fact] + public async Task FlushPendingBroadcastWaitsForInFlightFlush() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + sendStarted.TrySetResult(); + await releaseSend.Task.WaitAsync(cancellationToken); + transport.BroadcastBatches.Add((target, batch)); + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1, timeProvider); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("in-flight", sequence: 1), TestContext.Current.CancellationToken)); + timeProvider.Advance(ns.Options.MaxCoalescingDelay); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + try + { + Assert.False(flushTask.IsCompleted); + } + finally + { + releaseSend.TrySetResult(); + } + + await flushTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Single(transport.BroadcastBatches); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task NotificationDuringInFlightSendRepairsFromAcknowledgedVersion() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastResponseHandler = async (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + if (Interlocked.Increment(ref sendCount) == 1) + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task.WaitAsync(cancellationToken); + } + + return FakeTransport.CreateAcknowledgment(batch); + }; + + var ns = new FakeNamespace(local) { ReturnRepairChain = true }; + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + timeProvider.Advance(ns.Options.MaxCoalescingDelay); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2, fromVersion: 1), + TestContext.Current.CancellationToken)); + releaseFirstSend.TrySetResult(); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal( + new[] { (From: 0L, To: 1L), (From: 1L, To: 2L) }, + transport.BroadcastBatches.Select(batch => + { + var value = Assert.Single(GetBroadcastValues(batch.Batch)); + return (value.Value.FromVersion, value.Value.ToVersion); + })); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task MembershipRefreshCompletesFlushWaitersForRemovedPeers() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + sendStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1, timeProvider); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("in-flight", sequence: 1), TestContext.Current.CancellationToken)); + timeProvider.Advance(ns.Options.MaxCoalescingDelay); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("pending", sequence: 1), TestContext.Current.CancellationToken)); + var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.False(flushTask.IsCompleted); + + transport.Peers.Remove(peer); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("after-removal", sequence: 1), TestContext.Current.CancellationToken)); + + await flushTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task ReceiveBroadcastForwardsOnlyToLocalTreeChildren() + { + var silos = Enumerable.Range(11111, 8).Select(CreateSilo).OrderBy(static silo => silo).ToArray(); + var root = silos[0]; + var local = silos[1]; + var peers = silos.Where(silo => !Equals(silo, local)).ToArray(); + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 2; + }); + var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 1); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(root, item), TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var expectedChildren = GetForwardingTreeTargets(local, root, peers, fanout: 2, sender: root); + Assert.Equal(expectedChildren.OrderBy(static peer => peer), transport.BroadcastBatches.Select(batch => batch.Peer).OrderBy(static peer => peer)); + } + + [Fact] + public async Task ReceiveBroadcastAppliesAllValuesBeforeForwarding() + { + var root = CreateSilo(11111); + var local = CreateSilo(11112); + var sender = CreateSilo(11113); + var peer = CreateSilo(11114); + var transport = new FakeTransport(local, sender, peer); + DisseminationKey firstKey = new("first"); + DisseminationKey secondKey = new("second"); + var ns = new FakeNamespace(local); + var forwardingObserved = false; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + forwardingObserved = true; + Assert.Equal(1, ns.GetVersion(firstKey)); + Assert.Equal(2, ns.GetVersion(secondKey)); + transport.BroadcastBatches.Add((target, batch)); + return Task.CompletedTask; + }; + + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 2); + var first = ns.CreateItem(root, firstKey, sequence: 1); + var second = ns.CreateItem(root, secondKey, sequence: 2); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(sender, first, second), TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.True(forwardingObserved); + Assert.Equal(1, ns.GetVersion(firstKey)); + Assert.Equal(2, ns.GetVersion(secondKey)); + } + + [Fact] + public async Task ReceiveBroadcastContinuesAfterFailedValue() + { + var root = CreateSilo(11111); + var local = CreateSilo(11112); + var child = CreateSilo(11113); + var transport = new FakeTransport(local, root, child); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1); + DisseminationKey firstKey = new("first"); + DisseminationKey failedKey = new("failed"); + DisseminationKey secondKey = new("second"); + var first = ns.CreateItem(root, firstKey, sequence: 1); + var failed = CreateDisseminationValue( + root, + new DisseminationValue(failedKey, fromVersion: 0, toVersion: 1, Array.Empty())); + var second = ns.CreateItem(root, secondKey, sequence: 1); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(root, first, failed, second), TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(1, ns.GetVersion(firstKey)); + Assert.Equal(0, ns.GetVersion(failedKey)); + Assert.Equal(1, ns.GetVersion(secondKey)); + var forwarded = Assert.Single(transport.BroadcastBatches); + Assert.Equal(child, forwarded.Peer); + Assert.Equal( + new[] { firstKey, secondKey }.OrderBy(static key => key), + GetBroadcastValues(forwarded.Batch).Select(static item => item.Value.Key).OrderBy(static key => key)); + } + + [Fact] + public async Task ReceiveBroadcastDoesNotRefreshMembershipForRemovedOriginators() + { + var local = CreateSilo(11111); + var sender = CreateSilo(11112); + var firstOriginator = CreateSilo(11120); + var secondOriginator = CreateSilo(11121); + var transport = new FakeTransport(local, sender); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + var first = ns.CreateItem(firstOriginator, "first", sequence: 1); + var second = ns.CreateItem(secondOriginator, "second", sequence: 1); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(sender, first, second), TestContext.Current.CancellationToken); + + Assert.Equal(0, transport.RefreshMembershipCallCount); + Assert.Equal(1, ns.GetVersion("first")); + Assert.Equal(1, ns.GetVersion("second")); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task DuplicateBroadcastDoesNotForwardAgain() + { + var silos = Enumerable.Range(11111, 8).Select(CreateSilo).OrderBy(static silo => silo).ToArray(); + var root = silos[0]; + var local = silos[1]; + var peers = silos.Where(silo => !Equals(silo, local)).ToArray(); + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 2; + }); + var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 1); + var batch = CreateBroadcastBatch(root, item); + + await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var expectedChildren = GetForwardingTreeTargets(local, root, peers, fanout: 2, sender: root); + Assert.Equal(expectedChildren.Count, transport.BroadcastBatches.Count); + Assert.Equal(1, ns.ApplyCounts[item.Value.Key]); + } + + [Fact] + public async Task DuplicateBroadcastSchedulesForwardingWhenNoPriorQueueStateExists() + { + var silos = Enumerable.Range(11111, 8).Select(CreateSilo).OrderBy(static silo => silo).ToArray(); + var sender = silos[0]; + var local = silos[1]; + var peers = silos.Where(silo => !Equals(silo, local)).ToArray(); + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 2); + var item = ns.CreateItem(sender, FakeNamespace.DefaultKey, sequence: 1); + + await protocol.ReceiveBroadcast( + CreateBroadcastBatch(sender, item), + TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var expectedChildren = GetForwardingTreeTargets(local, sender, peers, fanout: 2, sender: sender); + Assert.Equal( + expectedChildren.OrderBy(static peer => peer), + transport.BroadcastBatches.Select(static batch => batch.Peer).OrderBy(static peer => peer)); + Assert.False(ns.ApplyCounts.ContainsKey(FakeNamespace.DefaultKey)); + } + + [Fact] + public async Task ReceiveBroadcastAppliesFullValueOverOlderLocalVersion() + { + var root = CreateSilo(11111); + var local = CreateSilo(11112); + var transport = new FakeTransport(local, root); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 3, fromVersion: 0); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(root, item), TestContext.Current.CancellationToken); + + Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(1, ns.ApplyCounts[FakeNamespace.DefaultKey]); + } + + [Fact] + public async Task ReceiveBroadcastAppliesDeltaValueOnlyWhenFromVersionMatches() + { + var root = CreateSilo(11111); + var local = CreateSilo(11112); + var transport = new FakeTransport(local, root); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + ns.SetValue(FakeNamespace.DefaultKey, version: 2); + var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 3, fromVersion: 2); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(root, item), TestContext.Current.CancellationToken); + + Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(1, ns.ApplyCounts[FakeNamespace.DefaultKey]); + } + + [Fact] + public async Task ReceiveBroadcastRejectsDeltaValueWhenFromVersionIsMissing() + { + var root = CreateSilo(11111); + var local = CreateSilo(11112); + var transport = new FakeTransport(local, root); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 3, fromVersion: 2); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(root, item), TestContext.Current.CancellationToken); + + Assert.Equal(1, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.False(ns.ApplyCounts.ContainsKey(FakeNamespace.DefaultKey)); + } + + [Fact] + public async Task ReceiveBroadcastAcknowledgesCurrentVersionAfterRejectedRepair() + { + var sender = CreateSilo(11111); + var local = CreateSilo(11112); + var transport = new FakeTransport(local, sender); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var item = ns.CreateItem( + sender, + FakeNamespace.DefaultKey, + sequence: 3, + fromVersion: 2); + + var response = await protocol.ReceiveBroadcast( + CreateBroadcastBatch(sender, item), + TestContext.Current.CancellationToken); + + var acknowledgment = Assert.Single(response.Acknowledgments[ns.Name]); + Assert.Equal(FakeNamespace.DefaultKey, acknowledgment.Key); + Assert.Equal(1, acknowledgment.Version); + } + + [Fact] + public async Task ReceiveBroadcastDoesNotRefreshMembershipForMissingRoot() + { + var root = CreateSilo(11111); + var local = CreateSilo(11112); + var sender = CreateSilo(11113); + var peer = CreateSilo(11114); + var transport = new FakeTransport(local, sender, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 2; + }); + var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 1); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(sender, item), TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(1, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(0, transport.RefreshMembershipCallCount); + Assert.Equal(new[] { peer }, transport.BroadcastBatches.Select(static batch => batch.Peer)); + } + + [Fact] + public async Task ReceiveBroadcastRoutesWithoutRefreshingForMissingRoot() + { + var local = CreateSilo(11111); + var sender = CreateSilo(11112); + var peer = CreateSilo(11113); + var root = CreateSilo(11120); + var transport = new FakeTransport(local, sender, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 2); + var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 1); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(sender, item), TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(1, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(0, transport.RefreshMembershipCallCount); + Assert.Equal( + GetForwardingTreeTargets(local, sender, new[] { sender, peer }, fanout: 2, sender: sender), + transport.BroadcastBatches.Select(static batch => batch.Peer)); + } + + [Fact] + public async Task TreeRoutingInvalidatesCachedTopologyWhenActivePeersChange() + { + var local = CreateSilo(11115); + var initialPeers = Enumerable.Range(11112, 3).Select(CreateSilo).ToList(); + var transport = new FakeTransport(local, initialPeers.ToArray()); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 2; + }); + var item = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1); + + var initialResult = await PublishValue(protocol, ns, item, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var initialChildren = GetOriginatorTreeTargets(local, transport.Peers, fanout: 2); + + foreach (var peer in Enumerable.Range(11116, 8).Select(CreateSilo)) + { + transport.Peers.Add(peer); + var updatedChildren = GetOriginatorTreeTargets(local, transport.Peers, fanout: 2); + if (!initialChildren.SequenceEqual(updatedChildren)) + { + transport.BroadcastBatches.Clear(); + var updatedItem = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2); + + var updatedResult = await PublishValue(protocol, ns, updatedItem, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.True(initialResult); + Assert.True(updatedResult); + Assert.Equal(updatedChildren.OrderBy(static peer => peer), transport.BroadcastBatches.Select(batch => batch.Peer).OrderBy(static peer => peer)); + return; + } + } + + throw new InvalidOperationException("The test did not find a peer set which changes the local tree children."); + } + + [Fact] + public async Task BroadcastBatchingCoalescesSamePeerValuesByLatestVersion() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + + var first = await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken); + var second = await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.True(first); + Assert.True(second); + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(peer, batch.Peer); + var value = Assert.Single(GetBroadcastValues(batch.Batch)); + Assert.Equal(2, value.Value.ToVersion); + } + + [Fact] + public async Task BroadcastRepairsFromAcknowledgedPeerVersion() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local) { ReturnRepairChain = true }; + var protocol = CreateProtocol(transport, ns); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2, fromVersion: 1), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal( + new[] { (From: 0L, To: 1L), (From: 1L, To: 2L) }, + transport.BroadcastBatches.Select(batch => + { + var value = Assert.Single(GetBroadcastValues(batch.Batch)); + return (value.Value.FromVersion, value.Value.ToVersion); + })); + } + + [Fact] + public async Task BroadcastBatchingPreservesOrderedSameKeyValues() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local) { ReturnRepairChain = true }; + var protocol = CreateProtocol(transport, ns); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken)); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2, fromVersion: 1), TestContext.Current.CancellationToken)); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 3, fromVersion: 2), TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal( + new[] { (From: 0L, To: 1L), (From: 1L, To: 2L), (From: 2L, To: 3L) }, + GetBroadcastValues(batch.Batch).Select(static value => (value.Value.FromVersion, value.Value.ToVersion))); + } + + [Fact] + public async Task BroadcastBatchingLetsFullValueSupersedeOrderedValues() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local) { ReturnRepairChain = true }; + var protocol = CreateProtocol(transport, ns); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2, fromVersion: 1), TestContext.Current.CancellationToken)); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 3, fromVersion: 2), TestContext.Current.CancellationToken)); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 4), TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var value = Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)); + Assert.Equal(0, value.Value.FromVersion); + Assert.Equal(4, value.Value.ToVersion); + } + + [Fact] + public async Task BroadcastPeerSenderCachesSystemTarget() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("first", sequence: 1), TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("second", sequence: 2), TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(2, transport.BroadcastBatches.Count); + Assert.Equal(1, transport.GetTargetResolutionCount(peer)); + } + + [Fact] + public async Task BroadcastBatchingWakesScheduledFlushWhenBatchLimitIsReached() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + sent.TrySetResult(true); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + var protocol = CreateProtocol(transport, ns, options => options.MaxBatchItems = 2); + + var first = await PublishValue(protocol, ns, ns.CreateValue("first", sequence: 1), TestContext.Current.CancellationToken); + await Task.Delay(TimeSpan.FromMilliseconds(50), TestContext.Current.CancellationToken); + var second = await PublishValue(protocol, ns, ns.CreateValue("second", sequence: 1), TestContext.Current.CancellationToken); + + await sent.Task.WaitAsync(TimeSpan.FromSeconds(2), TestContext.Current.CancellationToken); + Assert.True(first); + Assert.True(second); + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(new DisseminationKey[] { "first", "second" }, GetBroadcastValues(batch.Batch).Select(static value => value.Value.Key)); + } + + [Fact] + public async Task BroadcastBatchingUsesShortestConfiguredCoalescingDelay() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + sent.TrySetResult(); + return Task.CompletedTask; + }; + + var slowNamespace = new FakeNamespace(local, "slow"); + slowNamespace.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + var fastNamespace = new FakeNamespace(local, "fast"); + fastNamespace.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var protocol = CreateProtocol( + transport, + new IDisseminationNamespace[] { slowNamespace, fastNamespace }, + options => options.Overlay.FanOutFactor = static _ => 1, + timeProvider); + + Assert.True(await PublishValue(protocol, slowNamespace, slowNamespace.CreateValue("slow", sequence: 1), TestContext.Current.CancellationToken)); + Assert.True(await PublishValue(protocol, fastNamespace, fastNamespace.CreateValue("fast", sequence: 1), TestContext.Current.CancellationToken)); + + timeProvider.Advance(TimeSpan.FromMilliseconds(999)); + Assert.False(sent.Task.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + await sent.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(2, batch.Batch.Values.Count); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public void FixedTreeRoutingSatisfiesReachabilityAndFanoutInvariants() + { + Gen.Select(Gen.Int[1, 64], Gen.Int[1, 8], Gen.Int[0, 63], static (count, fanout, rootSeed) => + { + var rootIndex = rootSeed % count; + return (Count: count, Fanout: fanout, RootIndex: rootIndex); + }).Sample(testCase => + { + var silos = CreateSilos(testCase.Count); + var root = silos[testCase.RootIndex]; + var options = new DisseminationOverlayOptions + { + FanOutFactor = _ => testCase.Fanout, + }; + var members = silos.ToImmutableArray(); + var snapshots = silos.ToDictionary( + static silo => silo, + silo => new DisseminationMembershipSnapshot( + new MembershipVersion(1), + silo, + members, + options)); + var directTargets = snapshots[root].OriginatorTreeTargets; + + Assert.DoesNotContain(root, directTargets); + Assert.Equal(directTargets.Length, directTargets.Distinct().Count()); + Assert.True(directTargets.Length <= Math.Min((testCase.Fanout * 2), Math.Max(0, testCase.Count - 1))); + + var reached = GetReachedParticipants(root, snapshots); + Assert.Equal(silos.OrderBy(static silo => silo), reached.OrderBy(static silo => silo)); + }); + } + + [Fact] + public void AntiEntropyPeerSelectionSamplesDistinctMembers() + { + Gen.Select(Gen.Int[2, 64], Gen.Int[0, 63], Gen.Int[1, 64], static (count, localSeed, peerCount) => + { + var localIndex = localSeed % count; + return (Count: count, LocalIndex: localIndex, PeerCount: peerCount); + }).Sample(testCase => + { + var silos = CreateSilos(testCase.Count); + var local = silos[testCase.LocalIndex]; + var transport = new FakeTransport(local, silos.Where(silo => !Equals(silo, local)).ToArray()); + var ns = new FakeNamespace(local); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + var protocol = CreateProtocol(transport, ns, options => + { + options.Overlay.AntiEntropyPeerCount = testCase.PeerCount; + }); + + protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken).GetAwaiter().GetResult(); + var peers = transport.AntiEntropyRequests.Select(static request => request.Peer).ToArray(); + + Assert.Equal(Math.Min(testCase.PeerCount, testCase.Count - 1), peers.Length); + Assert.Equal(peers.Length, peers.Distinct().Count()); + Assert.DoesNotContain(local, peers); + Assert.All(peers, peer => Assert.Contains(peer, silos)); + }); + } + + [Fact] + public async Task AntiEntropyResponseIncludesNewerLocalValues() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + ns.SetValue(FakeNamespace.DefaultKey, version: 5); + + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = CreateAntiEntropyRequestDigest( + ns.Name, + (FakeNamespace.DefaultKey, 3)), + }, TestContext.Current.CancellationToken); + + var item = Assert.Single(GetAntiEntropyResponseValues(response)); + Assert.Equal(5, item.Value.ToVersion); + Assert.False(response.Truncated); + } + + [Fact] + public async Task AntiEntropyResponseOnlyIncludesRequestedDigestKeys() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + ns.SetValue("requested", version: 5); + ns.SetValue("omitted", version: 5); + + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = CreateAntiEntropyRequestDigest( + ns.Name, + ("requested", 3)), + }, TestContext.Current.CancellationToken); + + var item = Assert.Single(GetAntiEntropyResponseValues(response)); + Assert.Equal(new DisseminationKey("requested"), item.Value.Key); + } + + [Fact] + public async Task OversizedAntiEntropyKeyDoesNotStarveLaterRepairs() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.Options.MaxPayloadBytes = sizeof(long); + ns.PublishValue(new DisseminationValue( + "oversized", + fromVersion: 0, + toVersion: 1, + new byte[sizeof(long) + 1])); + ns.SetValue("valid", version: 2); + var protocol = CreateProtocol(transport, ns); + + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = CreateAntiEntropyRequestDigest( + ns.Name, + ("oversized", 0), + ("valid", 0)), + }, TestContext.Current.CancellationToken); + + var value = Assert.Single(GetAntiEntropyResponseValues(response)); + Assert.Equal(new DisseminationKey("valid"), value.Value.Key); + Assert.False(response.Truncated); + } + + [Fact] + public async Task AntiEntropyDigestRequestsAreSuppressedUntilExpectedCadenceExpires() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var timeProvider = new TestTimeProvider(); + ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(2); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + await protocol.ReceiveBroadcast( + CreateBroadcastBatch(peer, ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 1)), + TestContext.Current.CancellationToken); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + var capabilityRequest = Assert.Single(transport.AntiEntropyRequests).Request; + Assert.Empty(capabilityRequest.Digests); + Assert.Equal([ns.Name], capabilityRequest.SupportedNamespaces); + transport.AntiEntropyRequests.Clear(); + + timeProvider.Advance(TimeSpan.FromSeconds(2) - TimeSpan.FromMilliseconds(1)); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + capabilityRequest = Assert.Single(transport.AntiEntropyRequests).Request; + Assert.Empty(capabilityRequest.Digests); + Assert.Equal([ns.Name], capabilityRequest.SupportedNamespaces); + transport.AntiEntropyRequests.Clear(); + + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + var request = Assert.Single(transport.AntiEntropyRequests).Request; + var digest = Assert.Single(request.Digests[ns.Name]); + Assert.Equal(FakeNamespace.DefaultKey, digest.Key); + } + + [Fact] + public async Task AntiEntropyCapabilityExchangeConfirmsNamespaceWithoutRepairDigests() + { + var local = CreateSilo(11113); + var peer = CreateSilo(11114); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + transport.ExchangeAntiEntropyHandler = (target, request, _) => + { + Assert.Empty(request.Digests); + Assert.Equal([ns.Name], request.SupportedNamespaces); + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + SupportedNamespaces = [ns.Name], + }); + }; + var protocol = CreateProtocol(transport, ns); + + Assert.Equal([peer], protocol.GetUnconfirmedPeers(ns)); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Empty(protocol.GetUnconfirmedPeers(ns)); + } + + [Fact] + public async Task AntiEntropyCapabilityOnlyRoundUsesNamespaceStaleItemTtl() + { + var local = CreateSilo(11115); + var peer = CreateSilo(11116); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var ns = new FakeNamespace(local); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + var exchangeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cancellationObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.ExchangeAntiEntropyHandler = async (target, request, cancellationToken) => + { + Assert.Empty(request.Digests); + Assert.Equal([ns.Name], request.SupportedNamespaces); + exchangeStarted.TrySetResult(); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + finally + { + cancellationObserved.TrySetResult(); + } + + return new DisseminationAntiEntropyResponse { Sender = target }; + }; + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + var round = protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + await exchangeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + timeProvider.Advance(ns.Options.StaleItemTtl); + + await cancellationObserved.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + [Fact] + public async Task DuplicateBroadcastDoesNotPostponeAntiEntropyProbe() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var timeProvider = new TestTimeProvider(); + ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(2); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + var batch = CreateBroadcastBatch(peer, ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 1)); + + await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + timeProvider.Advance(TimeSpan.FromSeconds(2) - TimeSpan.FromMilliseconds(1)); + await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + var request = Assert.Single(transport.AntiEntropyRequests).Request; + Assert.Single(request.Digests[ns.Name]); + } + + [Fact] + public async Task AntiEntropySendsOneDigestRequestPerPeer() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + DisseminationKey firstKey = new("first"); + DisseminationKey secondKey = new("second"); + var firstNamespace = new FakeNamespace(local, new DisseminationNamespace("first-namespace")); + firstNamespace.ExpectedKeys.Add(firstKey); + var secondNamespace = new FakeNamespace(local, new DisseminationNamespace("second-namespace")); + secondNamespace.ExpectedKeys.Add(secondKey); + var protocol = CreateProtocol(transport, new IDisseminationNamespace[] { firstNamespace, secondNamespace }, options => + { + options.Overlay.AntiEntropyPeerCount = 1; + }); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + var request = Assert.Single(transport.AntiEntropyRequests).Request; + Assert.Equal( + new[] { firstNamespace.Name, secondNamespace.Name }.OrderBy(static name => name), + request.Digests.Keys.OrderBy(static name => name)); + var firstDigest = Assert.Single(request.Digests[firstNamespace.Name]); + Assert.Equal(firstKey, firstDigest.Key); + Assert.Equal(0, firstDigest.Version); + var secondDigest = Assert.Single(request.Digests[secondNamespace.Name]); + Assert.Equal(secondKey, secondDigest.Key); + Assert.Equal(0, secondDigest.Version); + } + + [Fact] + public async Task AntiEntropyExchangeFailureDoesNotBackOffPeer() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + var exchangeCount = 0; + transport.ExchangeAntiEntropyHandler = (target, request, _) => + { + if (Interlocked.Increment(ref exchangeCount) == 1) + { + throw new InvalidOperationException("transient anti-entropy failure"); + } + + return ValueTask.FromResult(new DisseminationAntiEntropyResponse { Sender = target }); + }; + + var protocol = CreateProtocol(transport, ns, options => + { + options.Overlay.AntiEntropyPeerCount = 1; + }); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal(2, Volatile.Read(ref exchangeCount)); + Assert.Equal(2, transport.AntiEntropyRequests.Count); + } + + [Fact] + public async Task AntiEntropyExchangePropagatesCancellation() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + var exchangeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var cancellation = new CancellationTokenSource(); + transport.ExchangeAntiEntropyHandler = async (target, request, cancellationToken) => + { + exchangeStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return new DisseminationAntiEntropyResponse { Sender = target }; + }; + + var protocol = CreateProtocol(transport, ns, options => options.Overlay.AntiEntropyPeerCount = 1); + var exchangeTask = protocol.RunAntiEntropyRound(cancellation.Token); + await exchangeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + async () => await exchangeTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task AntiEntropyResponseHonorsCancellation() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + async () => await protocol.ReceiveAntiEntropy( + new DisseminationAntiEntropyRequest { Sender = peer }, + cancellation.Token)); + } + + [Fact] + public async Task AntiEntropyExchangesAreNotLimitedByMaxConcurrentSends() + { + var local = CreateSilo(11111); + var peers = Enumerable.Range(11112, 3).Select(CreateSilo).ToArray(); + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + var gate = new object(); + var allStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseExchanges = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var inFlight = 0; + var started = 0; + var observedMax = 0; + transport.ExchangeAntiEntropyHandler = async (target, request, _) => + { + lock (gate) + { + inFlight++; + started++; + observedMax = Math.Max(observedMax, inFlight); + if (started == peers.Length) + { + allStarted.TrySetResult(true); + } + } + + try + { + await releaseExchanges.Task; + return new DisseminationAntiEntropyResponse { Sender = target }; + } + finally + { + lock (gate) + { + inFlight--; + } + } + }; + + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.AntiEntropyPeerCount = peers.Length; + }); + + var exchangeTask = protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + try + { + await allStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + lock (gate) + { + Assert.Equal(peers.Length, observedMax); + } + } + finally + { + releaseExchanges.TrySetResult(true); + } + + await exchangeTask; + + Assert.Equal(peers.Length, transport.AntiEntropyRequests.Count); + } + + [Fact] + public async Task AntiEntropyAppliesCompletedResponsesWhenSlowPeerExceedsRoundLifetime() + { + var local = CreateSilo(11111); + var fastPeer = CreateSilo(11112); + var slowPeer = CreateSilo(11113); + var transport = new FakeTransport(local, fastPeer, slowPeer); + var timeProvider = new FakeTimeProvider(); + var ns = new FakeNamespace(local); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + ns.ApplyObserved = new(TaskCreationOptions.RunContinuationsAsynchronously); + var fastResponseReturned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var slowExchangeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var slowCancellationObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var timeoutMetric = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == DisseminationInstruments.MeterName + && instrument.Name == DisseminationInstruments.AntiEntropyFailuresName) + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + if (tags.ToArray().Any(static tag => tag.Key == "reason" && object.Equals(tag.Value, "timeout"))) + { + timeoutMetric.TrySetResult(measurement); + } + }); + listener.Start(); + var repair = ns.CreateItem(fastPeer, FakeNamespace.DefaultKey, sequence: 1); + transport.ExchangeAntiEntropyHandler = async (target, _, cancellationToken) => + { + if (target.Equals(slowPeer)) + { + slowExchangeStarted.TrySetResult(); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + finally + { + slowCancellationObserved.TrySetResult(); + } + } + + fastResponseReturned.TrySetResult(); + return new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(repair), + }; + }; + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyPeerCount = 2, + timeProvider); + + var round = protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + await Task.WhenAll(fastResponseReturned.Task, slowExchangeStarted.Task) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(0, ns.GetVersion(FakeNamespace.DefaultKey)); + + timeProvider.Advance(ns.Options.StaleItemTtl); + await slowCancellationObserved.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await ns.ApplyObserved.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal( + 1, + await timeoutMetric.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(1, ns.GetVersion(FakeNamespace.DefaultKey)); + } + + [Fact] + public async Task AntiEntropyRanksStaggeredResponsesWhichCompleteWithinRoundLifetime() + { + var local = CreateSilo(11111); + var fastPeer = CreateSilo(11112); + var slowerPeer = CreateSilo(11113); + var transport = new FakeTransport(local, fastPeer, slowerPeer); + var ns = new FakeNamespace(local); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var fastResponseReturned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var slowExchangeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSlowResponse = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.ExchangeAntiEntropyHandler = async (target, request, cancellationToken) => + { + var requestedVersion = Assert.Single(request.Digests[ns.Name]).Version; + var responseVersion = target.Equals(fastPeer) ? 2 : 3; + if (target.Equals(slowerPeer)) + { + slowExchangeStarted.TrySetResult(); + await releaseSlowResponse.Task.WaitAsync(cancellationToken); + } + else + { + fastResponseReturned.TrySetResult(); + } + + return new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(ns.CreateItem( + target, + FakeNamespace.DefaultKey, + sequence: responseVersion, + fromVersion: requestedVersion)), + }; + }; + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyPeerCount = 2); + + var round = protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + await Task.WhenAll(fastResponseReturned.Task, slowExchangeStarted.Task) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, ns.GetVersion(FakeNamespace.DefaultKey)); + + releaseSlowResponse.TrySetResult(); + await round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); + } + + [Fact] + public async Task AntiEntropyAppliesReturnedRepairItemsWithoutForwarding() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + var repairItem = ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 7); + transport.ExchangeAntiEntropyHandler = (target, request, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(repairItem), + }); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal(7, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Empty(transport.BroadcastBatches); + Assert.Single(transport.AntiEntropyRequests); + var digestByNamespace = Assert.Single(transport.AntiEntropyRequests[0].Request.Digests); + Assert.Equal(ns.Name, digestByNamespace.Key); + var digest = Assert.Single(digestByNamespace.Value); + Assert.Equal(FakeNamespace.DefaultKey, digest.Key); + Assert.Equal(0, digest.Version); + } + + [Fact] + public async Task AntiEntropyAppliesRepairChainInOrder() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var protocol = CreateProtocol(transport, ns); + var second = ns.CreateItem( + peer, + FakeNamespace.DefaultKey, + sequence: 2, + fromVersion: 1); + var third = ns.CreateItem( + peer, + FakeNamespace.DefaultKey, + sequence: 3, + fromVersion: 2); + transport.ExchangeAntiEntropyHandler = (target, request, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(second, third), + }); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(2, ns.ApplyCounts[FakeNamespace.DefaultKey]); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task AntiEntropyAppliesNewestRepairFirstForEachStream() + { + var local = CreateSilo(11111); + var peers = new[] { CreateSilo(11112), CreateSilo(11113) }; + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var exchangeCount = 0; + transport.ExchangeAntiEntropyHandler = (target, request, _) => + { + var requestedVersion = Assert.Single(request.Digests[ns.Name]).Version; + var responseVersion = Interlocked.Increment(ref exchangeCount) == 1 ? 2 : 3; + var repair = ns.CreateItem( + target, + FakeNamespace.DefaultKey, + sequence: responseVersion, + fromVersion: requestedVersion); + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(repair), + }); + }; + + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyPeerCount = peers.Length); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(peers.Length, exchangeCount); + } + + [Fact] + public async Task AntiEntropyResponseOmitsNamespacesWithoutRepairs() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.SetValue(FakeNamespace.DefaultKey, version: 5); + var protocol = CreateProtocol(transport, ns); + + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = CreateAntiEntropyRequestDigest( + ns.Name, + (FakeNamespace.DefaultKey, 5)), + }, TestContext.Current.CancellationToken); + + Assert.Empty(response.Values); + } + + [Fact] + public async Task AntiEntropyTruncationRotatesPastContinuouslyAdvancingKey() + { + var local = CreateSilo(11111); + var member = CreateSilo(11112); + var requester = CreateSilo(11113); + var transport = new FakeTransport(local, member); + var ns = new FakeNamespace(local); + DisseminationKey hotKey = "hot"; + DisseminationKey waitingKey = "waiting"; + ns.SetValue(hotKey, version: 1); + ns.SetValue(waitingKey, version: 1); + var protocol = CreateProtocol( + transport, + ns, + options => options.MaxBatchItems = 1); + var request = new DisseminationAntiEntropyRequest + { + Sender = requester, + Digests = CreateAntiEntropyRequestDigest( + ns.Name, + (hotKey, 0), + (waitingKey, 0)), + }; + + var first = await protocol.ReceiveAntiEntropy(request, TestContext.Current.CancellationToken); + await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = member, + Digests = request.Digests, + }, TestContext.Current.CancellationToken); + ns.SetValue(hotKey, version: 2); + var second = await protocol.ReceiveAntiEntropy(request, TestContext.Current.CancellationToken); + + Assert.True(first.Truncated); + Assert.True(second.Truncated); + Assert.Equal(hotKey, Assert.Single(GetAntiEntropyResponseValues(first)).Value.Key); + Assert.Equal(waitingKey, Assert.Single(GetAntiEntropyResponseValues(second)).Value.Key); + } + + [Fact] + public async Task AntiEntropyAppliesValidItemsAfterFailedRepairItem() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var badRepairItem = new DisseminationBroadcastValue + { + Value = new DisseminationValue("bad", fromVersion: 0, toVersion: 1, Array.Empty()), + TimeToLive = TimeSpan.FromMinutes(1), + }; + var goodRepairItem = ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 3); + transport.ExchangeAntiEntropyHandler = (target, request, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(badRepairItem, goodRepairItem), + }); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); + } + + [Fact] + public async Task AntiEntropyAppliesValidItemsAfterFailedRepairRound() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + + var badRepairItem = new DisseminationBroadcastValue + { + Value = new DisseminationValue(FakeNamespace.DefaultKey, fromVersion: 1, toVersion: 2, Array.Empty()), + TimeToLive = TimeSpan.FromMinutes(1), + }; + var goodRepairItem = ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 3); + var exchangeCount = 0; + transport.ExchangeAntiEntropyHandler = (target, request, _) => + { + var count = Interlocked.Increment(ref exchangeCount); + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = count switch + { + 1 => CreateValueGroups(badRepairItem), + 2 => CreateValueGroups(goodRepairItem), + _ => [], + }, + }); + }; + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(2, Volatile.Read(ref exchangeCount)); + } + +#if NET10_0_OR_GREATER + [Fact] + public async Task MonotonicDisseminationModelConformsToAccordantSpec() + { + var spec = CreateMonotonicDisseminationSpec(); + spec.ExecuteWith() + .BindAsync("Receive", static (harness, version) => harness.Receive(version, TestContext.Current.CancellationToken)) + .BindAsync("RepairPeer", static (harness, peerVersion) => harness.RepairPeer(peerVersion, TestContext.Current.CancellationToken)); + + var receive = spec.GetOperation("Receive"); + var repairPeer = spec.GetOperation("RepairPeer"); + var inputs = new InputSet + { + receive.With(1, "Receive version 1"), + receive.With(2, "Receive version 2"), + receive.With(3, "Receive version 3"), + repairPeer.With(0, "Repair peer with no value"), + repairPeer.With(1, "Repair peer at version 1"), + repairPeer.With(3, "Repair peer at version 3"), + }; + var initialState = new MonotonicDisseminationState(); + var testCases = spec.GenerateTests(initialState, inputs, new TestGenerationOptions { MaxDepth = 4 }); + var harness = new MonotonicDisseminationHarness(CreateSilo(11111)); + var context = spec.CreateTestingContext(); + context.Register(harness); + + var results = await spec.RunTests( + context, + initialState, + testCases, + new TestExecutionOptions + { + BeforeEachAsync = testContext => + { + testContext.Context.Get().Reset(); + return Task.CompletedTask; + }, + }); + + var failures = results.Where(static result => !result.Success).ToArray(); + Assert.Empty(failures); + } +#endif + + [Fact] + public async Task MembershipNamespaceReturnsAndAppliesDiffWhenPeerVersionIsRetained() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var baseSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + var updatedSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceManager = new FakeMembershipManager(baseSnapshot); + var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); + var peerDigest = Assert.Single(sourceNamespace.Digests); + sourceManager.CurrentSnapshot = updatedSnapshot; + var localDigest = Assert.Single(sourceNamespace.Digests); + + var repair = sourceNamespace.CreateRepair(new DisseminationRepairRequest( + localDigest.Key, + peerDigest.Version, + toVersion: null, + maxItemCount: 1, + maxBatchBytes: 1024 * 1024, + maxPayloadBytes: 1024 * 1024)); + Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); + var value = Assert.Single(repair.Values); + Assert.Equal(peerDigest.Version, value.FromVersion); + Assert.Equal(localDigest.Version, value.ToVersion); + var update = Assert.IsType( + serializer.Deserialize(value.Payload)); + Assert.NotNull(update.Diff); + Assert.Null(update.Snapshot); + var receiverManager = new FakeMembershipManager(baseSnapshot); + var receiverNamespace = CreateMembershipNamespace(receiverManager, serializer); + var result = await receiverNamespace.ApplyValueAsync(value, TestContext.Current.CancellationToken); + + Assert.Equal(DisseminationApplyResult.Applied, result); + Assert.Equal(updatedSnapshot.Version, receiverManager.CurrentSnapshot.Version); + Assert.True(receiverManager.CurrentSnapshot.Entries.ContainsKey(peer)); + } + + [Fact] + public async Task MembershipNamespaceAppliesSameVersionDiffWhenLivenessAdvances() + { + var local = CreateSilo(11111); + var receiverSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2))); + var updatedEntry = CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(3)); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var update = new MembershipTableSnapshotUpdate + { + Diff = new MembershipTableSnapshotDiff( + new MembershipVersion(1), + new MembershipVersion(2), + [updatedEntry], + []), + }; + var value = new DisseminationValue( + DisseminationKey.Default, + fromVersion: 1, + toVersion: 2, + serializer.SerializeToArray(update)); + var receiverManager = new FakeMembershipManager(receiverSnapshot); + var receiverNamespace = CreateMembershipNamespace(receiverManager, serializer); + + var result = await receiverNamespace.ApplyValueAsync(value, TestContext.Current.CancellationToken); + + Assert.Equal(DisseminationApplyResult.Applied, result); + Assert.Equal( + DateTime.UnixEpoch.AddSeconds(3), + receiverManager.CurrentSnapshot.Entries[local].IAmAliveTime); + } + + [Fact] + public void MembershipNamespaceInvalidatesCachedPayloadForSameVersionUpdate() + { + var local = CreateSilo(11111); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(1))); + var updatedSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var manager = new FakeMembershipManager(firstSnapshot); + var disseminationNamespace = CreateMembershipNamespace(manager, serializer); + var request = new DisseminationRepairRequest( + DisseminationKey.Default, + fromVersion: null, + toVersion: null, + maxItemCount: 1, + maxBatchBytes: 1024 * 1024, + maxPayloadBytes: 1024 * 1024); + + var firstRepair = disseminationNamespace.CreateRepair(request); + manager.CurrentSnapshot = updatedSnapshot; + var updatedRepair = disseminationNamespace.CreateRepair(request); + + var firstValue = Assert.Single(firstRepair.Values); + var updatedValue = Assert.Single(updatedRepair.Values); + var firstUpdate = Assert.IsType( + serializer.Deserialize(firstValue.Payload)); + var updatedUpdate = Assert.IsType( + serializer.Deserialize(updatedValue.Payload)); + Assert.Equal( + DateTime.UnixEpoch.AddSeconds(1), + firstUpdate.Snapshot!.Entries[local].IAmAliveTime); + Assert.Equal( + DateTime.UnixEpoch.AddSeconds(2), + updatedUpdate.Snapshot!.Entries[local].IAmAliveTime); + } + + [Fact] + public async Task MembershipNamespaceRetainsPostApplySnapshotForRepair() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var currentSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + var incomingSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + var appliedSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(1)), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var manager = new FakeMembershipManager(currentSnapshot); + manager.ProcessGossipSnapshotHandler = (_, _) => + { + manager.CurrentSnapshot = appliedSnapshot; + return Task.FromResult(true); + }; + var disseminationNamespace = CreateMembershipNamespace(manager, serializer); + var value = new DisseminationValue( + DisseminationKey.Default, + fromVersion: 0, + toVersion: incomingSnapshot.Version.Value, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = incomingSnapshot })); + + var result = await disseminationNamespace.ApplyValueAsync(value, TestContext.Current.CancellationToken); + var repair = disseminationNamespace.CreateRepair(new DisseminationRepairRequest( + DisseminationKey.Default, + fromVersion: null, + toVersion: appliedSnapshot.Version.Value, + maxItemCount: 1, + maxBatchBytes: 1024 * 1024, + maxPayloadBytes: 1024 * 1024)); + + Assert.Equal(DisseminationApplyResult.Applied, result); + var repairedValue = Assert.Single(repair.Values); + var repairedUpdate = Assert.IsType( + serializer.Deserialize(repairedValue.Payload)); + var repairedSnapshot = Assert.IsType(repairedUpdate.Snapshot); + Assert.Equal(appliedSnapshot.Version, repairedSnapshot.Version); + Assert.True(repairedSnapshot.Entries.ContainsKey(peer)); + Assert.Equal( + DateTime.UnixEpoch.AddSeconds(1), + repairedSnapshot.Entries[local].IAmAliveTime); + } + + [Fact] + public async Task MembershipNamespacePublishesSameVersionSuccessorAsFullSnapshot() + { + var local = CreateSilo(11111); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(1))); + var updatedSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceManager = new FakeMembershipManager(firstSnapshot); + var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); + var disseminationService = new FakeDisseminationService(); + + Assert.True(await sourceNamespace.PublishAsync( + disseminationService, + firstSnapshot, + TestContext.Current.CancellationToken)); + sourceManager.CurrentSnapshot = updatedSnapshot; + Assert.True(await sourceNamespace.PublishAsync( + disseminationService, + updatedSnapshot, + TestContext.Current.CancellationToken)); + + var update = Assert.Single(disseminationService.Values.Skip(1)); + Assert.Equal((0L, 1L), (update.FromVersion, update.ToVersion)); + var receiverManager = new FakeMembershipManager(firstSnapshot); + var receiverNamespace = CreateMembershipNamespace(receiverManager, serializer); + Assert.Equal( + DisseminationApplyResult.Applied, + await receiverNamespace.ApplyValueAsync(update, TestContext.Current.CancellationToken)); + Assert.Equal( + DateTime.UnixEpoch.AddSeconds(2), + receiverManager.CurrentSnapshot.Entries[local].IAmAliveTime); + } + + [Fact] + public async Task MembershipAntiEntropyRepairsSameVersionSuccessor() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(1))); + var updatedSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceNamespace = CreateMembershipNamespace( + new FakeMembershipManager(updatedSnapshot), + serializer); + var receiverManager = new FakeMembershipManager(firstSnapshot); + var receiverNamespace = CreateMembershipNamespace(receiverManager, serializer); + var peerDigest = Assert.Single(receiverNamespace.Digests); + var transport = new FakeTransport(local, peer); + var protocol = CreateProtocol( + transport, + new IDisseminationNamespace[] { sourceNamespace }); + + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = new() + { + [sourceNamespace.Name] = [peerDigest], + }, + }, TestContext.Current.CancellationToken); + + var value = Assert.Single(GetAntiEntropyResponseValues(response)).Value; + Assert.Equal((0L, 1L), (value.FromVersion, value.ToVersion)); + Assert.Equal( + DisseminationApplyResult.Applied, + await receiverNamespace.ApplyValueAsync(value, TestContext.Current.CancellationToken)); + Assert.Equal( + DateTime.UnixEpoch.AddSeconds(2), + receiverManager.CurrentSnapshot.Entries[local].IAmAliveTime); + } + + [Fact] + public async Task MembershipNamespacePublishesOrderedDiffChain() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + var secondSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2)), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + var thirdSnapshot = CreateMembershipSnapshot( + version: 3, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2)), + CreateMembershipEntry( + peer, + SiloStatus.Active, + DateTime.UnixEpoch.AddSeconds(1), + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(3))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceManager = new FakeMembershipManager(firstSnapshot); + var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); + var disseminationService = new FakeDisseminationService(); + + Assert.True(await sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken)); + sourceManager.CurrentSnapshot = secondSnapshot; + Assert.True(await sourceNamespace.PublishAsync(disseminationService, secondSnapshot, TestContext.Current.CancellationToken)); + sourceManager.CurrentSnapshot = thirdSnapshot; + Assert.True(await sourceNamespace.PublishAsync(disseminationService, thirdSnapshot, TestContext.Current.CancellationToken)); + var (first, second, third) = ( + disseminationService.Values[0], + disseminationService.Values[1], + disseminationService.Values[2]); + + Assert.Equal((0L, 1L), (first.FromVersion, first.ToVersion)); + Assert.Equal((1L, 2L), (second.FromVersion, second.ToVersion)); + Assert.Equal((2L, 3L), (third.FromVersion, third.ToVersion)); + Assert.NotNull(Assert.IsType( + serializer.Deserialize(first.Payload)).Snapshot); + Assert.NotNull(Assert.IsType( + serializer.Deserialize(second.Payload)).Diff); + Assert.NotNull(Assert.IsType( + serializer.Deserialize(third.Payload)).Diff); + + var receiverManager = new FakeMembershipManager(CreateMembershipSnapshot(MembershipVersion.MinValue.Value)); + var receiverNamespace = CreateMembershipNamespace(receiverManager, serializer); + Assert.Equal(DisseminationApplyResult.Applied, await receiverNamespace.ApplyValueAsync(first, TestContext.Current.CancellationToken)); + Assert.Equal(DisseminationApplyResult.Applied, await receiverNamespace.ApplyValueAsync(second, TestContext.Current.CancellationToken)); + Assert.Equal(DisseminationApplyResult.Applied, await receiverNamespace.ApplyValueAsync(third, TestContext.Current.CancellationToken)); + Assert.Equal(thirdSnapshot.Version, receiverManager.CurrentSnapshot.Version); + Assert.Equal(SiloStatus.Active, receiverManager.CurrentSnapshot.Entries[peer].Status); + } + + [Fact] + public async Task MembershipNamespacePublishesFullSnapshotAfterFailedBaseline() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + var secondSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2)), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceManager = new FakeMembershipManager(firstSnapshot); + var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); + var disseminationService = new FakeDisseminationService(); + disseminationService.Results.Enqueue(false); + disseminationService.Results.Enqueue(true); + + Assert.False(await sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken)); + sourceManager.CurrentSnapshot = secondSnapshot; + Assert.True(await sourceNamespace.PublishAsync(disseminationService, secondSnapshot, TestContext.Current.CancellationToken)); + + Assert.Equal(0, disseminationService.Values[1].FromVersion); + Assert.NotNull(Assert.IsType( + serializer.Deserialize(disseminationService.Values[1].Payload)).Snapshot); + } + + [Fact] + public async Task MembershipNamespacePublishesDiffWhenTopologyChanges() + { + var local = CreateSilo(11111); + var firstPeer = CreateSilo(11112); + var secondPeer = CreateSilo(11113); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(firstPeer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + var secondSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(firstPeer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1)), + CreateMembershipEntry(secondPeer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(2))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceManager = new FakeMembershipManager(firstSnapshot); + var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); + var disseminationService = new FakeDisseminationService(); + + Assert.True(await sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken)); + sourceManager.CurrentSnapshot = secondSnapshot; + Assert.True(await sourceNamespace.PublishAsync(disseminationService, secondSnapshot, TestContext.Current.CancellationToken)); + + Assert.Equal(1, disseminationService.Values[1].FromVersion); + Assert.NotNull(Assert.IsType( + serializer.Deserialize(disseminationService.Values[1].Payload)).Diff); + } + + [Fact] + public async Task MembershipNamespacePublishesDirectDiffAcrossVersionGap() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + var thirdSnapshot = CreateMembershipSnapshot( + version: 3, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2)), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceManager = new FakeMembershipManager(firstSnapshot); + var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); + var disseminationService = new FakeDisseminationService(); + + Assert.True(await sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken)); + sourceManager.CurrentSnapshot = thirdSnapshot; + Assert.True(await sourceNamespace.PublishAsync(disseminationService, thirdSnapshot, TestContext.Current.CancellationToken)); + + Assert.Equal(1, disseminationService.Values[1].FromVersion); + Assert.NotNull(Assert.IsType( + serializer.Deserialize(disseminationService.Values[1].Payload)).Diff); + } + + [Fact] + public async Task MembershipNamespaceAllowsConcurrentPublicationNotifications() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + var secondSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2)), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceManager = new FakeMembershipManager(firstSnapshot); + var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); + var disseminationService = new FakeDisseminationService(); + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var callCount = 0; + disseminationService.PublishHandler = async (value, cancellationToken) => + { + if (Interlocked.Increment(ref callCount) == 1) + { + firstStarted.TrySetResult(); + await releaseFirst.Task.WaitAsync(cancellationToken); + } + + return true; + }; + + var firstPublish = sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken).AsTask(); + await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + sourceManager.CurrentSnapshot = secondSnapshot; + var secondPublish = sourceNamespace.PublishAsync(disseminationService, secondSnapshot, TestContext.Current.CancellationToken).AsTask(); + try + { + await Task.Delay(TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + Assert.Equal(2, disseminationService.Values.Count); + Assert.True(secondPublish.IsCompletedSuccessfully); + } + finally + { + releaseFirst.TrySetResult(); + } + + Assert.True(await firstPublish); + Assert.True(await secondPublish); + Assert.Equal(new long[] { 1, 2 }, disseminationService.Values.Select(static value => value.ToVersion).Order()); + } + + [Fact] + public void DisseminationKeyRoundTripsDefaultAndSiloAddressKeys() + { + var earlierPeer = CreateSilo(11111); + var peer = CreateSilo(11112); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var membershipValue = new DisseminationValue(DisseminationKey.Default, fromVersion: 0, toVersion: 7, Array.Empty()); + var deploymentLoadValue = new DisseminationValue(peer, fromVersion: 0, toVersion: 11, Array.Empty()); + + var roundTripMembershipValue = serializer.Deserialize(serializer.SerializeToArray(membershipValue)); + var roundTripDeploymentLoadValue = serializer.Deserialize(serializer.SerializeToArray(deploymentLoadValue)); + + Assert.Equal(DisseminationKey.Default, roundTripMembershipValue.Key); + Assert.Equal(new DisseminationKey(peer), roundTripDeploymentLoadValue.Key); + + Span expected = stackalloc char[128]; + Span actual = stackalloc char[128]; + Assert.True(((ISpanFormattable)peer).TryFormat(expected, out var expectedLength, "H", null)); + Assert.True(roundTripDeploymentLoadValue.Key.TryFormat(actual, out var actualLength, "H", null)); + Assert.Equal(expected[..expectedLength].ToString(), actual[..actualLength].ToString()); + Assert.True(roundTripMembershipValue.Key.TryFormat(Span.Empty, out var nullKeyLength, default, null)); + Assert.Equal(0, nullKeyLength); + Assert.Equal( + Math.Sign(earlierPeer.CompareTo(peer)), + Math.Sign(Comparer.Default.Compare(earlierPeer, peer))); + Assert.Equal( + Math.Sign(earlierPeer.CompareTo(peer)), + Math.Sign(new DisseminationKey(earlierPeer).CompareTo(new DisseminationKey(peer)))); + } + + [Fact] + public void DisseminationNamespaceWrapsNonEmptyStringsForDictionaryKeys() + { + DisseminationNamespace namespaceName = "load"; + var namespaces = new Dictionary + { + [namespaceName] = 1, + }; + + Assert.Equal("load", (string)namespaceName); + Assert.True(namespaces.TryGetValue(new DisseminationNamespace("load"), out var value)); + Assert.Equal(1, value); + Assert.True(namespaceName.CompareTo(new DisseminationNamespace("membership")) < 0); + Assert.ThrowsAny(() => new DisseminationNamespace(string.Empty)); + } + + [Fact] + public void DisseminationNamespaceDefaultValueSupportsKeyOperations() + { + var namespaceName = default(DisseminationNamespace); + var namespaces = new Dictionary { [namespaceName] = 1 }; + + Assert.True(namespaces.TryGetValue(default, out var value)); + Assert.Equal(1, value); + Assert.Equal(0, namespaceName.GetHashCode()); + Assert.Equal(0, namespaceName.CompareTo(default)); + Assert.True(namespaceName.CompareTo(new DisseminationNamespace("membership")) < 0); + Assert.Equal(string.Empty, namespaceName.ToString()); + Assert.True(namespaceName.TryFormat([], out var charsWritten, default, null)); + Assert.Equal(0, charsWritten); + Assert.Throws(() => namespaceName.Value); + } + + [Fact] + public void OptionsValidatorRejectsInvalidFanoutBounds() + { + var options = new DisseminationOptions(); + options.Overlay.MinFanOutFactor = 8; + options.Overlay.MaxFanOutFactor = 4; + var result = new DisseminationOptionsValidator().Validate(Options.DefaultName, options); + + Assert.True(result.Failed); + } + + [Fact] + public void NamespaceOptionsValidatorAllowsStaleTtlShorterThanCoalescingDelay() + { + var options = new DeploymentLoadPublisherOptions(); + options.Dissemination.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + options.Dissemination.StaleItemTtl = TimeSpan.FromMilliseconds(1); + + var result = new DeploymentLoadPublisherOptionsValidator().Validate(Options.DefaultName, options); + + Assert.Equal(ValidateOptionsResult.Success, result); + } + + [Fact] + public void NamespaceOptionsValidatorRejectsNonPositiveStaleTtl() + { + var options = new DeploymentLoadPublisherOptions(); + options.Dissemination.StaleItemTtl = TimeSpan.Zero; + + var result = new DeploymentLoadPublisherOptionsValidator().Validate(Options.DefaultName, options); + + Assert.Equal( + [ + $"{nameof(DeploymentLoadPublisherOptions)}.{nameof(DeploymentLoadPublisherOptions.Dissemination)}." + + $"{nameof(DisseminationNamespaceOptions.StaleItemTtl)} must be greater than 0.", + ], + result.Failures); + } + + [Fact] + public void ClusterMembershipNamespaceOptionsValidatorIncludesOwningOptionsType() + { + var options = new ClusterMembershipOptions(); + options.Dissemination.MaxPayloadBytes = 0; + + var result = new ClusterMembershipOptionsDisseminationValidator().Validate(Options.DefaultName, options); + + Assert.Equal( + [ + $"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.Dissemination)}." + + $"{nameof(DisseminationNamespaceOptions.MaxPayloadBytes)} must be greater than 0.", + ], + result.Failures); + } + + [Theory] + [InlineData(0)] + [InlineData(4294967295)] + public void OptionsValidatorRejectsUnsupportedAntiEntropyIntervals(long milliseconds) + { + var options = new DisseminationOptions(); + options.Overlay.AntiEntropyInterval = TimeSpan.FromMilliseconds(milliseconds); + var result = new DisseminationOptionsValidator().Validate(Options.DefaultName, options); + + Assert.True(result.Failed); + } + + [Theory] + [InlineData(0)] + [InlineData(4294967295)] + public void NamespaceOptionsValidatorRejectsUnsupportedCoalescingDelays(long milliseconds) + { + var options = new DeploymentLoadPublisherOptions(); + options.Dissemination.MaxCoalescingDelay = TimeSpan.FromMilliseconds(milliseconds); + options.Dissemination.StaleItemTtl = options.Dissemination.MaxCoalescingDelay + TimeSpan.FromSeconds(1); + var result = new DeploymentLoadPublisherOptionsValidator().Validate(Options.DefaultName, options); + + Assert.True(result.Failed); + } + + [Fact] + public async Task ReceiveBroadcastUsesRelativeLifetimeAcrossClockSkew() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var timeProvider = new FakeTimeProvider(DateTimeOffset.MaxValue - TimeSpan.FromSeconds(1)); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + var item = ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 1); + item = new DisseminationBroadcastValue + { + Value = item.Value, + TimeToLive = TimeSpan.FromMilliseconds(1), + }; + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(peer, item), TestContext.Current.CancellationToken); + + Assert.Equal(1, ns.GetVersion(FakeNamespace.DefaultKey)); + } + + [Fact] + public async Task ReceiveBroadcastDropsNonPositiveRelativeLifetime() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + var item = ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 1); + item = new DisseminationBroadcastValue + { + Value = item.Value, + TimeToLive = TimeSpan.Zero, + }; + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(peer, item), TestContext.Current.CancellationToken); + + Assert.Equal(0, ns.GetVersion(FakeNamespace.DefaultKey)); + } + + [Fact] + public async Task BroadcastTransportCancelsWhenRelativeLifetimeExpires() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cancellationObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = async (_, _, cancellationToken) => + { + sendStarted.TrySetResult(); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + finally + { + cancellationObserved.TrySetResult(); + } + + return new DisseminationBroadcastResponse(); + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + timeProvider.Advance(ns.Options.MaxCoalescingDelay); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + timeProvider.Advance(ns.Options.StaleItemTtl); + await cancellationObserved.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + ns.Options.Enabled = false; + await protocol.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + [Fact] + public void OptionsValidatorRejectsInvalidExpectedUpdateCadence() + { + var options = new DeploymentLoadPublisherOptions(); + options.Dissemination.ExpectedUpdateCadence = TimeSpan.Zero; + var result = new DeploymentLoadPublisherOptionsValidator().Validate(Options.DefaultName, options); + + Assert.True(result.Failed); + } + + [Fact] + public void NamespaceOptionsUseExpectedUpdateCadenceDefaults() + { + Assert.Equal(TimeSpan.FromSeconds(5), new DeploymentLoadPublisherOptions().Dissemination.ExpectedUpdateCadence); + Assert.Equal(TimeSpan.FromSeconds(10), new ClusterMembershipOptions().Dissemination.ExpectedUpdateCadence); + } + + [Fact] + public void DisseminationOptionsUseBatchDefaults() + { + var options = new DisseminationOptions(); + + Assert.Equal(1024 * 1024, options.MaxBatchBytes); + Assert.Equal(8 * 1024, options.MaxBatchItems); + Assert.Equal(options.MaxBatchBytes, new DisseminationNamespaceOptions().MaxPayloadBytes); + } + + [Fact] + public void DisseminationMembershipReturnsCachedSnapshotForSameMembershipVersion() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var membershipManager = new FakeMembershipManager(CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Joining, DateTime.UnixEpoch.AddSeconds(1)))); + var membership = new DisseminationMembership( + membershipManager, + new FakeLocalSiloDetails(local), + Options.Create(new DisseminationOptions())); + + var first = membership.CurrentSnapshot; + var second = membership.CurrentSnapshot; + + Assert.Same(first, second); + Assert.Equal(new MembershipVersion(1), first.MembershipVersion); + Assert.Equal(new[] { local, peer }, first.Members); + } + + [Fact] + public void DisseminationMembershipRecomputesSnapshotWhenMembershipVersionChanges() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var membershipManager = new FakeMembershipManager(CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch))); + var membership = new DisseminationMembership( + membershipManager, + new FakeLocalSiloDetails(local), + Options.Create(new DisseminationOptions())); + var first = membership.CurrentSnapshot; + + membershipManager.CurrentSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + + var second = membership.CurrentSnapshot; + + Assert.NotSame(first, second); + Assert.Equal(new MembershipVersion(2), second.MembershipVersion); + Assert.Equal(new[] { local, peer }, second.Members); + } + + [Fact] + public async Task DisseminationMembershipRefreshDelegatesToMembershipManager() + { + var local = CreateSilo(11111); + var membershipManager = new FakeMembershipManager(CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch))); + var membership = new DisseminationMembership( + membershipManager, + new FakeLocalSiloDetails(local), + Options.Create(new DisseminationOptions())); + + await membership.RefreshMembership(TestContext.Current.CancellationToken); + + Assert.Equal(1, membershipManager.RefreshCallCount); + Assert.Null(Assert.Single(membershipManager.RefreshTargetVersions)); + } + + [Fact] + public void DisseminationMembershipSnapshotRejectsDuplicateMembers() + { + var local = CreateSilo(11111); + + var exception = Assert.Throws(() => new DisseminationMembershipSnapshot( + new MembershipVersion(1), + local, + [local, local], + new DisseminationOverlayOptions())); + + Assert.Equal("members", exception.ParamName); + } + + [Fact] + public void DisseminationMembershipSnapshotUsesStoredTreeOrderForOriginatorTargets() + { + var root = CreateSilo(11113); + var first = CreateSilo(11115); + var second = CreateSilo(11112); + var snapshot = new DisseminationMembershipSnapshot( + new MembershipVersion(1), + root, + [root, first, second], + CreateOverlayOptions(fanout: 2)); + + var targets = snapshot.OriginatorTreeTargets; + + Assert.Equal(new[] { first, second }, targets); + } + + [Fact] + public void DisseminationMembershipSnapshotUsesStoredTreeOrderForForwardingTargets() + { + var local = CreateSilo(11111); + var root = CreateSilo(11112); + var sender = CreateSilo(11113); + var child = CreateSilo(11114); + var snapshot = new DisseminationMembershipSnapshot( + new MembershipVersion(1), + local, + [local, root, sender, child], + CreateOverlayOptions(fanout: 2)); + + var targets = snapshot.ForwardingTreeTargets; + + Assert.Equal(new[] { sender, child }, targets); + } + + [Fact] + public void DisseminationMembershipSnapshotDoesNotAddLocalSiloWhenMissing() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var snapshot = new DisseminationMembershipSnapshot( + new MembershipVersion(1), + local, + [peer], + CreateOverlayOptions(fanout: 1)); + + var targets = snapshot.OriginatorTreeTargets; + var antiEntropyPeers = snapshot.SelectAntiEntropyPeers(1); + + Assert.False(snapshot.ContainsMember(local)); + Assert.Single(snapshot.Members); + Assert.Empty(targets); + Assert.Empty(antiEntropyPeers); + } + + [Fact] + public void DisseminationMembershipSnapshotReturnsAllAntiEntropyCandidatesWhenRequestedCountIsLarge() + { + const int peerCount = 20; + const int localIndex = 7; + var silos = CreateSilos(12); + var local = silos[localIndex]; + var expected = silos.Where(silo => !Equals(silo, local)).OrderBy(static silo => silo); + var snapshot = new DisseminationMembershipSnapshot( + new MembershipVersion(1), + local, + [.. silos], + new DisseminationOverlayOptions()); + + var selectedPeers = snapshot.SelectAntiEntropyPeers(peerCount); + + Assert.Equal(silos.Length - 1, selectedPeers.Length); + Assert.Equal(selectedPeers.Length, selectedPeers.Distinct().Count()); + Assert.DoesNotContain(local, selectedPeers); + Assert.Equal(expected, selectedPeers.OrderBy(static silo => silo)); + } + + [Fact] + public async Task DisabledNamespaceBetweenPublishAndReschedule_UsesFiniteBoundedFallbackWithoutInfiniteTimerDueTime() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new RecordingFakeTimeProvider(); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFailedSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastHandler = async (_, _, _) => + { + Interlocked.Increment(ref sendCount); + sendStarted.TrySetResult(); + await releaseFailedSend.Task; + throw new InvalidOperationException("The initial send fails after the namespace is disabled."); + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(4), + timeProvider); + using var schedule = new BroadcastScheduleObserver(); + var retryScheduled = schedule.WaitAsync( + e => e.Peer.Equals(peer) + && e.Reason == DisseminationBroadcastScheduleReason.Retry + && e.Attempt == 1, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + timeProvider.Advance(TimeSpan.FromSeconds(1)); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + ns.Options.Enabled = false; + releaseFailedSend.TrySetResult(); + + var retry = await retryScheduled; + Assert.Equal(TimeSpan.FromMilliseconds(100), retry.DueTime); + Assert.DoesNotContain(TimeSpan.MaxValue, timeProvider.TimerDueTimes); + + timeProvider.Advance(retry.DueTime); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await protocol.StopAsync(TestContext.Current.CancellationToken); + + Assert.Equal(1, sendCount); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task UnexpectedPeerPumpIterationFailure_IsRetriedAndCompletesFlushAndStopDrainWaiters() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new RecordingFakeTimeProvider(); + var logger = new RecordingLogger(); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var retrySendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseRetrySend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastResponseHandler = async (_, batch, _) => + { + if (Interlocked.Increment(ref sendCount) == 1) + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task; + return new DisseminationBroadcastResponse + { + Acknowledgments = new() + { + [FakeNamespace.DefaultName] = + [ + new DigestEntry(FakeNamespace.DefaultKey, version: 0), + ], + }, + }; + } + + retrySendStarted.TrySetResult(); + await releaseRetrySend.Task; + transport.BroadcastBatches.Add((peer, batch)); + return FakeTransport.CreateAcknowledgment(batch); + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var protocol = CreateProtocolWithBroadcastLogger( + transport, + ns, + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(4), + timeProvider, + logger); + using var schedule = new BroadcastScheduleObserver(); + var recoveredRetryScheduled = schedule.WaitAsync( + e => e.Peer.Equals(peer) + && e.Reason == DisseminationBroadcastScheduleReason.Retry + && e.Attempt == 2, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + timeProvider.Advance(TimeSpan.FromSeconds(1)); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.False(flushTask.IsCompleted); + timeProvider.ThrowOnNextTimerChange(); + releaseFirstSend.TrySetResult(); + + await flushTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var retry = await recoveredRetryScheduled; + Assert.Equal(TimeSpan.FromSeconds(2), retry.DueTime); + Assert.Equal(1, logger.WarningCount); + Assert.DoesNotContain(TimeSpan.MaxValue, timeProvider.TimerDueTimes); + + timeProvider.Advance(retry.DueTime); + await retrySendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var stopTask = protocol.StopAsync(TestContext.Current.CancellationToken); + Assert.False(stopTask.IsCompleted); + releaseRetrySend.TrySetResult(); + + await stopTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(2, sendCount); + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(1, Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion); + } + + [Fact] + public async Task PermanentPeerPumpRecoveryFailure_FaultsFlushAndStopDrainWaiters() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new RecordingFakeTimeProvider(); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = async (_, _, _) => + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task; + return new DisseminationBroadcastResponse + { + Acknowledgments = new() + { + [FakeNamespace.DefaultName] = + [ + new DigestEntry(FakeNamespace.DefaultKey, version: 0), + ], + }, + }; + }; + + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(4), + timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + timeProvider.Advance(TimeSpan.FromSeconds(1)); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + timeProvider.ThrowOnNextTimerChanges(2); + releaseFirstSend.TrySetResult(); + + var flushException = await Assert.ThrowsAsync(() => flushTask); + Assert.Contains("could not recover", flushException.Message, StringComparison.Ordinal); + + var stopException = await Assert.ThrowsAsync( + () => protocol.StopAsync(TestContext.Current.CancellationToken)); + Assert.Same(flushException, stopException); + } + + private static DisseminationProtocol CreateProtocol( + FakeTransport transport, + FakeNamespace ns, + Action? configure = null, + TimeProvider? timeProvider = null) => + CreateProtocol(transport, new IDisseminationNamespace[] { ns }, configure, timeProvider); + + private static ValueTask PublishValue( + DisseminationProtocol protocol, + FakeNamespace disseminationNamespace, + DisseminationValue value, + CancellationToken cancellationToken) + { + disseminationNamespace.PublishValue(value); + return protocol.Publish( + disseminationNamespace, + value.Key, + value.ToVersion, + cancellationToken); + } + + private static DisseminationProtocol CreateProtocol( + FakeTransport transport, + IReadOnlyList namespaces, + Action? configure = null, + TimeProvider? timeProvider = null) + { + var options = new DisseminationOptions { Enabled = true }; + configure?.Invoke(options); + var localSiloDetails = new FakeLocalSiloDetails(transport.LocalSilo); + return new DisseminationProtocol( + localSiloDetails, + transport.GrainFactory, + new DisseminationMembership( + transport.MembershipManager, + localSiloDetails, + Options.Create(options)), + new TestOptionsMonitor(options), + namespaces, + timeProvider ?? TimeProvider.System, + NullLogger.Instance, + NullLogger.Instance); + } + + private static DisseminationBroadcastQueue CreateBroadcastQueue( + FakeTransport transport, + IReadOnlyList namespaces, + Action? configure = null, + TimeProvider? timeProvider = null) + { + var options = new DisseminationOptions { Enabled = true }; + configure?.Invoke(options); + return new DisseminationBroadcastQueue( + timeProvider ?? TimeProvider.System, + transport.LocalSilo, + transport.GrainFactory, + new TestOptionsMonitor(options), + namespaces, + NullLogger.Instance); + } + + private static void UpdateMaximum(ref int maximum, int candidate) + { + var current = Volatile.Read(ref maximum); + while (candidate > current) + { + var observed = Interlocked.CompareExchange(ref maximum, candidate, current); + if (observed == current) + { + return; + } + + current = observed; + } + } + + private static DisseminationProtocol CreateProtocolWithBroadcastLogger( + FakeTransport transport, + FakeNamespace ns, + Action configure, + TimeProvider timeProvider, + Microsoft.Extensions.Logging.ILogger broadcastLogger) + { + var options = new DisseminationOptions { Enabled = true }; + configure(options); + var localSiloDetails = new FakeLocalSiloDetails(transport.LocalSilo); + return new DisseminationProtocol( + localSiloDetails, + transport.GrainFactory, + new DisseminationMembership( + transport.MembershipManager, + localSiloDetails, + Options.Create(options)), + new TestOptionsMonitor(options), + [ns], + timeProvider, + NullLogger.Instance, + broadcastLogger); + } + + private static DisseminationOverlayOptions CreateOverlayOptions(int fanout) => new() + { + FanOutFactor = _ => fanout, + }; + + private static SiloAddress CreateSilo(int port) => SiloAddress.New(new IPEndPoint(IPAddress.Loopback, port), port); + + private static SiloAddress[] CreateSilos(int count) => + Enumerable.Range(11111, count).Select(CreateSilo).OrderBy(static silo => silo).ToArray(); + + private static Dictionary> CreateAntiEntropyRequestDigest( + DisseminationNamespace namespaceName, + params (DisseminationKey Key, long Version)[] versions) => + new() + { + [namespaceName] = versions + .Select(static entry => new DigestEntry(entry.Key, entry.Version)) + .ToList(), + }; + + private static DisseminationBroadcastBatch CreateBroadcastBatch(SiloAddress sender, params DisseminationBroadcastValue[] values) => new() + { + Sender = sender, + Values = CreateBroadcastValueGroups(values), + }; + + private static IEnumerable GetBroadcastValues(DisseminationBroadcastBatch batch) => + batch.Values.Values.SelectMany(static values => values); + + private static SiloAddress[] GetSentBroadcastPeers(FakeTransport transport) + { + lock (transport.BroadcastBatches) + { + return [.. transport.BroadcastBatches.Select(static batch => batch.Peer)]; + } + } + + private static void ClearBroadcastBatches(FakeTransport transport) + { + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Clear(); + } + } + + private static IEnumerable GetAntiEntropyResponseValues(DisseminationAntiEntropyResponse response) => + response.Values.Values.SelectMany(static values => values); + + private static Dictionary> CreateBroadcastValueGroups(params DisseminationBroadcastValue[] values) => + new() + { + [FakeNamespace.DefaultName] = [.. values], + }; + + private static Dictionary> CreateValueGroups(params DisseminationBroadcastValue[] values) => + CreateValueGroups(FakeNamespace.DefaultName, values); + + private static Dictionary> CreateValueGroups(DisseminationNamespace namespaceName, params DisseminationBroadcastValue[] values) => + new() + { + [namespaceName] = [.. values], + }; + + private static MembershipDisseminationNamespace CreateMembershipNamespace( + FakeMembershipManager membershipManager, + Serializer serializer) + { + var options = new ClusterMembershipOptions(); + options.Dissemination.Enabled = true; + return new( + membershipManager, + new TestOptionsMonitor(options), + serializer); + } + + private static DisseminationBroadcastValue CreateDisseminationValue(SiloAddress originator, DisseminationValue value) + { + _ = originator; + return new() + { + Value = value, + TimeToLive = TimeSpan.FromMinutes(1), + }; + } + + private static MembershipTableSnapshot CreateMembershipSnapshot(long version, params MembershipEntry[] entries) => + new(new MembershipVersion(version), entries.ToImmutableDictionary(static entry => entry.SiloAddress)); + + private static MembershipEntry CreateMembershipEntry( + SiloAddress silo, + SiloStatus status, + DateTime startTime, + DateTime? iAmAliveTime = null) => new() + { + SiloAddress = silo, + Status = status, + ProxyPort = silo.Endpoint.Port, + HostName = "localhost", + SiloName = silo.ToParsableString(), + RoleName = "test", + StartTime = startTime, + IAmAliveTime = iAmAliveTime ?? startTime, + }; + + private static IReadOnlyList GetOriginatorTreeTargets( + SiloAddress root, + IEnumerable peers, + int fanout) + { + var participants = GetSortedParticipants(root, root, peers); + var rootIndex = participants.FindIndex(silo => Equals(silo, root)); + var result = new List(); + AddTopLevelTargets(participants, fanout, root, result); + AddFixedChildren(participants, rootIndex, fanout, root, except: null, result); + return result; + } + + private static IReadOnlyList GetForwardingTreeTargets( + SiloAddress local, + SiloAddress root, + IEnumerable peers, + int fanout, + SiloAddress sender) + { + var participants = GetSortedParticipants(local, root, peers); + var localIndex = participants.FindIndex(silo => Equals(silo, local)); + var result = new List(); + AddFixedChildren(participants, localIndex, fanout, root, sender, result); + return result; + } + + private static List GetSortedParticipants( + SiloAddress local, + SiloAddress root, + IEnumerable peers) => + peers + .Append(local) + .Append(root) + .Distinct() + .OrderBy(static silo => silo) + .ToList(); + + private static void AddTopLevelTargets( + IReadOnlyList participants, + int fanout, + SiloAddress root, + List result) + { + var count = Math.Min(fanout, participants.Count); + for (var i = 0; i < count; i++) + { + AddTarget(participants[i], root, except: null, result); + } + } + + private static void AddFixedChildren( + IReadOnlyList participants, + int index, + int fanout, + SiloAddress root, + SiloAddress? except, + List result) + { + if (index < 0) + { + return; + } + + var firstChild = (long)fanout * (index + 1); + for (var i = 0; i < fanout; i++) + { + var childIndex = firstChild + i; + if (childIndex >= participants.Count) + { + break; + } + + AddTarget(participants[(int)childIndex], root, except, result); + } + } + + private static void AddTarget(SiloAddress peer, SiloAddress root, SiloAddress? except, List result) + { + if (Equals(peer, root) || (except is { } excluded && Equals(peer, excluded)) || result.Contains(peer)) + { + return; + } + + result.Add(peer); + } + + private static HashSet GetReachedParticipants( + SiloAddress root, + IReadOnlyDictionary snapshots) + { + var reached = new HashSet { root }; + var pending = new Queue(snapshots[root].OriginatorTreeTargets); + while (pending.Count > 0) + { + var current = pending.Dequeue(); + if (!reached.Add(current)) + { + continue; + } + + foreach (var child in snapshots[current].ForwardingTreeTargets) + { + pending.Enqueue(child); + } + } + + return reached; + } + +#if NET10_0_OR_GREATER + private static Spec CreateMonotonicDisseminationSpec() + { + var spec = new Spec() + .WithJsonPrinters(); + + spec.Operation("Receive", static (version, state) => + { + if (version < state.Version) + { + return Expect.That( + response => response.Result == ModelApplyResult.Obsolete && response.Version == state.Version, + "Older values are obsolete and do not change state") + .SameState(); + } + + if (version == state.Version && state.Version > 0) + { + return Expect.That( + response => response.Result == ModelApplyResult.Duplicate && response.Version == state.Version, + "Equal versions are duplicates") + .SameState(); + } + + return Expect.That( + response => response.Result == ModelApplyResult.Applied && response.Version == version, + "Newer versions are applied") + .ThenState(nextState => nextState.Version = version); + }); + + spec.Operation("RepairPeer", static (peerVersion, state) => + { + if (state.Version > peerVersion) + { + return Expect.That( + response => response.HasValue && response.Version == state.Version, + "Repair returns the local newer value") + .SameState(); + } + + return Expect.That( + response => !response.HasValue && response.Version == 0, + "Repair returns no value when the peer is current or newer") + .SameState(); + }); + + return spec; + } +#endif + + private sealed class FakeNamespace : IDisseminationNamespace + { + public static readonly DisseminationNamespace DefaultName = new("fake-namespace"); + public static readonly DisseminationKey DefaultKey = new("value"); + private readonly object _lock = new(); + private readonly Dictionary _versions = new(); + private readonly Dictionary> _publishedValues = new(); + private readonly DisseminationNamespace _name; + private int _repairRequestCount; + + public FakeNamespace(SiloAddress localSilo, DisseminationNamespace? name = null) + { + _ = localSilo; + _name = name ?? DefaultName; + } + + public Dictionary ApplyCounts { get; } = new(); + + public HashSet ExpectedKeys { get; } = new(); + + public DisseminationNamespace Name => _name; + + public DisseminationNamespaceOptions Options { get; } = new() { Enabled = true }; + + public bool ReturnRepairChain { get; set; } + + public int RepairRequestCount => Volatile.Read(ref _repairRequestCount); + + public TaskCompletionSource? ApplyObserved { get; set; } + + public DisseminationValue CreateValue(DisseminationKey key, long sequence, long fromVersion = 0) => new( + key, + fromVersion, + sequence, + BitConverter.GetBytes(sequence)); + + public DisseminationBroadcastValue CreateItem(SiloAddress originator, DisseminationKey key, long sequence, long fromVersion = 0) => + CreateDisseminationValue(originator, CreateValue(key, sequence, fromVersion)); + + public void PublishValue(DisseminationValue value) + { + lock (_lock) + { + if (!_publishedValues.TryGetValue(value.Key, out var values)) + { + values = []; + _publishedValues.Add(value.Key, values); + } + + values[value.ToVersion] = value; + if (!_versions.TryGetValue(value.Key, out var version) || value.ToVersion > version) + { + _versions[value.Key] = value.ToVersion; + } + } + } + + public void SetValue(DisseminationKey key, long version) => PublishValue(CreateValue(key, version)); + + public void RemoveValue(DisseminationKey key) + { + lock (_lock) + { + _versions.Remove(key); + _publishedValues.Remove(key); + } + } + + public void Clear() + { + lock (_lock) + { + _versions.Clear(); + _publishedValues.Clear(); + } + } + + public (DisseminationKey Key, long Version, long Payload)[] GetState() + { + lock (_lock) + { + return _versions + .OrderBy(static entry => entry.Key.ToString(), StringComparer.Ordinal) + .Select(entry => ( + entry.Key, + entry.Value, + BitConverter.ToInt64(_publishedValues[entry.Key][entry.Value].Payload.Span))) + .ToArray(); + } + } + + public long GetVersion(DisseminationKey key) + { + lock (_lock) + { + return _versions.TryGetValue(key, out var version) ? version : 0; + } + } + + public IEnumerable Digests + { + get + { + KeyValuePair[] versions; + lock (_lock) + { + versions = [.. _versions]; + } + + var present = new HashSet(versions.Length); + foreach (var (key, version) in versions) + { + present.Add(key); + yield return new DigestEntry(key, version); + } + + foreach (var key in ExpectedKeys) + { + if (!present.Contains(key)) + { + yield return new DigestEntry(key, 0); + } + } + } + } + + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) + { + Interlocked.Increment(ref _repairRequestCount); + long version; + List candidates; + lock (_lock) + { + if (!_versions.TryGetValue(request.Key, out var currentVersion)) + { + return DisseminationRepairResult.Unavailable(version: 0); + } + + version = request.ToVersion ?? currentVersion; + if (version > currentVersion) + { + return DisseminationRepairResult.Unavailable(currentVersion); + } + + if (request.FromVersion is { } peerVersion && peerVersion >= version) + { + return DisseminationRepairResult.Current(version); + } + + if (ReturnRepairChain + && _publishedValues.TryGetValue(request.Key, out var publishedValues)) + { + candidates = []; + long? expectedVersion = request.FromVersion; + foreach (var value in publishedValues.Values) + { + if (value.ToVersion > version + || expectedVersion is { } expected && value.ToVersion <= expected) + { + continue; + } + + if (expectedVersion is null && value.FromVersion != 0 + || expectedVersion is { } fromVersion + && value.FromVersion != 0 + && value.FromVersion != fromVersion) + { + continue; + } + + candidates.Add(value); + expectedVersion = value.ToVersion; + if (value.ToVersion == version) + { + break; + } + } + } + else + { + candidates = []; + } + + if (candidates.Count == 0 || candidates[^1].ToVersion != version) + { + var value = _publishedValues.TryGetValue(request.Key, out var valuesByVersion) + && valuesByVersion.TryGetValue(version, out var publishedValue) + && publishedValue.FromVersion == 0 + ? publishedValue + : CreateValue(request.Key, version); + candidates = [value]; + } + } + + if (request.MaxItemCount <= 0) + { + return DisseminationRepairResult.InsufficientCapacity(version); + } + + var values = ImmutableArray.CreateBuilder(); + var byteCount = 0; + foreach (var value in candidates) + { + if (values.Count >= request.MaxItemCount + || value.Payload.Length > request.MaxPayloadBytes + || value.Payload.Length > request.MaxBatchBytes - byteCount) + { + break; + } + + values.Add(value); + byteCount += value.Payload.Length; + } + + if (values.Count == 0) + { + return DisseminationRepairResult.InsufficientCapacity(version); + } + + var isComplete = values[^1].ToVersion == version; + return DisseminationRepairResult.Produced( + version, + values.ToImmutable(), + isComplete); + } + + public ValueTask ApplyValueAsync( + DisseminationValue value, + CancellationToken cancellationToken) + { + var version = BitConverter.ToInt64(value.Payload.Span); + if (version != value.ToVersion) + { + return ValueTask.FromResult(DisseminationApplyResult.Rejected); + } + + lock (_lock) + { + if (_versions.TryGetValue(value.Key, out var current)) + { + if (current > version) + { + return ValueTask.FromResult(DisseminationApplyResult.Obsolete); + } + + if (current == version) + { + return ValueTask.FromResult(DisseminationApplyResult.Duplicate); + } + } + + _versions[value.Key] = version; + if (!_publishedValues.TryGetValue(value.Key, out var values)) + { + values = []; + _publishedValues.Add(value.Key, values); + } + + values[version] = CreateValue(value.Key, version); + ApplyCounts[value.Key] = ApplyCounts.TryGetValue(value.Key, out var count) ? count + 1 : 1; + } + + ApplyObserved?.TrySetResult(); + return ValueTask.FromResult(DisseminationApplyResult.Applied); + } + } + + private sealed class FakeDisseminationService : IDisseminationService + { + private readonly Dictionary<(DisseminationNamespace Namespace, DisseminationKey Key), long> _knownVersions = []; + + public List Values { get; } = new(); + + public Queue Results { get; } = new(); + + public Func>? PublishHandler { get; set; } + + public async ValueTask Publish( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken) + { + var stream = (disseminationNamespace.Name, key); + var repair = disseminationNamespace.CreateRepair(new DisseminationRepairRequest( + key, + _knownVersions.TryGetValue(stream, out var knownVersion) ? knownVersion : null, + version, + maxItemCount: 1024, + maxBatchBytes: 1024 * 1024, + maxPayloadBytes: disseminationNamespace.Options.MaxPayloadBytes)); + if (repair.Status is not DisseminationRepairStatus.Produced || !repair.IsComplete) + { + return false; + } + + Values.AddRange(repair.Values); + var result = PublishHandler is null + ? Results.Count == 0 || Results.Dequeue() + : await PublishHandler(repair.Values[^1], cancellationToken); + if (result) + { + _knownVersions[stream] = repair.Version; + } + + return result; + } + + public IReadOnlyList GetUnconfirmedPeers( + IDisseminationNamespace disseminationNamespace) => []; + } + + private sealed class FakeTransport + { + private readonly SiloAddress _localSilo; + private readonly List _peers; + private readonly Dictionary _targets = new(); + private readonly object _membershipVersionLock = new(); + private long _membershipFingerprint = long.MinValue; + private long _membershipVersion; + + public FakeTransport(SiloAddress localSilo, params SiloAddress[] peers) + { + _localSilo = localSilo; + _peers = peers.ToList(); + MembershipManager = new FakeMembershipManager(GetMembershipSnapshot, RefreshMembership); + GrainFactory = Substitute.For(); + GrainFactory + .GetSystemTarget(Constants.DisseminationSystemTargetType, Arg.Any()) + .Returns(callInfo => GetSystemTarget(callInfo.ArgAt(1))); + } + + public List<(SiloAddress Peer, DisseminationBroadcastBatch Batch)> BroadcastBatches { get; } = new(); + + public List<(SiloAddress Peer, DisseminationAntiEntropyRequest Request)> AntiEntropyRequests { get; } = new(); + + public IInternalGrainFactory GrainFactory { get; } + + public FakeMembershipManager MembershipManager { get; } + + public List Peers => _peers; + + public Dictionary PeerStatuses { get; } = new(); + + public Dictionary StartTimes { get; } = new(); + + public Dictionary TargetResolutionCounts { get; } = new(); + + public Func? SendBroadcastHandler { get; set; } + + public Func>? SendBroadcastResponseHandler { get; set; } + + public Func> ExchangeAntiEntropyHandler { get; set; } = + static (peer, _, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse { Sender = peer }); + + public Func? RefreshMembershipHandler { get; set; } + + public SiloAddress LocalSilo => _localSilo; + + public int RefreshMembershipCallCount { get; private set; } + + public int GetTargetResolutionCount(SiloAddress peer) + { + lock (_targets) + { + return TargetResolutionCounts.TryGetValue(peer, out var count) ? count : 0; + } + } + + private IDisseminationSystemTarget GetSystemTarget(SiloAddress peer) + { + lock (_targets) + { + TargetResolutionCounts[peer] = TargetResolutionCounts.TryGetValue(peer, out var count) ? count + 1 : 1; + if (!_targets.TryGetValue(peer, out var target)) + { + target = new FakeDisseminationSystemTarget(peer, this); + _targets.Add(peer, target); + } + + return target; + } + } + + private MembershipTableSnapshot GetMembershipSnapshot() + { + var entries = _peers.Append(_localSilo).Distinct().Select(peer => + { + var status = PeerStatuses.TryGetValue(peer, out var peerStatus) ? peerStatus : SiloStatus.Active; + var startTime = StartTimes.TryGetValue(peer, out var value) ? value : DateTime.UnixEpoch; + return CreateMembershipEntry(peer, status, startTime); + }).ToArray(); + + var fingerprint = ComputeMembershipVersion(entries); + long version; + lock (_membershipVersionLock) + { + if (fingerprint != _membershipFingerprint) + { + _membershipFingerprint = fingerprint; + _membershipVersion++; + } + + version = _membershipVersion; + } + + return new MembershipTableSnapshot( + new MembershipVersion(version), + entries.ToImmutableDictionary(static entry => entry.SiloAddress)); + } + + private Task RefreshMembership(CancellationToken cancellationToken) + { + RefreshMembershipCallCount++; + return RefreshMembershipHandler?.Invoke(cancellationToken) ?? Task.CompletedTask; + } + + public async Task SendBroadcast( + SiloAddress peer, + DisseminationBroadcastBatch batch, + CancellationToken cancellationToken) + { + if (SendBroadcastResponseHandler is not null) + { + return await SendBroadcastResponseHandler(peer, batch, cancellationToken); + } + + if (SendBroadcastHandler is not null) + { + await SendBroadcastHandler(peer, batch, cancellationToken); + } + else + { + lock (BroadcastBatches) + { + BroadcastBatches.Add((peer, batch)); + } + } + + return CreateAcknowledgment(batch); + } + + public ValueTask ExchangeAntiEntropy( + SiloAddress peer, + DisseminationAntiEntropyRequest request, + CancellationToken cancellationToken) + { + lock (AntiEntropyRequests) + { + AntiEntropyRequests.Add((peer, request)); + } + + return ExchangeAntiEntropyHandler(peer, request, cancellationToken); + } + + public static DisseminationBroadcastResponse CreateAcknowledgment(DisseminationBroadcastBatch batch) + { + var acknowledgments = new Dictionary>(); + foreach (var (namespaceName, values) in batch.Values) + { + acknowledgments[namespaceName] = values + .GroupBy(static value => value.Value.Key) + .Select(static stream => new DigestEntry( + stream.Key, + stream.Max(static value => value.Value.ToVersion))) + .ToList(); + } + + return new DisseminationBroadcastResponse { Acknowledgments = acknowledgments }; + } + + private static long ComputeMembershipVersion(IEnumerable entries) + { + var result = 17L; + foreach (var entry in entries.OrderBy(static entry => entry.SiloAddress)) + { + result = unchecked((result * 31) + entry.SiloAddress.GetConsistentHashCode()); + result = unchecked((result * 31) + (int)entry.Status); + result = unchecked((result * 31) + entry.StartTime.Ticks); + } + + return result == MembershipVersion.MinValue.Value ? result + 1 : result; + } + } + + private sealed class FakeDisseminationSystemTarget(SiloAddress peer, FakeTransport transport) : IDisseminationSystemTarget + { + public Task PushBroadcast( + DisseminationBroadcastBatch batch, + CancellationToken cancellationToken) => + transport.SendBroadcast(peer, batch, cancellationToken); + + public async Task ExchangeAntiEntropy( + DisseminationAntiEntropyRequest request, + CancellationToken cancellationToken) => + await transport.ExchangeAntiEntropy(peer, request, cancellationToken); + } + + private sealed class FakeLocalSiloDetails(SiloAddress siloAddress) : ILocalSiloDetails + { + public string Name => "test"; + + public string ClusterId => "test"; + + public string DnsHostName => "localhost"; + + public SiloAddress SiloAddress => siloAddress; + + public SiloAddress GatewayAddress => siloAddress; + } + +#if NET10_0_OR_GREATER + private sealed class MonotonicDisseminationHarness(SiloAddress localSilo) + { + private readonly FakeNamespace _topic = new(localSilo); + + public void Reset() => _topic.Clear(); + + public async Task Receive(long version, CancellationToken cancellationToken) + { + var value = _topic.CreateValue(FakeNamespace.DefaultKey, version); + var result = await _topic.ApplyValueAsync(value, cancellationToken); + return new ModelApplyResponse(ToModelResult(result), _topic.GetVersion(FakeNamespace.DefaultKey)); + } + + public Task RepairPeer(long peerVersion, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var localDigest = _topic.Digests.SingleOrDefault(); + if (localDigest.Version == 0) + { + return Task.FromResult(new ModelRepairResponse(false, 0)); + } + + if (localDigest.Version <= peerVersion) + { + return Task.FromResult(new ModelRepairResponse(false, 0)); + } + + var repair = _topic.CreateRepair(new DisseminationRepairRequest( + FakeNamespace.DefaultKey, + peerVersion, + toVersion: null, + maxItemCount: 1, + maxBatchBytes: 1024, + maxPayloadBytes: 1024)); + return Task.FromResult(repair.Status is DisseminationRepairStatus.Produced + ? new ModelRepairResponse(true, repair.Version) + : new ModelRepairResponse(false, 0)); + } + + private static ModelApplyResult ToModelResult(DisseminationApplyResult result) => result switch + { + DisseminationApplyResult.Applied => ModelApplyResult.Applied, + DisseminationApplyResult.Duplicate => ModelApplyResult.Duplicate, + DisseminationApplyResult.Obsolete => ModelApplyResult.Obsolete, + _ => ModelApplyResult.Rejected, + }; + } +#endif + + private sealed class FakeMembershipManager : IMembershipManager + { + private readonly Func? _getCurrentSnapshot; + private readonly Func? _refresh; + private MembershipTableSnapshot _currentSnapshot; + + public FakeMembershipManager(MembershipTableSnapshot currentSnapshot) + { + _currentSnapshot = currentSnapshot; + } + + public FakeMembershipManager( + Func getCurrentSnapshot, + Func refresh) + { + _getCurrentSnapshot = getCurrentSnapshot; + _refresh = refresh; + _currentSnapshot = getCurrentSnapshot(); + } + + public MembershipTableSnapshot CurrentSnapshot + { + get => _getCurrentSnapshot?.Invoke() ?? _currentSnapshot; + set => _currentSnapshot = value; + } + + public int RefreshCallCount { get; private set; } + + public List RefreshTargetVersions { get; } = new(); + + public Func? ProcessGossipSnapshotHandler { get; set; } + + public IAsyncEnumerable MembershipUpdates => EmptyUpdates(TestContext.Current.CancellationToken); + + public SiloStatus LocalSiloStatus => SiloStatus.Active; + + public Task UpdateLocalStatus(SiloStatus status, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task TryKillSilo(SiloAddress silo, CancellationToken cancellationToken) => Task.FromResult(false); + + public Task TrySuspectSilo(SiloAddress silo, SiloAddress? indirectProbingSilo, CancellationToken cancellationToken) => Task.FromResult(false); + + public Task Refresh(MembershipVersion? targetVersion, CancellationToken cancellationToken) + { + RefreshCallCount++; + RefreshTargetVersions.Add(targetVersion); + return _refresh?.Invoke(cancellationToken) ?? Task.CompletedTask; + } + + public Task ProcessGossipSnapshot(MembershipTableSnapshot snapshot, CancellationToken cancellationToken) + { + if (ProcessGossipSnapshotHandler is { } handler) + { + return handler(snapshot, cancellationToken); + } + + CurrentSnapshot = snapshot; + return Task.CompletedTask; + } + + public Task UpdateIAmAlive(CancellationToken cancellationToken) => Task.CompletedTask; + + public bool CheckHealth(DateTime lastCheckTime, out string reason) + { + reason = string.Empty; + return true; + } + + public void Participate(ISiloLifecycle lifecycle) + { + } + + private static async IAsyncEnumerable EmptyUpdates( + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.CompletedTask; + yield break; + } + } + + + + private sealed class TestOptionsMonitor(T currentValue) : IOptionsMonitor + { + public T CurrentValue => currentValue; + + public T Get(string? name) => currentValue; + + public IDisposable? OnChange(Action listener) => null; + } + + private sealed class TestTimeProvider : TimeProvider + { + private readonly object _lock = new(); + private DateTimeOffset _utcNow = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + private long _timestamp; + + public override DateTimeOffset GetUtcNow() + { + lock (_lock) + { + return _utcNow; + } + } + + public override long GetTimestamp() + { + lock (_lock) + { + return _timestamp; + } + } + + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + public void Advance(TimeSpan value) + { + lock (_lock) + { + _utcNow += value; + _timestamp += value.Ticks; + } + } + } + + private sealed class AutoAdvancingTimeProvider(TimeSpan step) : TimeProvider + { + private readonly object _lock = new(); + private DateTimeOffset _utcNow = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + public override DateTimeOffset GetUtcNow() + { + lock (_lock) + { + var result = _utcNow; + _utcNow += step; + return result; + } + } + } + + private sealed class RecordingFakeTimeProvider : TimeProvider + { + private readonly FakeTimeProvider _inner = new(); + private readonly object _lock = new(); + private readonly List _timerDueTimes = []; + private int _throwOnNextTimerChange; + + public IReadOnlyList TimerDueTimes + { + get + { + lock (_lock) + { + return [.. _timerDueTimes]; + } + } + } + + public override DateTimeOffset GetUtcNow() => _inner.GetUtcNow(); + + public override long GetTimestamp() => _inner.GetTimestamp(); + + public override long TimestampFrequency => _inner.TimestampFrequency; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + RecordTimerChange(dueTime); + return new RecordingTimer(this, _inner.CreateTimer(callback, state, dueTime, period)); + } + + public void Advance(TimeSpan duration) => _inner.Advance(duration); + + public void ThrowOnNextTimerChange() => ThrowOnNextTimerChanges(1); + + public void ThrowOnNextTimerChanges(int count) => Interlocked.Exchange(ref _throwOnNextTimerChange, count); + + private bool ShouldThrowOnTimerChange() + { + while (true) + { + var current = Volatile.Read(ref _throwOnNextTimerChange); + if (current <= 0) + { + return false; + } + + if (Interlocked.CompareExchange(ref _throwOnNextTimerChange, current - 1, current) == current) + { + return true; + } + } + } + + private void RecordTimerChange(TimeSpan dueTime) + { + lock (_lock) + { + _timerDueTimes.Add(dueTime); + } + } + + private sealed class RecordingTimer(RecordingFakeTimeProvider owner, ITimer inner) : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) + { + owner.RecordTimerChange(dueTime); + if (owner.ShouldThrowOnTimerChange()) + { + throw new InvalidOperationException("The test timer fails one scheduled change."); + } + + return inner.Change(dueTime, period); + } + + public void Dispose() => inner.Dispose(); + + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + } + + private sealed class RecordingLogger : Microsoft.Extensions.Logging.ILogger + { + private int _warningCount; + + public int WarningCount => Volatile.Read(ref _warningCount); + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true; + + public void Log( + Microsoft.Extensions.Logging.LogLevel logLevel, + Microsoft.Extensions.Logging.EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (logLevel == Microsoft.Extensions.Logging.LogLevel.Warning) + { + Interlocked.Increment(ref _warningCount); + } + } + } + + // Subscribes to the dissemination DiagnosticListener and lets a test deterministically await the moment a + // peer pump (re)arms its flush timer, replacing wall-clock Task.Delay bridges. Buffered, consume-once + // semantics mean an event emitted before the wait is registered is still observed. + private sealed class BroadcastScheduleObserver : IObserver>, IDisposable + { + private readonly object _lock = new(); + private readonly List _events = new(); + private readonly HashSet _consumed = new(); + private readonly List _waiters = new(); + private readonly IDisposable _subscription; + + public BroadcastScheduleObserver() + { + _subscription = DisseminationEvents.Listener.Subscribe( + this, + static name => name == DisseminationEvents.BroadcastScheduledEventName); + } + + public Task WaitAsync( + Func predicate, + TimeSpan timeout, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + TaskCompletionSource completion; + lock (_lock) + { + foreach (var scheduled in _events) + { + if (!_consumed.Contains(scheduled) && predicate(scheduled)) + { + _consumed.Add(scheduled); + return Task.FromResult(scheduled); + } + } + + completion = new(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Add(new Waiter(predicate, completion)); + } + + return completion.Task.WaitAsync(timeout, cancellationToken); + } + + public void OnNext(KeyValuePair value) + { + if (value.Value is not DisseminationBroadcastScheduledEvent scheduled) + { + return; + } + + TaskCompletionSource? completion = null; + lock (_lock) + { + _events.Add(scheduled); + for (var i = 0; i < _waiters.Count; i++) + { + if (_waiters[i].Predicate(scheduled)) + { + completion = _waiters[i].Completion; + _waiters.RemoveAt(i); + _consumed.Add(scheduled); + break; + } + } + } + + completion?.TrySetResult(scheduled); + } + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void Dispose() => _subscription.Dispose(); + + private sealed record Waiter( + Func Predicate, + TaskCompletionSource Completion); + } + + private sealed class AdmissionRejectionObserver : IObserver>, IDisposable + { + private readonly IDisposable _subscription; + + public AdmissionRejectionObserver() + { + _subscription = DisseminationEvents.Listener.Subscribe( + this, + static name => name == DisseminationEvents.QueueAdmissionRejectedEventName); + } + + public List Events { get; } = []; + + public void OnNext(KeyValuePair value) + { + if (value.Value is DisseminationQueueAdmissionRejectedEvent rejection) + { + Events.Add(rejection); + } + } + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void Dispose() => _subscription.Dispose(); + } + + [Fact] + public async Task PublishRoutesDeploymentLoadOnlyToActiveSilos() + { + var topology = CreateScopeRoutingTopology(); + var transport = topology.Transport; + var ns = new ScopedNamespace( + topology.Local, + new DisseminationNamespace("deployment-load-publish"), + DisseminationMembershipScope.ActiveMembers); + var protocol = CreateProtocol( + transport, + [ns], + static options => options.Overlay.FanOutFactor = static _ => 32); + + Assert.True(await PublishScopedValue( + protocol, + ns, + ns.CreateValue("load", sequence: 1), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal( + new[] { topology.ActiveOne, topology.ActiveTwo }, + GetBroadcastPeersForNamespace(transport, ns.Name)); + Assert.DoesNotContain(topology.Joining, GetBroadcastPeersForNamespace(transport, ns.Name)); + Assert.DoesNotContain(topology.ShuttingDown, GetBroadcastPeersForNamespace(transport, ns.Name)); + Assert.DoesNotContain(topology.Stopping, GetBroadcastPeersForNamespace(transport, ns.Name)); + Assert.DoesNotContain(topology.Dead, GetBroadcastPeersForNamespace(transport, ns.Name)); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PublishRoutesMembershipToJoiningActiveShuttingDownAndStoppingSilos() + { + var topology = CreateScopeRoutingTopology(); + var transport = topology.Transport; + var ns = new ScopedNamespace( + topology.Local, + new DisseminationNamespace("membership-publish"), + DisseminationMembershipScope.AllMembers); + var protocol = CreateProtocol( + transport, + [ns], + static options => options.Overlay.FanOutFactor = static _ => 32); + + Assert.True(await PublishScopedValue( + protocol, + ns, + ns.CreateValue("membership", sequence: 1), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal( + new[] + { + topology.ActiveOne, + topology.ActiveTwo, + topology.Joining, + topology.ShuttingDown, + topology.Stopping, + }.OrderBy(static silo => silo), + GetBroadcastPeersForNamespace(transport, ns.Name).OrderBy(static silo => silo)); + Assert.DoesNotContain(topology.Dead, GetBroadcastPeersForNamespace(transport, ns.Name)); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task ReceiveBroadcastForwardsDeploymentLoadOnlyToActiveSilos() + { + var topology = CreateForwardingScopeTopology(); + var ns = new ScopedNamespace( + topology.Local, + new DisseminationNamespace("deployment-load-forward"), + DisseminationMembershipScope.ActiveMembers); + var protocol = CreateProtocol( + topology.Transport, + [ns], + static options => options.Overlay.FanOutFactor = static _ => 2); + + await protocol.ReceiveBroadcast( + CreateScopedBroadcastBatch( + topology.Sender, + ns, + ns.CreateItem(topology.Sender, "load", sequence: 1)), + TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal( + new[] { topology.LastActive }, + GetBroadcastPeersForNamespace(topology.Transport, ns.Name)); + Assert.DoesNotContain(topology.Joining, GetBroadcastPeersForNamespace(topology.Transport, ns.Name)); + Assert.DoesNotContain(topology.Dead, GetBroadcastPeersForNamespace(topology.Transport, ns.Name)); + Assert.Equal(1, ns.GetVersion("load")); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task ReceiveBroadcastForwardsMembershipToAllEligibleSilos() + { + var topology = CreateForwardingScopeTopology(); + var ns = new ScopedNamespace( + topology.Local, + new DisseminationNamespace("membership-forward"), + DisseminationMembershipScope.AllMembers); + var protocol = CreateProtocol( + topology.Transport, + [ns], + static options => options.Overlay.FanOutFactor = static _ => 2); + + await protocol.ReceiveBroadcast( + CreateScopedBroadcastBatch( + topology.Sender, + ns, + ns.CreateItem(topology.Sender, "membership", sequence: 1)), + TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal( + new[] { topology.LastActive, topology.Joining }, + GetBroadcastPeersForNamespace(topology.Transport, ns.Name)); + Assert.DoesNotContain(topology.Dead, GetBroadcastPeersForNamespace(topology.Transport, ns.Name)); + Assert.Equal(1, ns.GetVersion("membership")); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task AntiEntropyRoutesDeploymentLoadOnlyToActiveSilos() + { + var topology = CreateScopeRoutingTopology(); + var ns = new ScopedNamespace( + topology.Local, + new DisseminationNamespace("deployment-load-anti-entropy"), + DisseminationMembershipScope.ActiveMembers); + ns.SetValue("load", version: 1); + var protocol = CreateProtocol( + topology.Transport, + [ns], + static options => options.Overlay.AntiEntropyPeerCount = 32); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal( + new[] { topology.ActiveOne, topology.ActiveTwo }, + GetAntiEntropyPeersForNamespace(topology.Transport, ns.Name)); + Assert.DoesNotContain(topology.Joining, GetAntiEntropyPeersForNamespace(topology.Transport, ns.Name)); + Assert.DoesNotContain(topology.Dead, GetAntiEntropyPeersForNamespace(topology.Transport, ns.Name)); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task AntiEntropyRoutesMembershipToAllEligibleSilos() + { + var topology = CreateScopeRoutingTopology(); + var ns = new ScopedNamespace( + topology.Local, + new DisseminationNamespace("membership-anti-entropy"), + DisseminationMembershipScope.AllMembers); + ns.SetValue("membership", version: 1); + var protocol = CreateProtocol( + topology.Transport, + [ns], + static options => options.Overlay.AntiEntropyPeerCount = 32); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal( + new[] + { + topology.ActiveOne, + topology.ActiveTwo, + topology.Joining, + topology.ShuttingDown, + topology.Stopping, + }.OrderBy(static silo => silo), + GetAntiEntropyPeersForNamespace(topology.Transport, ns.Name).OrderBy(static silo => silo)); + Assert.DoesNotContain(topology.Dead, GetAntiEntropyPeersForNamespace(topology.Transport, ns.Name)); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task AntiEntropyPreservesCursorFairnessAcrossNamespaceScopes() + { + var topology = CreateScopeRoutingTopology(); + var load = new ScopedNamespace( + topology.Local, + new DisseminationNamespace("deployment-load-fairness"), + DisseminationMembershipScope.ActiveMembers); + var membership = new ScopedNamespace( + topology.Local, + new DisseminationNamespace("membership-fairness"), + DisseminationMembershipScope.AllMembers); + load.SetValue("load", version: 1); + membership.SetValue("membership", version: 1); + var protocol = CreateProtocol(topology.Transport, [load, membership], options => + { + options.Overlay.AntiEntropyPeerCount = 1; + options.MaxBatchItems = 1; + options.MaxBatchBytes = sizeof(long); + }); + + for (var i = 0; i < 5; i++) + { + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + } + + Assert.Equal(5, topology.Transport.AntiEntropyRequests.Count); + Assert.Equal( + new[] { topology.ActiveOne, topology.ActiveTwo }, + GetAntiEntropyPeersForNamespace(topology.Transport, load.Name).Distinct().OrderBy(static silo => silo)); + Assert.Equal( + new[] + { + topology.ActiveOne, + topology.ActiveTwo, + topology.Joining, + topology.ShuttingDown, + topology.Stopping, + }.OrderBy(static silo => silo), + GetAntiEntropyPeersForNamespace(topology.Transport, membership.Name).Distinct().OrderBy(static silo => silo)); + Assert.All( + topology.Transport.AntiEntropyRequests.Where(request => request.Request.Digests.ContainsKey(load.Name)), + request => Assert.Contains(request.Peer, new[] { topology.ActiveOne, topology.ActiveTwo })); + + var request = new DisseminationAntiEntropyRequest + { + Sender = topology.ActiveOne, + Digests = new() + { + [load.Name] = [new DigestEntry("load", version: 0)], + [membership.Name] = [new DigestEntry("membership", version: 0)], + }, + }; + var first = await protocol.ReceiveAntiEntropy(request, TestContext.Current.CancellationToken); + var second = await protocol.ReceiveAntiEntropy(request, TestContext.Current.CancellationToken); + var responses = new[] { first, second }; + + Assert.All(responses, response => + { + var value = Assert.Single(GetAntiEntropyResponseValues(response)); + Assert.Equal(sizeof(long), value.Value.Payload.Length); + Assert.True(response.Truncated); + }); + Assert.Equal( + new[] { load.Name, membership.Name }.OrderBy(static name => name), + responses.SelectMany(static response => response.Values.Keys).OrderBy(static name => name)); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task QueuePrunePreservesPeerStateRequiredByAnotherNamespaceScope() + { + var local = CreateSilo(35001); + var peer = CreateSilo(35002); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var load = new ScopedNamespace( + local, + new DisseminationNamespace("deployment-load-prune"), + DisseminationMembershipScope.ActiveMembers); + var membership = new ScopedNamespace( + local, + new DisseminationNamespace("membership-prune"), + DisseminationMembershipScope.AllMembers); + load.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + membership.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + var protocol = CreateProtocol( + transport, + [load, membership], + static options => options.Overlay.FanOutFactor = static _ => 1, + timeProvider); + using var schedule = new BroadcastScheduleObserver(); + var loadScheduled = schedule.WaitAsync( + scheduled => scheduled.Peer.Equals(peer), + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(await PublishScopedValue( + protocol, + load, + load.CreateValue("load", sequence: 1), + TestContext.Current.CancellationToken)); + await loadScheduled; + Assert.True(await PublishScopedValue( + protocol, + membership, + membership.CreateValue("membership", sequence: 1), + TestContext.Current.CancellationToken)); + + transport.PeerStatuses[peer] = SiloStatus.Joining; + await protocol.ReceiveBroadcast( + new DisseminationBroadcastBatch + { + Sender = peer, + Values = [], + }, + TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(peer, batch.Peer); + Assert.DoesNotContain(load.Name, batch.Batch.Values.Keys); + var membershipValue = Assert.Single(batch.Batch.Values[membership.Name]); + Assert.Equal(new DisseminationKey("membership"), membershipValue.Value.Key); + Assert.Equal(1, membershipValue.Value.ToVersion); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueRejectsNewDistinctKeyAtNamespacePendingLimit() + { + var local = CreateSilo(36001); + var peer = CreateSilo(36002); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.Options.MaxPendingItemCount = 2; + var queue = CreateBroadcastQueue(transport, [ns]); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + sendStarted.TrySetResult(); + await releaseSend.Task.WaitAsync(cancellationToken); + }; + ns.SetValue("first", version: 1); + ns.SetValue("second", version: 1); + ns.SetValue("rejected", version: 1); + + queue.Notify(peer, ns, "first"); + queue.Notify(peer, ns, "second"); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + queue.Notify(peer, ns, "rejected"); + releaseSend.TrySetResult(); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal( + new DisseminationKey[] { "first", "second" }, + GetBroadcastValues(batch.Batch).Select(static value => value.Value.Key).OrderBy(static key => key.Value)); + Assert.DoesNotContain(GetBroadcastValues(batch.Batch), static value => value.Value.Key == new DisseminationKey("rejected")); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueAllowsLatestReplacementForAdmittedKeyAtPendingLimit() + { + var local = CreateSilo(36011); + var peer = CreateSilo(36012); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.Options.MaxPendingItemCount = 1; + var queue = CreateBroadcastQueue(transport, [ns]); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + if (Interlocked.Increment(ref sendCount) == 1) + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task.WaitAsync(cancellationToken); + } + }; + ns.SetValue("admitted", version: 1); + + queue.Notify(peer, ns, "admitted"); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + ns.SetValue("admitted", version: 2); + queue.Notify(peer, ns, "admitted"); + ns.SetValue("rejected", version: 1); + queue.Notify(peer, ns, "rejected"); + releaseFirstSend.TrySetResult(); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(2, sendCount); + Assert.Equal( + new long[] { 1, 2 }, + transport.BroadcastBatches.Select(batch => Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion)); + Assert.All( + transport.BroadcastBatches, + batch => Assert.Equal( + new DisseminationKey("admitted"), + Assert.Single(GetBroadcastValues(batch.Batch)).Value.Key)); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueAdmissionRejectionEmitsBoundedDiagnostic() + { + var local = CreateSilo(36021); + var peer = CreateSilo(36022); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local, new DisseminationNamespace("diagnostic-bound")); + ns.Options.MaxPendingItemCount = 1; + var queue = CreateBroadcastQueue(transport, [ns]); + using var observer = new AdmissionRejectionObserver(); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + sendStarted.TrySetResult(); + await releaseSend.Task.WaitAsync(cancellationToken); + }; + ns.SetValue("admitted", version: 1); + ns.SetValue("sensitive-unbounded-key", version: 1); + + queue.Notify(peer, ns, "admitted"); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + queue.Notify(peer, ns, "sensitive-unbounded-key"); + + var rejection = Assert.Single(observer.Events); + Assert.Equal(local, rejection.LocalSilo); + Assert.Equal(peer, rejection.Peer); + Assert.Equal(ns.Name, rejection.Namespace); + Assert.Equal(1, rejection.Limit); + Assert.Equal(DisseminationEvents.NamespacePendingLimitReason, rejection.Reason); + Assert.Null(rejection.GetType().GetProperty("Key")); + releaseSend.TrySetResult(); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueAdmissionRejectionIncrementsBoundedReasonMetric() + { + const string instrumentName = DisseminationInstruments.QueueAdmissionRejectedName; + var local = CreateSilo(36031); + var peer = CreateSilo(36032); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local, new DisseminationNamespace("metric-bound")); + ns.Options.MaxPendingItemCount = 1; + var queue = CreateBroadcastQueue(transport, [ns]); + var observation = new TaskCompletionSource<(long Measurement, KeyValuePair[] Tags)>( + TaskCreationOptions.RunContinuationsAsynchronously); + Instrument? publishedInstrument = null; + var observationCount = 0; + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == DisseminationInstruments.MeterName && instrument.Name == instrumentName) + { + publishedInstrument = instrument; + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var capturedTags = tags.ToArray(); + if (capturedTags.Any(tag => tag.Key == "namespace" && Equals(tag.Value, ns.Name))) + { + Interlocked.Increment(ref observationCount); + observation.TrySetResult((measurement, capturedTags)); + } + }); + listener.Start(); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + sendStarted.TrySetResult(); + await releaseSend.Task.WaitAsync(cancellationToken); + }; + ns.SetValue("admitted", version: 1); + ns.SetValue("rejected", version: 1); + + queue.Notify(peer, ns, "admitted"); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + queue.Notify(peer, ns, "rejected"); + var result = await observation.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.IsType>(publishedInstrument); + Assert.Equal("keys", publishedInstrument.Unit); + Assert.Equal(1, result.Measurement); + Assert.Equal(1, observationCount); + Assert.Equal( + new[] { "namespace", "reason" }, + result.Tags.Select(static tag => tag.Key).OrderBy(static key => key)); + Assert.Contains(result.Tags, tag => tag.Key == "namespace" && Equals(tag.Value, ns.Name)); + Assert.Contains( + result.Tags, + tag => tag.Key == "reason" && Equals(tag.Value, DisseminationEvents.NamespacePendingLimitReason)); + releaseSend.TrySetResult(); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public void DisseminationFailureMetricsUseBoundedDimensionsOnTheOrleansMeter() + { + var expectedInstruments = new HashSet + { + DisseminationInstruments.BroadcastSendFailuresName, + DisseminationInstruments.BroadcastScheduledName, + DisseminationInstruments.AntiEntropyFailuresName, + DisseminationInstruments.PumpFailuresName, + DisseminationInstruments.PublicationsName, + }; + var observations = new ConcurrentDictionary[] Tags)>(); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == DisseminationInstruments.MeterName + && expectedInstruments.Contains(instrument.Name)) + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + observations[instrument.Name] = (measurement, tags.ToArray())); + listener.Start(); + + DisseminationInstruments.OnBroadcastSendFailure(DisseminationFailureReason.Timeout); + DisseminationInstruments.OnBroadcastScheduled(DisseminationBroadcastScheduleReason.Retry); + DisseminationInstruments.OnAntiEntropyFailure(DisseminationFailureReason.Error); + DisseminationInstruments.OnPumpFailure(DisseminationPumpFailureStatus.Permanent); + DisseminationInstruments.OnPublication( + new DisseminationNamespace("test"), + accepted: false, + reason: "disabled"); + + Assert.Equal(expectedInstruments.Order(), observations.Keys.Order()); + AssertMetric(DisseminationInstruments.BroadcastSendFailuresName, ("reason", "timeout")); + AssertMetric(DisseminationInstruments.BroadcastScheduledName, ("reason", "retry")); + AssertMetric(DisseminationInstruments.AntiEntropyFailuresName, ("reason", "error")); + AssertMetric(DisseminationInstruments.PumpFailuresName, ("status", "permanent")); + AssertMetric( + DisseminationInstruments.PublicationsName, + ("namespace", new DisseminationNamespace("test")), + ("result", "rejected"), + ("reason", "disabled")); + + void AssertMetric(string name, params (string Name, object Value)[] expectedTags) + { + var observation = observations[name]; + Assert.Equal(1, observation.Measurement); + Assert.Equal(expectedTags.Length, observation.Tags.Length); + foreach (var expectedTag in expectedTags) + { + Assert.Contains( + observation.Tags, + tag => tag.Key == expectedTag.Name && Equals(tag.Value, expectedTag.Value)); + } + } + } + + [Fact] + public async Task PeerQueueAdmitsNewKeyAfterAcknowledgedDrain() + { + var local = CreateSilo(36041); + var peer = CreateSilo(36042); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.Options.MaxPendingItemCount = 1; + var queue = CreateBroadcastQueue(transport, [ns]); + ns.SetValue("first", version: 1); + + queue.Notify(peer, ns, "first"); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + ns.SetValue("second", version: 1); + queue.Notify(peer, ns, "second"); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(2, transport.BroadcastBatches.Count); + Assert.Equal( + new DisseminationKey[] { "first", "second" }, + transport.BroadcastBatches.Select(batch => Assert.Single(GetBroadcastValues(batch.Batch)).Value.Key)); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueAdmitsNewKeyAfterMembershipPrune() + { + var local = CreateSilo(36051); + var peer = CreateSilo(36052); + var transport = new FakeTransport(local, peer); + var ns = new ScopedNamespace( + local, + new DisseminationNamespace("prune-recovery"), + DisseminationMembershipScope.ActiveMembers); + ns.Options.MaxPendingItemCount = 1; + ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + if (GetBroadcastValues(batch).Any(static value => value.Value.Key == new DisseminationKey("before-prune"))) + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task.WaitAsync(cancellationToken); + } + }; + var protocol = CreateProtocol( + transport, + [ns], + static options => options.Overlay.FanOutFactor = static _ => 1, + new FakeTimeProvider()); + + Assert.True(await PublishScopedValue( + protocol, + ns, + ns.CreateValue("before-prune", sequence: 1), + TestContext.Current.CancellationToken)); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + using var observer = new AdmissionRejectionObserver(); + Assert.False(await PublishScopedValue( + protocol, + ns, + ns.CreateValue("rejected-before-prune", sequence: 1), + TestContext.Current.CancellationToken)); + Assert.Single(observer.Events); + transport.PeerStatuses[peer] = SiloStatus.Joining; + await protocol.ReceiveBroadcast( + new DisseminationBroadcastBatch { Sender = peer, Values = [] }, + TestContext.Current.CancellationToken); + transport.PeerStatuses[peer] = SiloStatus.Active; + Assert.True(await PublishScopedValue( + protocol, + ns, + ns.CreateValue("after-prune", sequence: 1), + TestContext.Current.CancellationToken)); + Assert.Single(observer.Events); + releaseFirstSend.TrySetResult(); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Contains( + transport.BroadcastBatches.SelectMany(batch => GetBroadcastValues(batch.Batch)), + static value => value.Value.Key == new DisseminationKey("after-prune") && value.Value.ToVersion == 1); + Assert.DoesNotContain( + transport.BroadcastBatches.SelectMany(batch => GetBroadcastValues(batch.Batch)), + static value => value.Value.Key == new DisseminationKey("rejected-before-prune")); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueHardBoundIsIndependentPerPeerAndNamespace() + { + var local = CreateSilo(36061); + var firstPeer = CreateSilo(36062); + var secondPeer = CreateSilo(36063); + var transport = new FakeTransport(local, firstPeer, secondPeer); + var firstNamespace = new FakeNamespace(local, new DisseminationNamespace("first-bound")); + var secondNamespace = new FakeNamespace(local, new DisseminationNamespace("second-bound")); + firstNamespace.Options.MaxPendingItemCount = 1; + secondNamespace.Options.MaxPendingItemCount = 1; + firstNamespace.SetValue("first-peer", version: 1); + firstNamespace.SetValue("rejected-first-peer", version: 1); + firstNamespace.SetValue("second-peer", version: 1); + secondNamespace.SetValue("other-namespace", version: 1); + var queue = CreateBroadcastQueue(transport, [firstNamespace, secondNamespace]); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + if (target.Equals(firstPeer) + && batch.Values.TryGetValue(firstNamespace.Name, out var values) + && values.Any(static value => value.Value.Key == new DisseminationKey("first-peer"))) + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task.WaitAsync(cancellationToken); + } + }; + + queue.Notify(firstPeer, firstNamespace, "first-peer"); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + queue.Notify(firstPeer, firstNamespace, "rejected-first-peer"); + queue.Notify(secondPeer, firstNamespace, "second-peer"); + queue.Notify(firstPeer, secondNamespace, "other-namespace"); + releaseFirstSend.TrySetResult(); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var firstPeerBatches = transport.BroadcastBatches.Where(batch => batch.Peer.Equals(firstPeer)).ToArray(); + Assert.Equal( + new[] { firstNamespace.Name, secondNamespace.Name }.OrderBy(static name => name.Value), + firstPeerBatches.SelectMany(static batch => batch.Batch.Values.Keys).Distinct().OrderBy(static name => name.Value)); + Assert.Equal( + new DisseminationKey("first-peer"), + Assert.Single( + firstPeerBatches.SelectMany(batch => batch.Batch.Values.GetValueOrDefault(firstNamespace.Name) ?? [])).Value.Key); + Assert.Equal( + new DisseminationKey("other-namespace"), + Assert.Single( + firstPeerBatches.SelectMany(batch => batch.Batch.Values.GetValueOrDefault(secondNamespace.Name) ?? [])).Value.Key); + var secondPeerBatch = Assert.Single(transport.BroadcastBatches, batch => batch.Peer.Equals(secondPeer)).Batch; + Assert.Equal( + new DisseminationKey("second-peer"), + Assert.Single(secondPeerBatch.Values[firstNamespace.Name]).Value.Key); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueMaterializesNamespaceRepairAtSendTime() + { + var local = CreateSilo(36071); + var peer = CreateSilo(36072); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var queue = CreateBroadcastQueue(transport, [ns]); + ns.SetValue("latest", version: 1); + queue.Notify(peer, ns, "latest"); + + ns.SetValue("latest", version: 2); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var value = Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)).Value; + Assert.Equal(new DisseminationKey("latest"), value.Key); + Assert.Equal(2, value.ToVersion); + Assert.Equal(1, ns.RepairRequestCount); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueAdvancesKnownVersionOnlyFromResponseAcknowledgment() + { + var local = CreateSilo(36081); + var peer = CreateSilo(36082); + var transport = new FakeTransport(local, peer); + var sentVersions = new List(); + var sendCount = 0; + transport.SendBroadcastResponseHandler = (target, batch, cancellationToken) => + { + sentVersions.Add(GetBroadcastValues(batch).Select(static value => value.Value.ToVersion).ToArray()); + return Task.FromResult(Interlocked.Increment(ref sendCount) == 1 + ? new DisseminationBroadcastResponse() + : FakeTransport.CreateAcknowledgment(batch)); + }; + var ns = new FakeNamespace(local) { ReturnRepairChain = true }; + ns.Options.MaxPendingItemCount = 1; + var queue = CreateBroadcastQueue(transport, [ns]); + ns.PublishValue(ns.CreateValue("ack", sequence: 1)); + + queue.Notify(peer, ns, "ack"); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + ns.PublishValue(ns.CreateValue("ack", sequence: 2, fromVersion: 1)); + queue.Notify(peer, ns, "ack"); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + ns.PublishValue(ns.CreateValue("ack", sequence: 3, fromVersion: 2)); + queue.Notify(peer, ns, "ack"); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(3, sendCount); + Assert.Equal(new long[] { 1 }, sentVersions[0]); + Assert.Equal(new long[] { 1, 2 }, sentVersions[1]); + Assert.Equal(new long[] { 3 }, sentVersions[2]); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task BlockedPeerPumpDoesNotPreventAnotherPeerPumpFromDraining() + { + var local = CreateSilo(36091); + var blockedPeer = CreateSilo(36092); + var healthyPeer = CreateSilo(36093); + var transport = new FakeTransport(local, blockedPeer, healthyPeer); + var blockedStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var healthyCompleted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseBlocked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var inFlight = 0; + var observedMax = 0; + transport.SendBroadcastResponseHandler = async (target, batch, cancellationToken) => + { + var current = Interlocked.Increment(ref inFlight); + UpdateMaximum(ref observedMax, current); + var isHealthyPeer = target.Equals(healthyPeer); + try + { + if (target.Equals(blockedPeer)) + { + blockedStarted.TrySetResult(); + await releaseBlocked.Task.WaitAsync(cancellationToken); + } + + return FakeTransport.CreateAcknowledgment(batch); + } + finally + { + Interlocked.Decrement(ref inFlight); + if (isHealthyPeer) + { + healthyCompleted.TrySetResult(); + } + } + }; + var ns = new FakeNamespace(local); + ns.SetValue("blocked", version: 1); + ns.SetValue("healthy", version: 1); + var queue = CreateBroadcastQueue( + transport, + [ns], + static options => options.MaxConcurrentSends = 2); + + queue.Notify(blockedPeer, ns, "blocked"); + var blockedFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await blockedStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + queue.Notify(healthyPeer, ns, "healthy"); + var allFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await healthyCompleted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.False(blockedFlush.IsCompleted); + Assert.False(allFlush.IsCompleted); + Assert.Equal(2, observedMax); + Assert.Equal(1, inFlight); + releaseBlocked.TrySetResult(); + await Task.WhenAll(blockedFlush, allFlush).WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + Assert.Equal(0, inFlight); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + private static ScopeRoutingTopology CreateScopeRoutingTopology() + { + var local = CreateSilo(33001); + var activeOne = CreateSilo(33002); + var activeTwo = CreateSilo(33003); + var joining = CreateSilo(33004); + var shuttingDown = CreateSilo(33005); + var stopping = CreateSilo(33006); + var dead = CreateSilo(33007); + var transport = new FakeTransport( + local, + activeOne, + activeTwo, + joining, + shuttingDown, + stopping, + dead); + transport.PeerStatuses[joining] = SiloStatus.Joining; + transport.PeerStatuses[shuttingDown] = SiloStatus.ShuttingDown; + transport.PeerStatuses[stopping] = SiloStatus.Stopping; + transport.PeerStatuses[dead] = SiloStatus.Dead; + return new(local, activeOne, activeTwo, joining, shuttingDown, stopping, dead, transport); + } + + private static ForwardingScopeTopology CreateForwardingScopeTopology() + { + var sender = CreateSilo(34001); + var local = CreateSilo(34002); + var activeTwo = CreateSilo(34003); + var activeThree = CreateSilo(34004); + var lastActive = CreateSilo(34005); + var joining = CreateSilo(34006); + var shuttingDown = CreateSilo(34007); + var stopping = CreateSilo(34008); + var dead = CreateSilo(34009); + var transport = new FakeTransport( + local, + sender, + activeTwo, + activeThree, + lastActive, + joining, + shuttingDown, + stopping, + dead); + transport.PeerStatuses[joining] = SiloStatus.Joining; + transport.PeerStatuses[shuttingDown] = SiloStatus.ShuttingDown; + transport.PeerStatuses[stopping] = SiloStatus.Stopping; + transport.PeerStatuses[dead] = SiloStatus.Dead; + return new(sender, local, lastActive, joining, dead, transport); + } + + private static ValueTask PublishScopedValue( + DisseminationProtocol protocol, + ScopedNamespace disseminationNamespace, + DisseminationValue value, + CancellationToken cancellationToken) + { + disseminationNamespace.PublishValue(value); + return protocol.Publish( + disseminationNamespace, + value.Key, + value.ToVersion, + cancellationToken); + } + + private static DisseminationBroadcastBatch CreateScopedBroadcastBatch( + SiloAddress sender, + ScopedNamespace disseminationNamespace, + params DisseminationBroadcastValue[] values) => new() + { + Sender = sender, + Values = new() + { + [disseminationNamespace.Name] = [.. values], + }, + }; + + private static SiloAddress[] GetBroadcastPeersForNamespace( + FakeTransport transport, + DisseminationNamespace namespaceName) + { + lock (transport.BroadcastBatches) + { + return + [ + .. transport.BroadcastBatches + .Where(batch => batch.Batch.Values.ContainsKey(namespaceName)) + .Select(static batch => batch.Peer) + .OrderBy(static peer => peer), + ]; + } + } + + private static SiloAddress[] GetAntiEntropyPeersForNamespace( + FakeTransport transport, + DisseminationNamespace namespaceName) + { + lock (transport.AntiEntropyRequests) + { + return + [ + .. transport.AntiEntropyRequests + .Where(request => request.Request.Digests.ContainsKey(namespaceName)) + .Select(static request => request.Peer) + .OrderBy(static peer => peer), + ]; + } + } + + private readonly record struct ScopeRoutingTopology( + SiloAddress Local, + SiloAddress ActiveOne, + SiloAddress ActiveTwo, + SiloAddress Joining, + SiloAddress ShuttingDown, + SiloAddress Stopping, + SiloAddress Dead, + FakeTransport Transport); + + private readonly record struct ForwardingScopeTopology( + SiloAddress Sender, + SiloAddress Local, + SiloAddress LastActive, + SiloAddress Joining, + SiloAddress Dead, + FakeTransport Transport); + + private sealed class ScopedNamespace( + SiloAddress localSilo, + DisseminationNamespace name, + DisseminationMembershipScope membershipScope) : IDisseminationNamespace + { + private readonly FakeNamespace _inner = new(localSilo, name); + + public DisseminationNamespace Name => _inner.Name; + + public DisseminationMembershipScope MembershipScope { get; } = membershipScope; + + public DisseminationNamespaceOptions Options => _inner.Options; + + public IEnumerable Digests => _inner.Digests; + + public DisseminationValue CreateValue(DisseminationKey key, long sequence, long fromVersion = 0) => + _inner.CreateValue(key, sequence, fromVersion); + + public DisseminationBroadcastValue CreateItem( + SiloAddress originator, + DisseminationKey key, + long sequence, + long fromVersion = 0) => + _inner.CreateItem(originator, key, sequence, fromVersion); + + public void PublishValue(DisseminationValue value) => _inner.PublishValue(value); + + public void SetValue(DisseminationKey key, long version) => _inner.SetValue(key, version); + + public long GetVersion(DisseminationKey key) => _inner.GetVersion(key); + + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) => + _inner.CreateRepair(request); + + public ValueTask ApplyValueAsync( + DisseminationValue value, + CancellationToken cancellationToken) => + _inner.ApplyValueAsync(value, cancellationToken); + } + + [Fact] + public async Task QueueAcceptanceDoesNotConfirmPeerNamespaceSupport() + { + var local = CreateSilo(31001); + var peer = CreateSilo(31002); + var timeProvider = new FakeTimeProvider(); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + + Assert.Equal([peer], protocol.GetUnconfirmedPeers(ns)); + Assert.Empty(transport.BroadcastBatches); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task InboundNamespaceTrafficConfirmsPeerSupport() + { + var local = CreateSilo(31003); + var peer = CreateSilo(31004); + var transport = new FakeTransport(local, peer); + var supported = new FakeNamespace(local); + var other = new FakeNamespace(local, "other"); + var protocol = CreateProtocol(transport, [supported, other]); + + var response = await protocol.ReceiveBroadcast( + CreateBroadcastBatch(peer, supported.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 1)), + TestContext.Current.CancellationToken); + + Assert.Empty(protocol.GetUnconfirmedPeers(supported)); + Assert.Equal([peer], protocol.GetUnconfirmedPeers(other)); + Assert.Equal(1, Assert.Single(response.Acknowledgments[supported.Name]).Version); + } + + [Fact] + public async Task ResponseAcknowledgmentConfirmsPeerNamespaceSupport() + { + var local = CreateSilo(31005); + var peer = CreateSilo(31006); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Empty(protocol.GetUnconfirmedPeers(ns)); + Assert.Equal(1, Assert.Single(transport.BroadcastBatches).Batch.Values[ns.Name].Single().Value.ToVersion); + } + + [Fact] + public async Task UnsupportedNamespaceResponseRevokesPeerSupport() + { + var local = CreateSilo(31007); + var peer = CreateSilo(31008); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + await protocol.ReceiveAntiEntropy( + new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = CreateAntiEntropyRequestDigest(ns.Name, (FakeNamespace.DefaultKey, 0)), + }, + TestContext.Current.CancellationToken); + Assert.Empty(protocol.GetUnconfirmedPeers(ns)); + transport.SendBroadcastResponseHandler = (_, _, _) => Task.FromResult( + new DisseminationBroadcastResponse { UnsupportedNamespaces = [ns.Name] }); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal([peer], protocol.GetUnconfirmedPeers(ns)); + } + + [Fact] + public async Task AntiEntropyResponseRetainsUnrelatedPeerNamespaceConfirmations() + { + var local = CreateSilo(31009); + var peer = CreateSilo(31010); + var first = new FakeNamespace(local, "first"); + var second = new FakeNamespace(local, "second"); + first.SetValue(FakeNamespace.DefaultKey, version: 1); + second.SetValue(FakeNamespace.DefaultKey, version: 1); + var transport = new FakeTransport(local, peer); + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult( + new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = new() + { + [first.Name] = [first.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 2)], + }, + }); + var protocol = CreateProtocol( + transport, + [first, second], + options => options.Overlay.AntiEntropyPeerCount = 1); + await protocol.ReceiveAntiEntropy( + new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = new() + { + [first.Name] = [new(FakeNamespace.DefaultKey, 0)], + [second.Name] = [new(FakeNamespace.DefaultKey, 0)], + }, + }, + TestContext.Current.CancellationToken); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Empty(protocol.GetUnconfirmedPeers(first)); + Assert.Empty(protocol.GetUnconfirmedPeers(second)); + Assert.Equal(2, first.GetVersion(FakeNamespace.DefaultKey)); + } + + [Fact] + public async Task PeerNamespaceConfirmationPersistsUntilExplicitRevocationOrRemoval() + { + var local = CreateSilo(31011); + var peer = CreateSilo(31012); + var timeProvider = new FakeTimeProvider(); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(7); + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(5), + timeProvider); + await protocol.ReceiveAntiEntropy( + new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = CreateAntiEntropyRequestDigest(ns.Name, (FakeNamespace.DefaultKey, 0)), + }, + TestContext.Current.CancellationToken); + + timeProvider.Advance(TimeSpan.FromDays(1)); + Assert.Empty(protocol.GetUnconfirmedPeers(ns)); + } + + [Fact] + public async Task GetUnconfirmedPeersReturnsOnlyActiveUnconfirmedPeersForNamespace() + { + var local = CreateSilo(31013); + var confirmed = CreateSilo(31014); + var unconfirmed = CreateSilo(31015); + var joining = CreateSilo(31016); + var dead = CreateSilo(31017); + var transport = new FakeTransport(local, confirmed, unconfirmed, joining, dead); + transport.PeerStatuses[joining] = SiloStatus.Joining; + transport.PeerStatuses[dead] = SiloStatus.Dead; + var ns = new ScopedNamespace(local, "active-only", DisseminationMembershipScope.ActiveMembers); + var protocol = CreateProtocol(transport, [ns]); + await protocol.ReceiveAntiEntropy( + new DisseminationAntiEntropyRequest + { + Sender = confirmed, + Digests = CreateAntiEntropyRequestDigest(ns.Name, (FakeNamespace.DefaultKey, 0)), + }, + TestContext.Current.CancellationToken); + + Assert.Equal([unconfirmed], protocol.GetUnconfirmedPeers(ns)); + Assert.DoesNotContain(local, protocol.GetUnconfirmedPeers(ns)); + Assert.DoesNotContain(joining, protocol.GetUnconfirmedPeers(ns)); + Assert.DoesNotContain(dead, protocol.GetUnconfirmedPeers(ns)); + } + + [Fact] + public async Task DisseminationPushBroadcastWireContractPreservesAcknowledgmentsAndUnsupportedNamespaces() + { + var local = CreateSilo(31018); + var peer = CreateSilo(31019); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + var unsupported = new DisseminationNamespace("unsupported"); + var receivedNamespaces = new ConcurrentBag(); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == DisseminationInstruments.MeterName + && instrument.Name == DisseminationInstruments.ValuesReceivedName) + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + receivedNamespaces.Add(tags.ToArray().Single(static tag => tag.Key == "namespace").Value); + }); + listener.Start(); + var batch = new DisseminationBroadcastBatch + { + Sender = peer, + Values = new() + { + [ns.Name] = [ns.CreateItem(peer, "supported-key", sequence: 3)], + [unsupported] = [ns.CreateItem(peer, "unsupported-key", sequence: 4)], + }, + }; + + var response = await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + + var method = typeof(IDisseminationSystemTarget).GetMethod(nameof(IDisseminationSystemTarget.PushBroadcast)); + Assert.Equal(typeof(Task), method!.ReturnType); + Assert.Equal( + [("Acknowledgments", 0), ("UnsupportedNamespaces", 1)], + typeof(DisseminationBroadcastResponse) + .GetProperties() + .Select(property => ( + property.Name, + Convert.ToInt32(property.GetCustomAttributesData() + .Single(attribute => attribute.AttributeType == typeof(IdAttribute)) + .ConstructorArguments.Single().Value))) + .OrderBy(static property => property.Item2)); + var acknowledgment = Assert.Single(response.Acknowledgments); + Assert.Equal(ns.Name, acknowledgment.Key); + var digest = Assert.Single(acknowledgment.Value); + Assert.Equal(new DisseminationKey("supported-key"), digest.Key); + Assert.Equal(3, digest.Version); + Assert.Equal([unsupported], response.UnsupportedNamespaces); + Assert.Equal([ns.Name], receivedNamespaces); + } + + [Fact] + public async Task DeploymentLoadPublisherDirectSendsToUnconfirmedActivePeersAfterQueueAcceptance() + { + var harness = CreateDeploymentLoadPublisherHarness(); + harness.Dissemination.UnconfirmedPeers = [harness.ActiveTwo]; + + await harness.PublishStatistics(TestContext.Current.CancellationToken); + + AssertDirectRecipients(harness, harness.ActiveTwo); + Assert.Single(harness.Dissemination.PublishCalls); + Assert.Single(harness.Dissemination.QueryCalls); + } + + [Fact] + public async Task DeploymentLoadPublisherDoesNotDirectSendToConfirmedPeersAfterQueueAcceptance() + { + var harness = CreateDeploymentLoadPublisherHarness(); + harness.Dissemination.UnconfirmedPeers = []; + + await harness.PublishStatistics(TestContext.Current.CancellationToken); + + AssertDirectRecipients(harness); + Assert.Single(harness.Dissemination.PublishCalls); + Assert.Single(harness.Dissemination.QueryCalls); + } + + [Fact] + public async Task DeploymentLoadPublisherDoesNotDirectSendToNonActiveUnconfirmedPeers() + { + var harness = CreateDeploymentLoadPublisherHarness(); + harness.Dissemination.UnconfirmedPeers = [harness.Joining]; + + await harness.PublishStatistics(TestContext.Current.CancellationToken); + + AssertDirectRecipients(harness); + Assert.Single(harness.Dissemination.QueryCalls); + } + + [Fact] + public async Task DeploymentLoadPublisherFallsBackToAllActivePeersWhenDisseminationIsDisabled() + { + var harness = CreateDeploymentLoadPublisherHarness(namespaceEnabled: false); + + await harness.PublishStatistics(TestContext.Current.CancellationToken); + + AssertDirectRecipients(harness, harness.ActiveOne, harness.ActiveTwo); + Assert.Empty(harness.Dissemination.PublishCalls); + Assert.Empty(harness.Dissemination.QueryCalls); + } + + [Fact] + public async Task DeploymentLoadPublisherFallsBackToAllActivePeersWhenDisseminationReturnsFalse() + { + var harness = CreateDeploymentLoadPublisherHarness(); + harness.Dissemination.PublishHandler = (_, _, _, _) => ValueTask.FromResult(false); + + await harness.PublishStatistics(TestContext.Current.CancellationToken); + + AssertDirectRecipients(harness, harness.ActiveOne, harness.ActiveTwo); + Assert.Single(harness.Dissemination.PublishCalls); + Assert.Empty(harness.Dissemination.QueryCalls); + } + + [Fact] + public async Task DeploymentLoadPublisherFallsBackToAllActivePeersWhenDisseminationTimesOut() + { + var timeProvider = new FakeTimeProvider(); + var harness = CreateDeploymentLoadPublisherHarness(timeProvider: timeProvider); + harness.Dissemination.PublishHandler = async (_, _, _, cancellationToken) => + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return true; + }; + + var publish = harness.PublishStatistics(TestContext.Current.CancellationToken); + await harness.Dissemination.PublishStarted.Task.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + timeProvider.Advance(harness.RefreshTime); + await publish.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + AssertDirectRecipients(harness, harness.ActiveOne, harness.ActiveTwo); + Assert.Single(harness.Dissemination.PublishCalls); + Assert.Empty(harness.Dissemination.QueryCalls); + } + + [Fact] + public async Task DeploymentLoadPublisherCallerCancellationStopsDisseminationWithoutFallback() + { + var harness = CreateDeploymentLoadPublisherHarness(); + using var cancellation = new CancellationTokenSource(); + CancellationToken observedToken = default; + harness.Dissemination.PublishHandler = async (_, _, _, cancellationToken) => + { + observedToken = cancellationToken; + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return true; + }; + + var publish = harness.PublishStatistics(cancellation.Token); + await harness.Dissemination.PublishStarted.Task.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + async () => await publish.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.True(observedToken.IsCancellationRequested); + AssertDirectRecipients(harness); + Assert.Empty(harness.Dissemination.QueryCalls); + } + + [Fact] + public async Task DeploymentLoadPublisherFallsBackToAllActivePeersWhenDisseminationThrows() + { + var harness = CreateDeploymentLoadPublisherHarness(); + harness.Dissemination.PublishHandler = (_, _, _, _) => + ValueTask.FromException(new InvalidOperationException("test failure")); + + await harness.PublishStatistics(TestContext.Current.CancellationToken); + + AssertDirectRecipients(harness, harness.ActiveOne, harness.ActiveTwo); + Assert.Single(harness.Dissemination.PublishCalls); + Assert.Empty(harness.Dissemination.QueryCalls); + } + + [Fact] + public async Task DeploymentLoadPublisherQueueAcceptanceAloneNeverSuppressesLegacyPeerDelivery() + { + var harness = CreateDeploymentLoadPublisherHarness(); + harness.Dissemination.UnconfirmedPeers = [harness.ActiveOne, harness.ActiveTwo]; + + await harness.PublishStatistics(TestContext.Current.CancellationToken); + + AssertDirectRecipients(harness, harness.ActiveOne, harness.ActiveTwo); + Assert.True(Assert.Single(harness.Dissemination.PublishResults)); + Assert.Equal( + [harness.ActiveOne, harness.ActiveTwo], + harness.Dissemination.UnconfirmedPeers.OrderBy(static peer => peer)); + } + + private static DeploymentLoadPublisherHarness CreateDeploymentLoadPublisherHarness( + bool namespaceEnabled = true, + FakeTimeProvider? timeProvider = null) + { + var local = CreateSilo(31101); + var activeOne = CreateSilo(31102); + var activeTwo = CreateSilo(31103); + var joining = CreateSilo(31104); + var statusOracle = new Phase4FakeSiloStatusOracle(); + statusOracle.SetStatus(local, SiloStatus.Active); + statusOracle.SetStatus(activeOne, SiloStatus.Active); + statusOracle.SetStatus(activeTwo, SiloStatus.Active); + statusOracle.SetStatus(joining, SiloStatus.Joining); + var directTargets = new Dictionary + { + [activeOne] = new(), + [activeTwo] = new(), + [joining] = new(), + }; + var grainFactory = Substitute.For(); + grainFactory + .GetSystemTarget( + Constants.DeploymentLoadPublisherSystemTargetType, + Arg.Any()) + .Returns(call => directTargets[call.ArgAt(1)]); + var services = new Phase4MutableServiceProvider(); + timeProvider ??= new FakeTimeProvider(); + services.Add(timeProvider); + var refreshTime = TimeSpan.FromSeconds(5); + var options = new DeploymentLoadPublisherOptions + { + DeploymentLoadPublisherRefreshTime = refreshTime, + }; + options.Dissemination.Enabled = namespaceEnabled; + var publisher = new DeploymentLoadPublisher( + new FakeLocalSiloDetails(local), + statusOracle, + Options.Create(options), + grainFactory, + NullLoggerFactory.Instance, + new ActivationDirectory(CreatePhase4Instruments()), + new Phase4FakeActivationWorkingSet(), + new Phase4FakeEnvironmentStatisticsProvider(), + Options.Create(new LoadSheddingOptions()), + services, + CreatePhase4SystemTargetShared(local)); + var serializerProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var disseminationNamespace = new DeploymentLoadStatisticsDisseminationNamespace( + publisher, + new TestOptionsMonitor(options), + serializerProvider.GetRequiredService()); + var dissemination = new Phase4FakeDisseminationService(); + services.Add(dissemination); + services.Add(disseminationNamespace); + + return new( + publisher, + dissemination, + local, + activeOne, + activeTwo, + joining, + refreshTime, + directTargets); + } + + private static void AssertDirectRecipients( + DeploymentLoadPublisherHarness harness, + params SiloAddress[] expectedRecipients) + { + var actualRecipients = harness.DirectTargets + .Where(static entry => entry.Value.Updates.Count > 0) + .Select(static entry => entry.Key) + .OrderBy(static peer => peer) + .ToArray(); + Assert.Equal(expectedRecipients.OrderBy(static peer => peer), actualRecipients); + foreach (var recipient in expectedRecipients) + { + var update = Assert.Single(harness.DirectTargets[recipient].Updates); + Assert.Equal(harness.Local, update.Source); + Assert.Equal(harness.Publisher.LocalRuntimeStatistics.DateTime, update.Statistics.DateTime); + } + + Assert.All( + harness.DirectTargets.Where(entry => !expectedRecipients.Contains(entry.Key)), + static entry => Assert.Empty(entry.Value.Updates)); + } + + private static SystemTargetShared CreatePhase4SystemTargetShared(SiloAddress local) => new( + runtimeClient: null!, + new FakeLocalSiloDetails(local), + NullLoggerFactory.Instance, + Options.Create(new SchedulingOptions()), + grainReferenceActivator: null!, + timerRegistry: null!, + activations: new ActivationDirectory(CreatePhase4Instruments()), + schedulerInstruments: CreatePhase4Instruments(), + grainInstruments: CreatePhase4Instruments(), + messagingInstruments: CreatePhase4Instruments(), + messagingProcessingInstruments: CreatePhase4Instruments()); + + private static T CreatePhase4Instruments() where T : class + { + var services = new ServiceCollection(); + services.AddMetrics(); + services.AddSingleton(); + services.AddSingleton(); + return services.BuildServiceProvider().GetRequiredService(); + } + + private sealed record DeploymentLoadPublisherHarness( + DeploymentLoadPublisher Publisher, + Phase4FakeDisseminationService Dissemination, + SiloAddress Local, + SiloAddress ActiveOne, + SiloAddress ActiveTwo, + SiloAddress Joining, + TimeSpan RefreshTime, + Dictionary DirectTargets) + { + public Task PublishStatistics(CancellationToken cancellationToken) => + Publisher.RunOrQueueTask( + async token => + { + await Publisher.PublishStatistics(token); + return true; + }, + cancellationToken); + } + + private sealed class Phase4FakeDisseminationService : IDisseminationService + { + public List<(IDisseminationNamespace Namespace, DisseminationKey Key, long Version)> PublishCalls { get; } = []; + + public List PublishResults { get; } = []; + + public List QueryCalls { get; } = []; + + public IReadOnlyList UnconfirmedPeers { get; set; } = []; + + public TaskCompletionSource PublishStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Func> PublishHandler { get; set; } = + static (_, _, _, _) => ValueTask.FromResult(true); + + public async ValueTask Publish( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken) + { + PublishCalls.Add((disseminationNamespace, key, version)); + PublishStarted.TrySetResult(); + var result = await PublishHandler(disseminationNamespace, key, version, cancellationToken); + PublishResults.Add(result); + return result; + } + + public IReadOnlyList GetUnconfirmedPeers(IDisseminationNamespace disseminationNamespace) + { + QueryCalls.Add(disseminationNamespace); + return UnconfirmedPeers; + } + } + + private sealed class Phase4MutableServiceProvider : IServiceProvider + { + private readonly Dictionary _services = []; + + public void Add(T service) where T : class => _services[typeof(T)] = service; + + public object? GetService(Type serviceType) => + _services.TryGetValue(serviceType, out var service) ? service : null; + } + + private sealed class Phase4FakeSiloStatusOracle : ISiloStatusOracle + { + private readonly ConcurrentDictionary _statuses = new(); + + public SiloStatus CurrentStatus => SiloStatus.Active; + + public string SiloName => "local"; + + public SiloAddress SiloAddress => _statuses.Keys.OrderBy(static silo => silo).First(); + + public void SetStatus(SiloAddress silo, SiloStatus status) => _statuses[silo] = status; + + public SiloAddress[] GetActiveSilos() => + [.. _statuses.Where(static entry => entry.Value == SiloStatus.Active).Select(static entry => entry.Key)]; + + public SiloStatus GetApproximateSiloStatus(SiloAddress siloAddress) => + _statuses.TryGetValue(siloAddress, out var status) ? status : SiloStatus.None; + + public Dictionary GetApproximateSiloStatuses(bool onlyActive = false) => + _statuses + .Where(entry => !onlyActive || entry.Value == SiloStatus.Active) + .ToDictionary(static entry => entry.Key, static entry => entry.Value); + + public bool TryGetSiloName( + SiloAddress siloAddress, + [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out string? siloName) + { + siloName = siloAddress.ToParsableString(); + return true; + } + + public bool IsFunctionalDirectory(SiloAddress siloAddress) => true; + + public bool IsDeadSilo(SiloAddress silo) => GetApproximateSiloStatus(silo) == SiloStatus.Dead; + + public bool SubscribeToSiloStatusEvents(ISiloStatusListener observer) => true; + + public bool UnSubscribeFromSiloStatusEvents(ISiloStatusListener observer) => true; + } + + private sealed class Phase4FakeEnvironmentStatisticsProvider : Orleans.Statistics.IEnvironmentStatisticsProvider + { + public Orleans.Statistics.EnvironmentStatistics GetEnvironmentStatistics() => default; + } + + private sealed class Phase4FakeActivationWorkingSet : IActivationWorkingSet + { + public int Count => 0; + + public void OnActivated(IActivationWorkingSetMember member) + { + } + + public void OnActive(IActivationWorkingSetMember member) + { + } + + public void OnDeactivating(IActivationWorkingSetMember member) + { + } + + public void OnDeactivated(IActivationWorkingSetMember member) + { + } + } + + private sealed class Phase4FakeDeploymentLoadPublisherTarget : IDeploymentLoadPublisher + { + public List<(SiloAddress Source, SiloRuntimeStatistics Statistics)> Updates { get; } = []; + + public Task UpdateRuntimeStatistics( + SiloAddress siloAddress, + SiloRuntimeStatistics siloStats, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Updates.Add((siloAddress, siloStats)); + return Task.CompletedTask; + } + } + + [Fact] + public async Task LoadNotificationAppliesQueuedUpdatesInOrderLatestWins() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => + { + notifications.Add(Phase5Notification.Update(silo, statistics)); + if (silo.Equals(harness.Local)) + { + callbackEntered.TrySetResult(); + releaseCallback.Task.GetAwaiter().GetResult(); + } + })); + + var firstUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); + Task update10; + Task update20; + Task update30; + try + { + await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + update10 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken); + update20 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(20), TestContext.Current.CancellationToken); + update30 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(30), TestContext.Current.CancellationToken); + Assert.False(update10.IsCompleted); + Assert.False(update20.IsCompleted); + Assert.False(update30.IsCompleted); + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); + } + finally + { + releaseCallback.TrySetResult(); + } + + Assert.All(await Task.WhenAll(firstUpdate, update10, update20, update30), + static result => Assert.Equal(DisseminationApplyResult.Applied, result)); + Assert.Equal(30, harness.Publisher.PeriodicStatistics[harness.ActiveOne].ActivationCount); + Assert.Equal( + [ + Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1)), + Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(10)), + Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(20)), + Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(30)), + ], + notifications); + } + + [Fact] + public async Task LoadNotificationPreservesCrossSiloPublicationOrder() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => + { + notifications.Add(Phase5Notification.Update(silo, statistics)); + if (silo.Equals(harness.Local)) + { + callbackEntered.TrySetResult(); + releaseCallback.Task.GetAwaiter().GetResult(); + } + })); + + var firstUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); + Task update10; + Task update20; + Task update30; + Task update40; + try + { + await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + update10 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken); + update20 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveTwo, CreatePhase5Statistics(20), TestContext.Current.CancellationToken); + update30 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(30), TestContext.Current.CancellationToken); + update40 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveTwo, CreatePhase5Statistics(40), TestContext.Current.CancellationToken); + } + finally + { + releaseCallback.TrySetResult(); + } + + Assert.All(await Task.WhenAll(firstUpdate, update10, update20, update30, update40), + static result => Assert.Equal(DisseminationApplyResult.Applied, result)); + Assert.Equal( + [ + Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1)), + Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(10)), + Phase5Notification.Update(harness.ActiveTwo, CreatePhase5Statistics(20)), + Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(30)), + Phase5Notification.Update(harness.ActiveTwo, CreatePhase5Statistics(40)), + ], + notifications); + Assert.Equal(30, harness.Publisher.PeriodicStatistics[harness.ActiveOne].ActivationCount); + Assert.Equal(40, harness.Publisher.PeriodicStatistics[harness.ActiveTwo].ActivationCount); + } + + [Fact] + public async Task LoadNotificationTerminalRemovalRejectsQueuedUpdate() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => + { + notifications.Add(Phase5Notification.Update(silo, statistics)); + if (silo.Equals(harness.Local)) + { + callbackEntered.TrySetResult(); + releaseCallback.Task.GetAwaiter().GetResult(); + } + }, + onRemove: silo => notifications.Add(Phase5Notification.Removal(silo)))); + + var firstUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); + Task queuedUpdate; + Task removal; + try + { + await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + queuedUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken); + removal = SetPhase5SiloStatusAsync( + harness, harness.ActiveOne, SiloStatus.Dead, TestContext.Current.CancellationToken); + Assert.False(queuedUpdate.IsCompleted); + Assert.False(removal.IsCompleted); + } + finally + { + releaseCallback.TrySetResult(); + } + + Assert.Equal(DisseminationApplyResult.Applied, await firstUpdate); + Assert.Equal(DisseminationApplyResult.Rejected, await queuedUpdate); + await removal; + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); + Assert.Equal( + [ + Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1)), + Phase5Notification.Removal(harness.ActiveOne), + ], + notifications); + } + + [Fact] + public async Task ConcurrentLoadUpdateCannotOverwriteTerminalRemoval() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + var statusCheckEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseStatusCheck = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var blockStatusCheck = 0; + harness.StatusOracle.GetStatusHandler = silo => + { + if (silo.Equals(harness.ActiveOne) && Interlocked.Exchange(ref blockStatusCheck, 0) == 1) + { + statusCheckEntered.TrySetResult(); + releaseStatusCheck.Task.GetAwaiter().GetResult(); + } + + return harness.StatusOracle.GetStoredStatus(silo); + }; + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => notifications.Add(Phase5Notification.Update(silo, statistics)), + onRemove: silo => notifications.Add(Phase5Notification.Removal(silo)))); + + Assert.Equal( + DisseminationApplyResult.Applied, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken)); + Volatile.Write(ref blockStatusCheck, 1); + var concurrentUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(20), TestContext.Current.CancellationToken); + Task termination; + try + { + await statusCheckEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + termination = SetPhase5SiloStatusAsync( + harness, harness.ActiveOne, SiloStatus.Dead, TestContext.Current.CancellationToken); + Assert.False(termination.IsCompleted); + } + finally + { + releaseStatusCheck.TrySetResult(); + } + + Assert.Equal(DisseminationApplyResult.Rejected, await concurrentUpdate); + await termination; + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); + Assert.Equal( + DisseminationApplyResult.Rejected, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(30), TestContext.Current.CancellationToken)); + Assert.Equal( + [ + Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1)), + Phase5Notification.Removal(harness.ActiveOne), + ], + notifications); + } + + [Theory] + [InlineData(SiloStatus.Dead)] + [InlineData(SiloStatus.None)] + public async Task LoadNotificationDoesNotResurrectInactiveSilo(SiloStatus status) + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => notifications.Add(Phase5Notification.Update(silo, statistics)), + onRemove: silo => notifications.Add(Phase5Notification.Removal(silo)))); + + Assert.Equal( + DisseminationApplyResult.Applied, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken)); + await SetPhase5SiloStatusAsync( + harness, harness.ActiveOne, SiloStatus.Dead, TestContext.Current.CancellationToken); + harness.StatusOracle.SetStatus(harness.ActiveOne, status); + + var futureUpdate = await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, + CreatePhase5Statistics(20), + TestContext.Current.CancellationToken); + + Assert.Equal(DisseminationApplyResult.Rejected, futureUpdate); + Assert.True(harness.Publisher.IsRuntimeStatisticsObsolete(harness.ActiveOne, long.MaxValue)); + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); + Assert.Equal( + [ + Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(10)), + Phase5Notification.Removal(harness.ActiveOne), + ], + notifications); + } + + [Fact] + public async Task LoadNotificationAllowsActiveNewerIncarnation() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => notifications.Add(Phase5Notification.Update(silo, statistics)), + onRemove: silo => notifications.Add(Phase5Notification.Removal(silo)))); + var terminated = harness.ActiveOne; + var restarted = SiloAddress.New(terminated.Endpoint, terminated.Generation + 1); + + Assert.Equal( + DisseminationApplyResult.Applied, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + terminated, CreatePhase5Statistics(10), TestContext.Current.CancellationToken)); + notifications.Clear(); + await SetPhase5SiloStatusAsync( + harness, terminated, SiloStatus.Dead, TestContext.Current.CancellationToken); + harness.StatusOracle.SetStatus(restarted, SiloStatus.Active); + + var result = await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + restarted, + CreatePhase5Statistics(20), + TestContext.Current.CancellationToken); + + Assert.Equal(DisseminationApplyResult.Applied, result); + Assert.Equal(20, harness.Publisher.PeriodicStatistics[restarted].ActivationCount); + Assert.Equal( + [ + Phase5Notification.Removal(terminated), + Phase5Notification.Update(restarted, CreatePhase5Statistics(20)), + ], + notifications); + } + + [Fact] + public async Task LoadNotificationCleansOlderGenerationAfterNewerTermination() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => notifications.Add(Phase5Notification.Update(silo, statistics)), + onRemove: silo => notifications.Add(Phase5Notification.Removal(silo)))); + var older = harness.ActiveOne; + var newer = SiloAddress.New(older.Endpoint, older.Generation + 1); + harness.StatusOracle.SetStatus(older, SiloStatus.Active); + harness.StatusOracle.SetStatus(newer, SiloStatus.Active); + Assert.Equal( + DisseminationApplyResult.Applied, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + older, CreatePhase5Statistics(10), TestContext.Current.CancellationToken)); + Assert.Equal( + DisseminationApplyResult.Applied, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + newer, CreatePhase5Statistics(20), TestContext.Current.CancellationToken)); + + await SetPhase5SiloStatusAsync( + harness, newer, SiloStatus.Dead, TestContext.Current.CancellationToken); + await SetPhase5SiloStatusAsync( + harness, older, SiloStatus.Dead, TestContext.Current.CancellationToken); + + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(older)); + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(newer)); + Assert.Contains(Phase5Notification.Removal(older), notifications); + Assert.Contains(Phase5Notification.Removal(newer), notifications); + } + + [Fact] + public async Task LoadNotificationCallbacksRunOutsideSubscriberLock() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => + { + notifications.Add(Phase5Notification.Update(silo, statistics)); + if (silo.Equals(harness.Local)) + { + callbackEntered.TrySetResult(); + releaseCallback.Task.GetAwaiter().GetResult(); + } + })); + + var update = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); + var registrationCompleted = new TaskCompletionSource<(bool Added, bool Removed)>( + TaskCreationOptions.RunContinuationsAsynchronously); + Task registration; + try + { + await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var additionalSubscriber = new Phase5StatisticsListener(); + registration = Task.Run( + () => + { + var added = harness.Publisher.SubscribeToStatisticsChangeEvents(additionalSubscriber); + var removed = harness.Publisher.UnsubscribeStatisticsChangeEvents(additionalSubscriber); + registrationCompleted.TrySetResult((added, removed)); + }, + TestContext.Current.CancellationToken); + var result = await registrationCompleted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.True(result.Added); + Assert.True(result.Removed); + Assert.False(update.IsCompleted); + Assert.Equal([Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1))], notifications); + } + finally + { + releaseCallback.TrySetResult(); + } + + await registration; + Assert.Equal(DisseminationApplyResult.Applied, await update); + Assert.Equal( + DisseminationApplyResult.Applied, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken)); + Assert.Equal( + [ + Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1)), + Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(10)), + ], + notifications); + } + + [Fact] + public async Task LoadNotificationsAreSerializedOnPublisherScheduler() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new ConcurrentQueue(); + var callbackContexts = new ConcurrentQueue(); + var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var callbackConcurrencyLock = new object(); + var currentCallbackCount = 0; + var maximumCallbackCount = 0; + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => + { + lock (callbackConcurrencyLock) + { + currentCallbackCount++; + maximumCallbackCount = Math.Max(maximumCallbackCount, currentCallbackCount); + } + + try + { + notifications.Enqueue(Phase5Notification.Update(silo, statistics)); + callbackContexts.Enqueue(RuntimeContext.Current); + if (silo.Equals(harness.Local)) + { + callbackEntered.TrySetResult(); + releaseCallback.Task.GetAwaiter().GetResult(); + } + } + finally + { + lock (callbackConcurrencyLock) + { + currentCallbackCount--; + } + } + })); + + var firstUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); + Task firstProducer; + Task secondProducer; + try + { + await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + firstProducer = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken); + secondProducer = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveTwo, CreatePhase5Statistics(20), TestContext.Current.CancellationToken); + Assert.False(firstProducer.IsCompleted); + Assert.False(secondProducer.IsCompleted); + Assert.Equal(1, maximumCallbackCount); + } + finally + { + releaseCallback.TrySetResult(); + } + + Assert.All(await Task.WhenAll(firstUpdate, firstProducer, secondProducer), + static result => Assert.Equal(DisseminationApplyResult.Applied, result)); + Assert.Equal(1, maximumCallbackCount); + Assert.Equal(0, currentCallbackCount); + Assert.Equal(3, notifications.Count); + Assert.All(callbackContexts, context => Assert.Same(harness.Publisher, context)); + Assert.Equal( + [harness.Local, harness.ActiveOne, harness.ActiveTwo], + notifications.Select(static notification => notification.Silo).OrderBy(static silo => silo)); + } + + [Fact] + public async Task LoadNotificationContinuesAfterListenerExceptions() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var throwingListenerNotifications = new List(); + var observingListenerNotifications = new List(); + var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => + { + throwingListenerNotifications.Add(Phase5Notification.Update(silo, statistics)); + if (silo.Equals(harness.Local)) + { + callbackEntered.TrySetResult(); + releaseCallback.Task.GetAwaiter().GetResult(); + } + else if (silo.Equals(harness.ActiveOne)) + { + throw new InvalidOperationException("phase 5 listener failure"); + } + })); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => + observingListenerNotifications.Add(Phase5Notification.Update(silo, statistics)))); + + var firstUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); + Task failingUpdate; + Task laterUpdate; + try + { + await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + failingUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken); + laterUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveTwo, CreatePhase5Statistics(20), TestContext.Current.CancellationToken); + } + finally + { + releaseCallback.TrySetResult(); + } + + Assert.Equal(DisseminationApplyResult.Applied, await firstUpdate); + var exception = await Assert.ThrowsAsync(() => failingUpdate); + Assert.Equal("phase 5 listener failure", exception.Message); + Assert.Equal(DisseminationApplyResult.Applied, await laterUpdate); + var expected = new[] + { + Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1)), + Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(10)), + Phase5Notification.Update(harness.ActiveTwo, CreatePhase5Statistics(20)), + }; + Assert.Equal(expected, throwingListenerNotifications); + Assert.Equal(expected, observingListenerNotifications); + Assert.Equal(10, harness.Publisher.PeriodicStatistics[harness.ActiveOne].ActivationCount); + Assert.Equal(20, harness.Publisher.PeriodicStatistics[harness.ActiveTwo].ActivationCount); + } + + private static Task SetPhase5SiloStatusAsync( + Phase5DeploymentLoadPublisherHarness harness, + SiloAddress silo, + SiloStatus status, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + harness.StatusOracle.SetStatus(silo, status); + harness.Publisher.SiloStatusChangeNotification(silo, status); + return harness.Publisher.RunOrQueueTask( + token => + { + token.ThrowIfCancellationRequested(); + return Task.FromResult(true); + }, + cancellationToken); + } + + private static Phase5DeploymentLoadPublisherHarness CreatePhase5DeploymentLoadPublisherHarness() + { + var local = CreateSilo(32101); + var activeOne = CreateSilo(32102); + var activeTwo = CreateSilo(32103); + var statusOracle = new Phase5FakeSiloStatusOracle(); + statusOracle.SetStatus(local, SiloStatus.Active); + statusOracle.SetStatus(activeOne, SiloStatus.Active); + statusOracle.SetStatus(activeTwo, SiloStatus.Active); + var publisher = new DeploymentLoadPublisher( + new FakeLocalSiloDetails(local), + statusOracle, + Options.Create(new DeploymentLoadPublisherOptions + { + DeploymentLoadPublisherRefreshTime = TimeSpan.FromSeconds(5), + }), + Substitute.For(), + NullLoggerFactory.Instance, + new ActivationDirectory(CreatePhase4Instruments()), + new Phase4FakeActivationWorkingSet(), + new Phase4FakeEnvironmentStatisticsProvider(), + Options.Create(new LoadSheddingOptions()), + new Phase4MutableServiceProvider(), + CreatePhase4SystemTargetShared(local)); + return new(publisher, statusOracle, local, activeOne, activeTwo); + } + + private static SiloRuntimeStatistics CreatePhase5Statistics(int version) => new( + activationCount: version, + recentlyUsedActivationCount: version * 10, + new Phase4FakeEnvironmentStatisticsProvider(), + Options.Create(new LoadSheddingOptions()), + new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(version)); + + private sealed record Phase5DeploymentLoadPublisherHarness( + DeploymentLoadPublisher Publisher, + Phase5FakeSiloStatusOracle StatusOracle, + SiloAddress Local, + SiloAddress ActiveOne, + SiloAddress ActiveTwo); + + private readonly record struct Phase5Notification(string Kind, SiloAddress Silo, int ActivationCount) + { + public static Phase5Notification Update(SiloAddress silo, SiloRuntimeStatistics statistics) => + new("Update", silo, statistics.ActivationCount); + + public static Phase5Notification Removal(SiloAddress silo) => new("Remove", silo, -1); + } + + private sealed class Phase5StatisticsListener( + Action? onUpdate = null, + Action? onRemove = null) : ISiloStatisticsChangeListener + { + public void SiloStatisticsChangeNotification(SiloAddress updatedSilo, SiloRuntimeStatistics newStats) => + onUpdate?.Invoke(updatedSilo, newStats); + + public void RemoveSilo(SiloAddress removedSilo) => onRemove?.Invoke(removedSilo); + } + + private sealed class Phase5FakeSiloStatusOracle : ISiloStatusOracle + { + private readonly ConcurrentDictionary _statuses = new(); + + public Func? GetStatusHandler { get; set; } + + public SiloStatus CurrentStatus => SiloStatus.Active; + + public string SiloName => "local"; + + public SiloAddress SiloAddress => _statuses.Keys.OrderBy(static silo => silo).First(); + + public void SetStatus(SiloAddress silo, SiloStatus status) => _statuses[silo] = status; + + public SiloStatus GetStoredStatus(SiloAddress silo) => + _statuses.TryGetValue(silo, out var status) ? status : SiloStatus.None; + + public SiloAddress[] GetActiveSilos() => + [.. _statuses.Where(static entry => entry.Value == SiloStatus.Active).Select(static entry => entry.Key)]; + + public SiloStatus GetApproximateSiloStatus(SiloAddress siloAddress) => + GetStatusHandler?.Invoke(siloAddress) ?? GetStoredStatus(siloAddress); + + public Dictionary GetApproximateSiloStatuses(bool onlyActive = false) => + _statuses + .Where(entry => !onlyActive || entry.Value == SiloStatus.Active) + .ToDictionary(static entry => entry.Key, static entry => entry.Value); + + public bool TryGetSiloName( + SiloAddress siloAddress, + [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out string? siloName) + { + siloName = siloAddress.ToParsableString(); + return true; + } + + public bool IsFunctionalDirectory(SiloAddress siloAddress) => true; + + public bool IsDeadSilo(SiloAddress silo) => GetStoredStatus(silo) == SiloStatus.Dead; + + public bool SubscribeToSiloStatusEvents(ISiloStatusListener observer) => true; + + public bool UnSubscribeFromSiloStatusEvents(ISiloStatusListener observer) => true; + } + + [Fact] + public async Task ReceiveBroadcastContinuesAfterNamespaceApplyThrows() + { + var sender = CreateSilo(33001); + var local = CreateSilo(33002); + var transport = new FakeTransport(local, sender); + var failedKey = new DisseminationKey("throws"); + var validKey = new DisseminationKey("valid"); + var ns = new Phase6ThrowingNamespace(local, failedKey) + { + Failure = static (_, _) => throw new InvalidOperationException("phase 6 apply failure"), + }; + var protocol = CreatePhase6Protocol(transport, ns); + + var response = await protocol.ReceiveBroadcast( + CreateBroadcastBatch( + sender, + ns.CreateItem(sender, failedKey, sequence: 1), + ns.CreateItem(sender, validKey, sequence: 2)), + TestContext.Current.CancellationToken); + + Assert.Equal([failedKey, validKey], ns.ApplyAttempts); + Assert.Equal(0, ns.GetVersion(failedKey)); + Assert.Equal(2, ns.GetVersion(validKey)); + var acknowledgments = response.Acknowledgments[ns.Name].ToDictionary(static entry => entry.Key); + Assert.Equal(0, acknowledgments[failedKey].Version); + Assert.Equal(2, acknowledgments[validKey].Version); + Assert.Empty(response.UnsupportedNamespaces); + } + + [Fact] + public async Task ReceiveBroadcastForwardsRemainingValuesAfterNamespaceApplyThrows() + { + var sender = CreateSilo(33003); + var local = CreateSilo(33004); + var child = CreateSilo(33005); + var transport = new FakeTransport(local, sender, child); + var failedKey = new DisseminationKey("throws"); + var validKey = new DisseminationKey("forwarded"); + var ns = new Phase6ThrowingNamespace(local, failedKey) + { + Failure = static (_, _) => throw new InvalidOperationException("phase 6 apply failure"), + }; + var protocol = CreatePhase6Protocol( + transport, + ns, + configure: static options => options.Overlay.FanOutFactor = static _ => 1); + + await protocol.ReceiveBroadcast( + CreateBroadcastBatch( + sender, + ns.CreateItem(sender, failedKey, sequence: 1), + ns.CreateItem(sender, validKey, sequence: 2)), + TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal([failedKey, validKey], ns.ApplyAttempts); + var forwarded = Assert.Single(transport.BroadcastBatches); + Assert.Equal(child, forwarded.Peer); + var forwardedValue = Assert.Single(GetBroadcastValues(forwarded.Batch)); + Assert.Equal(validKey, forwardedValue.Value.Key); + Assert.Equal(2, forwardedValue.Value.ToVersion); + } + + [Fact] + public async Task ReceiveBroadcastConvertsUnrelatedOperationCanceledExceptionToRejected() + { + var sender = CreateSilo(33006); + var local = CreateSilo(33007); + var transport = new FakeTransport(local, sender); + var failedKey = new DisseminationKey("unrelated-cancellation"); + var validKey = new DisseminationKey("valid"); + using var unrelatedCancellation = new CancellationTokenSource(); + var ns = new Phase6ThrowingNamespace(local, failedKey) + { + Failure = (_, _) => throw new OperationCanceledException("unrelated", unrelatedCancellation.Token), + }; + var protocol = CreatePhase6Protocol(transport, ns); + + var response = await protocol.ReceiveBroadcast( + CreateBroadcastBatch( + sender, + ns.CreateItem(sender, failedKey, sequence: 1), + ns.CreateItem(sender, validKey, sequence: 3)), + TestContext.Current.CancellationToken); + + Assert.False(unrelatedCancellation.IsCancellationRequested); + Assert.Equal([failedKey, validKey], ns.ApplyAttempts); + Assert.Equal(0, ns.GetVersion(failedKey)); + Assert.Equal(3, ns.GetVersion(validKey)); + var acknowledgments = response.Acknowledgments[ns.Name].ToDictionary(static entry => entry.Key); + Assert.Equal(0, acknowledgments[failedKey].Version); + Assert.Equal(3, acknowledgments[validKey].Version); + } + + [Fact] + public async Task ReceiveBroadcastPropagatesCallerCancellation() + { + var sender = CreateSilo(33008); + var local = CreateSilo(33009); + var transport = new FakeTransport(local, sender); + var canceledKey = new DisseminationKey("caller-cancellation"); + var laterKey = new DisseminationKey("must-not-apply"); + var applyStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var ns = new Phase6ThrowingNamespace(local, canceledKey) + { + Failure = async (_, cancellationToken) => + { + applyStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return DisseminationApplyResult.Applied; + }, + }; + var protocol = CreatePhase6Protocol(transport, ns); + using var cancellation = new CancellationTokenSource(); + + var receive = protocol.ReceiveBroadcast( + CreateBroadcastBatch( + sender, + ns.CreateItem(sender, canceledKey, sequence: 1), + ns.CreateItem(sender, laterKey, sequence: 2)), + cancellation.Token); + await applyStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + cancellation.Cancel(); + + var exception = await Assert.ThrowsAnyAsync(() => receive); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal([canceledKey], ns.ApplyAttempts); + Assert.Equal(0, ns.GetVersion(laterKey)); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task ReceiveBroadcastStopsBeforeNextValueWhenCallerIsCanceledAfterApply() + { + var sender = CreateSilo(33014); + var local = CreateSilo(33015); + var transport = new FakeTransport(local, sender); + var cancelingKey = new DisseminationKey("canceling-apply"); + var laterKey = new DisseminationKey("must-not-apply"); + using var cancellation = new CancellationTokenSource(); + var ns = new Phase6ThrowingNamespace(local, cancelingKey) + { + Failure = (_, _) => + { + cancellation.Cancel(); + return ValueTask.FromResult(DisseminationApplyResult.Applied); + }, + }; + var protocol = CreatePhase6Protocol(transport, ns); + + var exception = await Assert.ThrowsAnyAsync(() => protocol.ReceiveBroadcast( + CreateBroadcastBatch( + sender, + ns.CreateItem(sender, cancelingKey, sequence: 1), + ns.CreateItem(sender, laterKey, sequence: 2)), + cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal([cancelingKey], ns.ApplyAttempts); + Assert.Equal(0, ns.GetVersion(laterKey)); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task ReceiveBroadcastEmitsRejectedDiagnosticWhenNamespaceApplyThrows() + { + var sender = CreateSilo(33010); + var local = CreateSilo(33011); + var transport = new FakeTransport(local, sender); + var failedKey = new DisseminationKey("diagnostic"); + var validKey = new DisseminationKey("valid"); + var ns = new Phase6ThrowingNamespace(local, failedKey) + { + Failure = static (_, _) => throw new InvalidOperationException("phase 6 apply failure"), + }; + var protocol = CreatePhase6Protocol(transport, ns); + using var observer = new Phase6ApplyObserver(ns.Name, failedKey, local, sender); + + await protocol.ReceiveBroadcast( + CreateBroadcastBatch( + sender, + ns.CreateItem(sender, failedKey, sequence: 4), + ns.CreateItem(sender, validKey, sequence: 5)), + TestContext.Current.CancellationToken); + + var rejection = Assert.Single(observer.Events); + Assert.Equal(ns.Name, rejection.Namespace); + Assert.Equal(local, rejection.LocalSilo); + Assert.Equal(sender, rejection.Peer); + Assert.Equal(failedKey, rejection.Key); + Assert.Equal(0, rejection.FromVersion); + Assert.Equal(4, rejection.ToVersion); + Assert.Equal(nameof(DisseminationApplyResult.Rejected), rejection.Result); + Assert.Equal(sizeof(long), rejection.PayloadBytes); + Assert.Equal(5, ns.GetVersion(validKey)); + } + + [Fact] + public async Task ReceiveBroadcastLogsNamespaceApplyFailure() + { + var sender = CreateSilo(33012); + var local = CreateSilo(33013); + var transport = new FakeTransport(local, sender); + var failedKey = new DisseminationKey("logged"); + var validKey = new DisseminationKey("valid"); + var ns = new Phase6ThrowingNamespace(local, failedKey) + { + Failure = static (_, _) => throw new InvalidOperationException("phase 6 apply failure"), + }; + var logger = new Phase6ProtocolLogger(); + var protocol = CreatePhase6Protocol(transport, ns, logger); + + await protocol.ReceiveBroadcast( + CreateBroadcastBatch( + sender, + ns.CreateItem(sender, failedKey, sequence: 6), + ns.CreateItem(sender, validKey, sequence: 7)), + TestContext.Current.CancellationToken); + + var entry = Assert.Single(logger.Entries); + Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Debug, entry.Level); + Assert.Equal(1360205587, entry.EventId.Id); + Assert.Equal("LogDebugDisseminationValueApplyFailed", entry.EventId.Name); + var exception = Assert.IsType(entry.Exception); + Assert.Equal("phase 6 apply failure", exception.Message); + Assert.Equal(sender, entry.State["Sender"]); + Assert.Equal(ns.Name, entry.State["Namespace"]); + Assert.Equal(failedKey, entry.State["Key"]); + Assert.Equal(6L, entry.State["Version"]); + Assert.Equal(7, ns.GetVersion(validKey)); + } + + [Fact] + public async Task ReceiveBroadcastContinuesWhenApplyDiagnosticObserverThrows() + { + var sender = CreateSilo(33014); + var local = CreateSilo(33015); + var transport = new FakeTransport(local, sender); + var failedKey = new DisseminationKey("throws"); + var validKey = new DisseminationKey("valid"); + var ns = new Phase6ThrowingNamespace(local, failedKey) + { + Failure = static (_, _) => throw new InvalidOperationException("phase 6 apply failure"), + }; + var logger = new Phase6ProtocolLogger(); + var protocol = CreatePhase6Protocol(transport, ns, logger); + using var subscription = DisseminationEvents.Listener.Subscribe( + new ThrowingApplyObserver(ns.Name, failedKey, local, sender), + static name => name == "Dissemination.ValueApply"); + + var response = await protocol.ReceiveBroadcast( + CreateBroadcastBatch( + sender, + ns.CreateItem(sender, failedKey, sequence: 8), + ns.CreateItem(sender, validKey, sequence: 9)), + TestContext.Current.CancellationToken); + + Assert.Equal([failedKey, validKey], ns.ApplyAttempts); + Assert.Equal(9, ns.GetVersion(validKey)); + Assert.Equal(9, response.Acknowledgments[ns.Name].Single(entry => entry.Key == validKey).Version); + Assert.Contains( + logger.Entries, + entry => entry.EventId.Name == "LogDebugDisseminationDiagnosticFailed" + && Equals(entry.State["Namespace"], ns.Name)); + } + + [Fact] + public async Task AntiEntropyStopsBeforeNextValueWhenCallerIsCanceledAfterApply() + { + var sender = CreateSilo(39001); + var local = CreateSilo(39002); + var transport = new FakeTransport(local, sender); + var cancelingKey = new DisseminationKey("canceling-apply"); + var laterKey = new DisseminationKey("must-not-apply"); + using var cancellation = new CancellationTokenSource(); + CancellationToken applyCancellation = default; + var ns = new Phase6ThrowingNamespace(local, cancelingKey) + { + Failure = (_, cancellationToken) => + { + applyCancellation = cancellationToken; + cancellation.Cancel(); + return ValueTask.FromResult(DisseminationApplyResult.Applied); + }, + }; + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = sender, + Values = new() + { + [ns.Name] = + [ + ns.CreateItem(sender, cancelingKey, sequence: 1), + ns.CreateItem(sender, laterKey, sequence: 2), + ], + }, + }); + var protocol = CreateProtocol(transport, [ns]); + + try + { + var exception = await Assert.ThrowsAnyAsync( + () => protocol.RunAntiEntropyRound(cancellation.Token)); + + Assert.True(cancellation.IsCancellationRequested); + Assert.True(applyCancellation.IsCancellationRequested); + Assert.Equal(applyCancellation, exception.CancellationToken); + Assert.Equal([cancelingKey], ns.ApplyAttempts); + Assert.Equal(0, ns.GetVersion(laterKey)); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MembershipDuplicatesStopForwardingWhileNewLivenessStillPropagates(bool duplicateDuringSend) + { + var sender = CreateSilo(39021); + var local = CreateSilo(39022); + var child = CreateSilo(39023); + SiloAddress[] members = [sender, local, child]; + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(CreateSnapshot(1)); + var ns = CreateMembershipNamespace(manager, serializer); + var transport = new FakeTransport(local, sender, child); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sends = 0; + transport.SendBroadcastHandler = async (peer, batch, cancellationToken) => + { + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Add((peer, batch)); + } + + if (Interlocked.Increment(ref sends) == 1) + { + sendStarted.TrySetResult(); + if (duplicateDuringSend) + { + await releaseSend.Task.WaitAsync(cancellationToken); + } + } + }; + var protocol = CreateProtocol( + transport, + [ns], + options => options.Overlay.FanOutFactor = static _ => 1, + new FakeTimeProvider()); + + try + { + var original = CreateBatch(1); + await protocol.ReceiveBroadcast(original, TestContext.Current.CancellationToken); + var firstFlush = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (duplicateDuringSend) + { + await protocol.ReceiveBroadcast(original, TestContext.Current.CancellationToken); + } + + releaseSend.TrySetResult(); + await firstFlush; + var acknowledgment = await protocol.ReceiveBroadcast(original, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(1, Assert.Single(acknowledgment.Acknowledgments[ns.Name]).Version); + Assert.Equal(child, Assert.Single(transport.BroadcastBatches).Peer); + + var updated = CreateBatch(2); + await protocol.ReceiveBroadcast(updated, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await protocol.ReceiveBroadcast(updated, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(2, Volatile.Read(ref sends)); + var forwarded = Assert.Single(GetBroadcastValues(transport.BroadcastBatches[1].Batch)); + var update = Assert.IsType( + serializer.Deserialize(forwarded.Value.Payload)); + Assert.NotNull(update.Snapshot); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(2), update.Snapshot.Entries[sender].IAmAliveTime); + Assert.Equal(1, forwarded.Value.ToVersion); + } + finally + { + releaseSend.TrySetResult(); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + MembershipTableSnapshot CreateSnapshot(int aliveSeconds) => + CreateMembershipSnapshot(1, members.Select(member => CreateMembershipEntry( + member, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(aliveSeconds))).ToArray()); + + DisseminationBroadcastBatch CreateBatch(int aliveSeconds) => new() + { + Sender = sender, + Values = CreateValueGroups(ns.Name, new DisseminationBroadcastValue + { + Value = new DisseminationValue( + DisseminationKey.Default, + 0, + 1, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = CreateSnapshot(aliveSeconds) })), + TimeToLive = ns.Options.StaleItemTtl, + }), + }; + } + + [Fact] + public async Task PeerQueueAdmitsNewKeyAfterOversizedRepair() + { + var local = CreateSilo(39003); + var peer = CreateSilo(39004); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.Options.MaxPendingItemCount = 1; + ns.Options.MaxPayloadBytes = sizeof(long); + ns.PublishValue(new DisseminationValue("oversized", 0, 1, new byte[sizeof(long) + 1])); + ns.SetValue("valid", version: 1); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + + try + { + queue.Notify(peer, ns, "oversized"); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Empty(transport.BroadcastBatches); + + queue.Notify(peer, ns, "valid"); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var batch = Assert.Single(transport.BroadcastBatches); + var value = Assert.Single(GetBroadcastValues(batch.Batch)); + Assert.Equal(new DisseminationKey("valid"), value.Value.Key); + Assert.Equal(1, value.Value.ToVersion); + } + finally + { + await queue.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task DisablingDisseminationStopsPreviouslyQueuedBroadcasts() + { + var local = CreateSilo(39025); + var peer = CreateSilo(39026); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + DisseminationOptions? options = null; + var protocol = CreateProtocol( + transport, ns, value => options = value, timeProvider: new FakeTimeProvider()); + + try + { + Assert.True(await PublishValue( + protocol, ns, ns.CreateValue("value", 1), TestContext.Current.CancellationToken)); + Assert.NotNull(options); + options.Enabled = false; + + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Empty(transport.BroadcastBatches); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task PublishReturnsFalseWhenPeerQueueRejectsNewKey() + { + var local = CreateSilo(39005); + var peer = CreateSilo(39006); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.Options.MaxPendingItemCount = 1; + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (_, _, cancellationToken) => + { + sendStarted.TrySetResult(); + await releaseSend.Task.WaitAsync(cancellationToken); + }; + var protocol = CreateProtocol(transport, ns, timeProvider: new FakeTimeProvider()); + + try + { + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("first", 1), TestContext.Current.CancellationToken)); + var flush = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.False(await PublishValue(protocol, ns, ns.CreateValue("second", 1), TestContext.Current.CancellationToken)); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("first", 2), TestContext.Current.CancellationToken)); + releaseSend.TrySetResult(); + await flush; + } + finally + { + releaseSend.TrySetResult(); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task BroadcastDeadlineCompletesFlushBeforeRemoteCancellationAcknowledgment() + { + var local = CreateSilo(39011); + var peer = CreateSilo(39012); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(10); + ns.SetValue("value", version: 1); + var clock = new FakeTimeProvider(); + var response = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = (_, _, cancellationToken) => + { + sendStarted.TrySetResult(cancellationToken); + return response.Task; + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); + + try + { + queue.Notify(peer, ns, "value"); + var flush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var sendCancellation = await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + clock.Advance(ns.Options.StaleItemTtl); + await flush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(sendCancellation.IsCancellationRequested); + Assert.False(response.Task.IsCompleted); + ns.Options.Enabled = false; + await queue.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.False(response.Task.IsCompleted); + } + finally + { + response.TrySetResult(new DisseminationBroadcastResponse()); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(1, false)] + [InlineData(2, true)] + [InlineData(32, false)] + public async Task TimedOutBroadcastReleasesLocalSlotBeforeRpcCompletes(int maxConcurrentSends, bool lateFault) + { + var local = CreateSilo(39013); + var peer = CreateSilo(39014); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(10); + ns.SetValue("value", version: 1); + var clock = new FakeTimeProvider(); + var response = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + var sentVersions = new ConcurrentQueue(); + transport.SendBroadcastResponseHandler = (_, batch, cancellationToken) => + { + var attempt = Interlocked.Increment(ref sendCount); + sentVersions.Enqueue(Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); + sendStarted.TrySetResult(cancellationToken); + return attempt == 1 ? response.Task : Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var queue = CreateBroadcastQueue(transport, [ns], options => options.MaxConcurrentSends = maxConcurrentSends, clock); + + try + { + Assert.True(queue.Notify(peer, ns, "value")); + var firstFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var sendCancellation = await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + clock.Advance(ns.Options.StaleItemTtl); + await firstFlush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(sendCancellation.IsCancellationRequested); + + ns.SetValue("value", 2); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(2, Volatile.Read(ref sendCount)); + Assert.Equal(2, ns.RepairRequestCount); + Assert.Equal(new long[] { 1, 2 }, sentVersions.ToArray()); + Assert.False(response.Task.IsCompleted); + + if (lateFault) + { + response.SetException(new InvalidOperationException("The previous attempt failed after its local deadline.")); + } + else + { + response.SetResult(new DisseminationBroadcastResponse + { + Acknowledgments = new() { [ns.Name] = [new("value", 99)] }, + }); + } + + ns.SetValue("value", 3); + Assert.True(queue.Notify(peer, ns, "value")); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(3, Volatile.Read(ref sendCount)); + Assert.Equal(new long[] { 1, 2, 3 }, sentVersions.ToArray()); + } + finally + { + ns.Options.Enabled = false; + response.TrySetResult(new DisseminationBroadcastResponse()); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task PublishReturnsFalseAfterBroadcastQueueStops() + { + var local = CreateSilo(39007); + var peer = CreateSilo(39008); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + await protocol.StopAsync(TestContext.Current.CancellationToken); + + Assert.False(await PublishValue(protocol, ns, ns.CreateValue("value", 1), TestContext.Current.CancellationToken)); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task BroadcastNotificationDiagnosticsRunOutsideQueueLock() + { + var local = CreateSilo(39009); + var peer = CreateSilo(39010); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.SetValue("value", version: 1); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var observer = DisseminationEvents.Listener.Subscribe( + new QueueReentryObserver(local, () => + { + callbackEntered.TrySetResult(); + releaseCallback.Task.GetAwaiter().GetResult(); + }), + static name => name == DisseminationEvents.BroadcastScheduledEventName); + var notification = Task.Run(() => queue.Notify(peer, ns, "value"), TestContext.Current.CancellationToken); + + try + { + await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await Task.Run(() => queue.ObservePeerVersion(peer, ns.Name, "value", 0), TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + finally + { + releaseCallback.TrySetResult(); + Assert.True(await notification); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task BroadcastLocalCompletionReleasesCapacityAcrossPeerRemovalAndRecreation(bool expireBeforePrune) + { + var local = CreateSilo(39301); + var blocked = CreateSilo(39302); + var healthy = CreateSilo(39303); + var restarted = SiloAddress.New(blocked.Endpoint, blocked.Generation + 1); + var transport = new FakeTransport(local, blocked, healthy); + var ns = new FakeNamespace(local); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); + ns.SetValue("value", 1); + var clock = new FakeTimeProvider(); + var blockedResponse = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var blockedStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sends = new ConcurrentDictionary(); + var blockedVersions = new ConcurrentQueue(); + transport.SendBroadcastResponseHandler = (peer, batch, cancellation) => + { + var attempt = sends.AddOrUpdate(peer, 1, static (_, count) => count + 1); + if (peer.Equals(blocked)) + { + blockedVersions.Enqueue(Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); + if (attempt == 1) + { + blockedStarted.TrySetResult(cancellation); + return blockedResponse.Task; + } + } + + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var queue = CreateBroadcastQueue(transport, [ns], options => + { + options.MaxConcurrentSends = 1; + options.Overlay.AntiEntropyInterval = TimeSpan.FromHours(1); + }, clock); + var membership = new DisseminationMembership( + transport.MembershipManager, new FakeLocalSiloDetails(local), Options.Create(new DisseminationOptions())); + using var admission = new SendGateObserver(local, "broadcast"); + + try + { + Assert.True(queue.Notify(blocked, ns, "value")); + var initialFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var remoteCancellation = await blockedStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var healthyQueued = admission.Wait(healthy, "queued", TestContext.Current.CancellationToken); + Assert.True(queue.Notify(healthy, ns, "value")); + var healthyFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await healthyQueued.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, sends[blocked]); + Assert.False(sends.ContainsKey(healthy)); + Assert.Equal(1, ns.RepairRequestCount); + if (expireBeforePrune) + { + clock.Advance(ns.Options.StaleItemTtl); + await Task.WhenAll(initialFlush, healthyFlush).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + transport.Peers.Remove(blocked); + await queue.Prune(membership.CurrentSnapshots, TestContext.Current.CancellationToken); + await Task.WhenAll(initialFlush, healthyFlush).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(remoteCancellation.IsCancellationRequested); + Assert.False(blockedResponse.Task.IsCompleted); + Assert.Equal(1, sends[healthy]); + + transport.Peers.Add(blocked); + Assert.True(queue.Notify(blocked, ns, "value")); + transport.Peers.Add(restarted); + Assert.True(queue.Notify(restarted, ns, "value")); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(2, sends[blocked]); + Assert.Equal(1, sends[restarted]); + Assert.Equal(2, transport.GetTargetResolutionCount(blocked)); + Assert.Equal(4, ns.RepairRequestCount); + Assert.False(blockedResponse.Task.IsCompleted); + + blockedResponse.SetResult(new DisseminationBroadcastResponse + { + Acknowledgments = new() { [ns.Name] = [new("value", 99)] }, + }); + ns.SetValue("value", 2); + Assert.True(queue.Notify(blocked, ns, "value")); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(3, sends[blocked]); + Assert.Equal(5, sends.Values.Sum()); + Assert.Equal(5, ns.RepairRequestCount); + Assert.Equal(new long[] { 1, 1, 2 }, blockedVersions.ToArray()); + } + finally + { + ns.Options.Enabled = false; + blockedResponse.TrySetResult(new DisseminationBroadcastResponse()); + await queue.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task CancelingFlushObserverPreservesQueuedBroadcastAttempt() + { + var local = CreateSilo(39331); + var first = CreateSilo(39332); + var waiting = CreateSilo(39333); + var transport = new FakeTransport(local, first, waiting); + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); + ns.SetValue("value", 1); + var firstResponse = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sent = new ConcurrentQueue(); + transport.SendBroadcastResponseHandler = (peer, batch, cancellationToken) => + { + sent.Enqueue(peer); + if (peer.Equals(first)) + { + firstStarted.TrySetResult(cancellationToken); + return firstResponse.Task; + } + + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var queue = CreateBroadcastQueue(transport, [ns], options => options.MaxConcurrentSends = 1, new FakeTimeProvider()); + using var admission = new SendGateObserver(local, "broadcast"); + using var flushCancellation = new CancellationTokenSource(); + + try + { + Assert.True(queue.Notify(first, ns, "value")); + var firstFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var sendCancellation = await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var queued = admission.Wait(waiting, "queued", TestContext.Current.CancellationToken); + Assert.True(queue.Notify(waiting, ns, "value")); + var canceledFlush = queue.FlushPendingBroadcast(flushCancellation.Token); + await queued.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await flushCancellation.CancelAsync(); + var exception = await Assert.ThrowsAnyAsync( + () => canceledFlush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(flushCancellation.Token, exception.CancellationToken); + Assert.False(sendCancellation.IsCancellationRequested); + Assert.Equal(new[] { first }, sent.ToArray()); + Assert.Equal(1, ns.RepairRequestCount); + + firstResponse.SetResult(new DisseminationBroadcastResponse + { + Acknowledgments = new() { [ns.Name] = [new("value", 1)] }, + }); + await firstFlush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(new[] { first, waiting }, sent.ToArray()); + Assert.Equal(2, ns.RepairRequestCount); + } + finally + { + ns.Options.Enabled = false; + firstResponse.TrySetResult(new DisseminationBroadcastResponse()); + await queue.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task WaitingBroadcastMaterializesLatestValueAfterAdmissionAndRotatesBehindReadyPeer() + { + var local = CreateSilo(39311); + var large = CreateSilo(39312); + var waiting = CreateSilo(39313); + var transport = new FakeTransport(local, large, waiting); + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); + ns.SetValue("large-first", 1); + ns.SetValue("large-second", 2); + ns.SetValue("large-third", 3); + ns.SetValue("waiting", 1); + var clock = new RecordingFakeTimeProvider(); + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var waitingStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseWaiting = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sent = new ConcurrentQueue<(SiloAddress Peer, DisseminationKey Key, long Version)>(); + transport.SendBroadcastResponseHandler = async (peer, batch, cancellation) => + { + var value = Assert.Single(GetBroadcastValues(batch)).Value; + sent.Enqueue((peer, value.Key, value.ToVersion)); + if (value.Key == new DisseminationKey("large-first")) + { + firstStarted.TrySetResult(); + await releaseFirst.Task.WaitAsync(cancellation); + } + else if (peer.Equals(waiting)) + { + waitingStarted.TrySetResult(); + await releaseWaiting.Task.WaitAsync(cancellation); + } + + return FakeTransport.CreateAcknowledgment(batch); + }; + var queue = CreateBroadcastQueue(transport, [ns], options => + { + options.MaxConcurrentSends = 1; + options.MaxBatchItems = 1; + }, clock); + using var admission = new SendGateObserver(local, "broadcast"); + + try + { + Assert.True(queue.Notify(large, ns, "large-first")); + await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(queue.Notify(large, ns, "large-second")); + Assert.True(queue.Notify(large, ns, "large-third")); + var waitingQueued = admission.Wait(waiting, "queued", TestContext.Current.CancellationToken); + Assert.True(queue.Notify(waiting, ns, "waiting")); + var flush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await waitingQueued.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, ns.RepairRequestCount); + Assert.Equal(0, transport.GetTargetResolutionCount(waiting)); + + ns.SetValue("waiting", 7); + var largeQueued = admission.Wait(large, "queued", TestContext.Current.CancellationToken); + releaseFirst.TrySetResult(); + await waitingStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await largeQueued.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(2, ns.RepairRequestCount); + Assert.Equal( + new[] { (large, new DisseminationKey("large-first"), 1L), (waiting, new DisseminationKey("waiting"), 7L) }, + sent.ToArray()); + releaseWaiting.TrySetResult(); + await flush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal( + new[] + { + (large, new DisseminationKey("large-first"), 1L), + (waiting, new DisseminationKey("waiting"), 7L), + (large, new DisseminationKey("large-second"), 2L), + (large, new DisseminationKey("large-third"), 3L), + }, + sent.ToArray()); + Assert.Equal(4, ns.RepairRequestCount); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(4, sent.Count); + } + finally + { + ns.Options.Enabled = false; + releaseFirst.TrySetResult(); + releaseWaiting.TrySetResult(); + await queue.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RepairAttemptsReleaseLocalCapacityBeforeLateCompletion(bool cancelRound) + { + var local = CreateSilo(39321); + var blocked = Enumerable.Range(39322, 3).Select(CreateSilo).ToArray(); + var healthy = CreateSilo(39325); + var transport = new FakeTransport(local, blocked); + var ns = new FakeNamespace(local); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(1); + ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); + var clock = new FakeTimeProvider(); + var responses = blocked.ToDictionary(static peer => peer, + static _ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + var remoteCancellations = new ConcurrentDictionary(); + var attempts = new ConcurrentDictionary(); + var allStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.ExchangeAntiEntropyHandler = (peer, _, cancellation) => + { + var attempt = attempts.AddOrUpdate(peer, 1, static (_, count) => count + 1); + if (attempt == 1 && responses.TryGetValue(peer, out var response)) + { + remoteCancellations[peer] = cancellation; + if (remoteCancellations.Count == blocked.Length) + { + allStarted.TrySetResult(); + } + + return new(response.Task); + } + + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = CreateValueGroups(ns.CreateItem(peer, FakeNamespace.DefaultKey, peer.Equals(healthy) ? 3 : 2)), + }); + }; + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.AntiEntropyPeerCount = 3; + options.Overlay.FanOutFactor = static _ => 8; + }, clock); + using var admission = new SendGateObserver(local, "repair"); + using var roundCancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + + try + { + var released = blocked.Select(peer => admission.Wait(peer, "released", TestContext.Current.CancellationToken)).ToArray(); + var firstRound = protocol.RunAntiEntropyRound(roundCancellation.Token); + await allStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + for (var round = 0; round < 3; round++) + { + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + Assert.False(firstRound.IsCompleted); + Assert.Equal(3, transport.AntiEntropyRequests.Count); + Assert.All(blocked, peer => Assert.Single(transport.AntiEntropyRequests, request => request.Peer.Equals(peer))); + Assert.All(remoteCancellations.Values, static cancellation => Assert.False(cancellation.IsCancellationRequested)); + + Assert.True(await PublishValue( + protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, 1), TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(blocked.Order(), transport.BroadcastBatches.Select(static batch => batch.Peer).Order()); + Assert.False(firstRound.IsCompleted); + + if (cancelRound) + { + await roundCancellation.CancelAsync(); + await Assert.ThrowsAnyAsync( + () => firstRound.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + else + { + clock.Advance(ns.Options.StaleItemTtl); + await firstRound.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + await Task.WhenAll(released).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.All(remoteCancellations.Values, static cancellation => Assert.True(cancellation.IsCancellationRequested)); + Assert.All(responses.Values, static response => Assert.False(response.Task.IsCompleted)); + AssertFakeState(ns, (FakeNamespace.DefaultKey, 1)); + + clock.Advance(ns.Options.StaleItemTtl); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + AssertFakeState(ns, (FakeNamespace.DefaultKey, 2)); + Assert.Equal(6, transport.AntiEntropyRequests.Count); + Assert.All(blocked, peer => Assert.Equal(2, attempts[peer])); + Assert.All(responses.Values, static response => Assert.False(response.Task.IsCompleted)); + + transport.Peers.Clear(); + transport.Peers.Add(healthy); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + AssertFakeState(ns, (FakeNamespace.DefaultKey, 3)); + Assert.Equal(7, transport.AntiEntropyRequests.Count); + Assert.Equal(healthy, transport.AntiEntropyRequests[^1].Peer); + + foreach (var (peer, response) in responses) + { + response.SetResult(new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = CreateValueGroups(ns.CreateItem(peer, FakeNamespace.DefaultKey, 99)), + }); + } + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + AssertFakeState(ns, (FakeNamespace.DefaultKey, 3)); + Assert.Equal(2, ns.ApplyCounts[FakeNamespace.DefaultKey]); + } + finally + { + ns.Options.Enabled = false; + await roundCancellation.CancelAsync(); + foreach (var response in responses.Values) + { + response.TrySetCanceled(TestContext.Current.CancellationToken); + } + + await protocol.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task AntiEntropyConvergesAfterPartitionWithEveryTreeBroadcastDropped() + { + var members = Enumerable.Range(39201, 4).Select(CreateSilo).ToArray(); + DisseminationKey[] keys = ["first", "second", "third", "fourth"]; + await using var network = new RecoveryNetwork(members, silo => + { + var ns = new FakeNamespace(silo); + ns.ExpectedKeys.UnionWith(keys); + return ns; + }); + network.Partitioned = true; + + for (var i = 0; i < members.Length; i++) + { + var node = network[members[i]]; + var ns = Assert.IsType(node.Namespace); + Assert.True(await PublishValue(node.Protocol, ns, ns.CreateValue(keys[i], i + 1), network.Cancellation)); + } + + await network.Flush(network.Cancellation); + Assert.Equal(12, network.Broadcasts.Count); + Assert.All(network.Broadcasts, static send => Assert.False(send.Delivered)); + await network.RunSweep(network.Cancellation); + for (var i = 0; i < members.Length; i++) + { + AssertFakeState(Assert.IsType(network[members[i]].Namespace), (keys[i], i + 1)); + } + + // Publications have ceased. Only the repair links heal; the tree remains completely disconnected. + network.Partitioned = false; + await network.RunSweep(network.Cancellation); + await network.RunSweep(network.Cancellation); + var expected = keys.Select((key, index) => (key, (long)index + 1)).ToArray(); + foreach (var member in members) + { + AssertFakeState(Assert.IsType(network[member].Namespace), expected); + } + + var repairCount = network.Replies.Sum(static reply => GetAntiEntropyResponseValues(reply).Count()); + Assert.Equal(24, repairCount); + await network.RunSweep(network.Cancellation); + await network.Flush(network.Cancellation); + Assert.Equal(4, network.Sweeps); + Assert.Equal(48, network.RequestCount); + Assert.Equal(repairCount, network.Replies.Sum(static reply => GetAntiEntropyResponseValues(reply).Count())); + Assert.Equal(12, network.Broadcasts.Count); + } + + [Fact] + public async Task LostBroadcastAcknowledgmentsAndReorderedRepairRepliesConvergeAndDrain() + { + var members = Enumerable.Range(39211, 3).Select(CreateSilo).ToArray(); + await using var network = new RecoveryNetwork(members, silo => + { + var ns = new FakeNamespace(silo); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + return ns; + }); + network.DropTreeBroadcasts = false; + network.LoseAcknowledgments = true; + var source = network[members[0]]; + var older = network[members[1]]; + var receiver = network[members[2]]; + var sourceNamespace = Assert.IsType(source.Namespace); + var olderNamespace = Assert.IsType(older.Namespace); + var receiverNamespace = Assert.IsType(receiver.Namespace); + Assert.True(await PublishValue( + source.Protocol, sourceNamespace, sourceNamespace.CreateValue(FakeNamespace.DefaultKey, 1), network.Cancellation)); + await network.Flush(network.Cancellation); + Assert.Equal(2, network.Broadcasts.Count); + Assert.All(network.Broadcasts, static send => Assert.True(send.Delivered)); + AssertFakeState(receiverNamespace, (FakeNamespace.DefaultKey, 1)); + + sourceNamespace.SetValue(FakeNamespace.DefaultKey, 3); + olderNamespace.SetValue(FakeNamespace.DefaultKey, 2); + var bothRepliesCaptured = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var newerReplyReturned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseNewer = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseOlder = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var replies = new ConcurrentDictionary(); + var exchange = receiver.Transport.ExchangeAntiEntropyHandler; + receiver.Transport.ExchangeAntiEntropyHandler = async (peer, request, cancellation) => + { + var reply = await exchange(peer, request, cancellation); + replies[peer] = reply; + if (replies.Count == 2) + { + bothRepliesCaptured.TrySetResult(); + } + + await (peer.Equals(source.Address) ? releaseNewer.Task : releaseOlder.Task).WaitAsync(cancellation); + if (peer.Equals(source.Address)) + { + newerReplyReturned.TrySetResult(); + } + + return reply; + }; + + try + { + network.Clock.Advance(TimeSpan.FromSeconds(1)); + var round = receiver.Protocol.RunAntiEntropyRound(network.Cancellation); + await bothRepliesCaptured.Task.WaitAsync(TimeSpan.FromSeconds(5), network.Cancellation); + Assert.Equal(3, Assert.Single(GetAntiEntropyResponseValues(replies[source.Address])).Value.ToVersion); + Assert.Equal(2, Assert.Single(GetAntiEntropyResponseValues(replies[older.Address])).Value.ToVersion); + releaseNewer.TrySetResult(); + await newerReplyReturned.Task.WaitAsync(TimeSpan.FromSeconds(5), network.Cancellation); + Assert.False(round.IsCompleted); + releaseOlder.TrySetResult(); + await round.WaitAsync(TimeSpan.FromSeconds(5), network.Cancellation); + + AssertFakeState(receiverNamespace, (FakeNamespace.DefaultKey, 3)); + Assert.Equal(2, receiverNamespace.ApplyCounts[FakeNamespace.DefaultKey]); + receiver.Transport.ExchangeAntiEntropyHandler = (peer, _, _) => ValueTask.FromResult(replies[peer]); + network.Clock.Advance(TimeSpan.FromSeconds(1)); + await receiver.Protocol.RunAntiEntropyRound(network.Cancellation); + AssertFakeState(receiverNamespace, (FakeNamespace.DefaultKey, 3)); + Assert.Equal(2, receiverNamespace.ApplyCounts[FakeNamespace.DefaultKey]); + + receiver.Transport.ExchangeAntiEntropyHandler = exchange; + network.LoseAcknowledgments = false; + await network.RunSweep(network.Cancellation); + await network.RunSweep(network.Cancellation); + await network.Flush(network.Cancellation); + foreach (var member in members) + { + AssertFakeState(Assert.IsType(network[member].Namespace), (FakeNamespace.DefaultKey, 3)); + } + + Assert.Equal(16, network.RequestCount); + Assert.Equal(2, network.Broadcasts.Count); + Assert.Equal(2, receiverNamespace.ApplyCounts[FakeNamespace.DefaultKey]); + } + finally + { + releaseNewer.TrySetResult(); + releaseOlder.TrySetResult(); + } + } + + [Fact] + public async Task MembershipRepairConvergesAcrossJoinDepartureAndRestartWithoutRevivingDepartedGeneration() + { + var first = CreateSilo(39221); + var second = CreateSilo(39222); + var departed = CreateSilo(39223); + var joined = CreateSilo(39224); + var restarted = SiloAddress.New(departed.Endpoint, departed.Generation + 1); + SiloAddress[] members = [first, second, departed, joined, restarted]; + var baseline = CreateMembershipSnapshot(1, + CreateMembershipEntry(first, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(second, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(departed, SiloStatus.Active, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var managers = members.ToDictionary(static silo => silo, _ => new FakeMembershipManager(baseline)); + await using var network = new RecoveryNetwork( + members, silo => CreateMembershipNamespace(managers[silo], serializer)); + network.SetTopology((first, SiloStatus.Active), (second, SiloStatus.Active), (departed, SiloStatus.Active)); + var joiningSnapshot = CreateMembershipSnapshot(2, + CreateMembershipEntry(first, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(second, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(departed, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(joined, SiloStatus.Joining, DateTime.UnixEpoch.AddSeconds(1))); + managers[first].CurrentSnapshot = joiningSnapshot; + network.SetTopology( + (first, SiloStatus.Active), (second, SiloStatus.Active), + (departed, SiloStatus.Active), (joined, SiloStatus.Joining)); + await network.RunSweep(network.Cancellation); + foreach (var member in new[] { first, second, departed, joined }) + { + AssertMembershipState(joiningSnapshot, managers[member].CurrentSnapshot); + } + + var delayedDepartedValue = network[departed].Namespace.CreateRepair(new DisseminationRepairRequest( + DisseminationKey.Default, null, null, 1, 1024 * 1024, 1024 * 1024)).Values.Single(); + var restartedSnapshot = CreateMembershipSnapshot(3, + CreateMembershipEntry(first, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(second, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(joined, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1)), + CreateMembershipEntry(restarted, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(2))); + managers[first].CurrentSnapshot = restartedSnapshot; + network.SetTopology( + (first, SiloStatus.Active), (second, SiloStatus.Active), + (joined, SiloStatus.Active), (restarted, SiloStatus.Active)); + var requestsBeforeRestart = network.Nodes.ToDictionary(static node => node.Address, static node => node.Transport.AntiEntropyRequests.Count); + await network.RunSweep(network.Cancellation); + var acknowledgment = await network[second].Protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = departed, + Values = CreateValueGroups(network[second].Namespace.Name, CreateDisseminationValue(departed, delayedDepartedValue)), + }, network.Cancellation); + Assert.Equal(3, Assert.Single(acknowledgment.Acknowledgments[network[second].Namespace.Name]).Version); + await network.RunSweep(network.Cancellation); + await network.Flush(network.Cancellation); + + foreach (var member in new[] { first, second, joined, restarted }) + { + AssertMembershipState(restartedSnapshot, managers[member].CurrentSnapshot); + Assert.DoesNotContain(departed, managers[member].CurrentSnapshot.Entries.Keys); + Assert.Contains(restarted, managers[member].CurrentSnapshot.Entries.Keys); + } + + Assert.All(network.Nodes, node => Assert.DoesNotContain( + node.Transport.AntiEntropyRequests.Skip(requestsBeforeRestart[node.Address]), + request => request.Peer.Equals(departed))); + Assert.Equal(3, network.Sweeps); + Assert.Equal(36, network.RequestCount); + Assert.Empty(network.Broadcasts); + } + + [Fact] + public async Task MembershipAntiEntropyConvergesSameVersionLivenessAndTerminatesDuplicates() + { + var members = Enumerable.Range(39231, 3).Select(CreateSilo).ToArray(); + var baseline = CreateMembershipSnapshot(7, members.Select(silo => CreateMembershipEntry( + silo, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(1))).ToArray()); + var advanced = CreateMembershipSnapshot(7, members.Select(silo => CreateMembershipEntry( + silo, SiloStatus.Active, DateTime.UnixEpoch, + DateTime.UnixEpoch.AddSeconds(silo.Equals(members[0]) ? 2 : 1))).ToArray()); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var managers = members.ToDictionary(static silo => silo, _ => new FakeMembershipManager(baseline)); + var applied = members.ToDictionary(static silo => silo, static _ => 0); + foreach (var member in members) + { + managers[member].ProcessGossipSnapshotHandler = (snapshot, _) => + { + applied[member]++; + managers[member].CurrentSnapshot = snapshot; + return Task.CompletedTask; + }; + } + + await using var network = new RecoveryNetwork(members, silo => CreateMembershipNamespace(managers[silo], serializer)); + await network.RunSweep(network.Cancellation); + Assert.All(network.Replies, static response => Assert.Empty(response.Values)); + var source = network[members[0]]; + var oldFingerprint = source.Namespace.Digests.Single().Fingerprint; + managers[source.Address].CurrentSnapshot = advanced; + Assert.NotEqual(oldFingerprint, source.Namespace.Digests.Single().Fingerprint); + Assert.True(await source.Protocol.Publish(source.Namespace, DisseminationKey.Default, 7, network.Cancellation)); + await network.Flush(network.Cancellation); + await network.RunSweep(network.Cancellation); + foreach (var member in members) + { + AssertMembershipState(advanced, managers[member].CurrentSnapshot); + } + + var repairCount = network.Replies.Sum(static response => GetAntiEntropyResponseValues(response).Count()); + await network.RunSweep(network.Cancellation); + var duplicate = source.Namespace.CreateRepair(new DisseminationRepairRequest( + DisseminationKey.Default, null, null, 1, 1024 * 1024, 1024 * 1024)).Values.Single(); + foreach (var member in members.Skip(1)) + { + var acknowledgment = await network[member].Protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = source.Address, + Values = CreateValueGroups(source.Namespace.Name, CreateDisseminationValue(source.Address, duplicate)), + }, network.Cancellation); + Assert.Equal(7, Assert.Single(acknowledgment.Acknowledgments[source.Namespace.Name]).Version); + AssertMembershipState(advanced, managers[member].CurrentSnapshot); + Assert.Equal(1, applied[member]); + } + + // Flush recipients to isolate duplicate forwarding from the source's intentionally dirty retry queue. + foreach (var member in members.Skip(1)) + { + await network[member].Protocol.FlushPendingBroadcast(network.Cancellation); + } + + Assert.Equal(0, applied[source.Address]); + Assert.Equal(18, network.RequestCount); + Assert.Equal(repairCount, network.Replies.Sum(static response => GetAntiEntropyResponseValues(response).Count())); + Assert.Equal(2, network.Broadcasts.Count); + Assert.All(network.Broadcasts, static send => Assert.False(send.Delivered)); + } + + [Fact] + public async Task RejectedQueueKeysRecoverThroughTruncatedDigestsDespiteAdvancingFirstKey() + { + var members = Enumerable.Range(39241, 2).Select(CreateSilo).ToArray(); + DisseminationKey[] keys = ["hot", "waiting-first", "waiting-last"]; + await using var network = new RecoveryNetwork(members, silo => + { + var ns = new FakeNamespace(silo); + ns.ExpectedKeys.UnionWith(keys); + ns.Options.MaxPendingItemCount = 1; + return ns; + }, options => options.MaxBatchItems = 1); + var source = network[members[0]]; + var receiver = network[members[1]]; + var sourceNamespace = Assert.IsType(source.Namespace); + var receiverNamespace = Assert.IsType(receiver.Namespace); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var send = source.Transport.SendBroadcastResponseHandler!; + source.Transport.SendBroadcastResponseHandler = async (peer, batch, cancellation) => + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task.WaitAsync(cancellation); + return await send(peer, batch, cancellation); + }; + + try + { + Assert.True(await PublishValue(source.Protocol, sourceNamespace, sourceNamespace.CreateValue(keys[0], 1), network.Cancellation)); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), network.Cancellation); + var flush = source.Protocol.FlushPendingBroadcast(network.Cancellation); + Assert.False(await PublishValue(source.Protocol, sourceNamespace, sourceNamespace.CreateValue(keys[1], 1), network.Cancellation)); + Assert.False(await PublishValue(source.Protocol, sourceNamespace, sourceNamespace.CreateValue(keys[2], 1), network.Cancellation)); + releaseFirstSend.TrySetResult(); + await flush.WaitAsync(TimeSpan.FromSeconds(5), network.Cancellation); + AssertFakeState(receiverNamespace); + + for (var round = 1; round <= 4; round++) + { + sourceNamespace.SetValue(keys[0], round); + network.Clock.Advance(TimeSpan.FromSeconds(1)); + await receiver.Protocol.RunAntiEntropyRound(network.Cancellation); + } + + Assert.Equal(1, receiverNamespace.GetVersion(keys[1])); + Assert.Equal(1, receiverNamespace.GetVersion(keys[2])); + Assert.Equal(keys, receiver.Transport.AntiEntropyRequests[0].Request.Digests[sourceNamespace.Name] + .Select(static digest => digest.Key)); + Assert.All(receiver.Transport.AntiEntropyRequests[0].Request.Digests[sourceNamespace.Name], + static digest => Assert.Equal(0, digest.Version)); + Assert.Equal(new[] { keys[0], keys[1], keys[0], keys[2] }, + network.Replies.Select(static response => Assert.Single(GetAntiEntropyResponseValues(response)).Value.Key)); + Assert.All(network.Replies, static response => Assert.True(response.Truncated)); + + for (var round = 0; round < 2; round++) + { + network.Clock.Advance(TimeSpan.FromSeconds(1)); + await receiver.Protocol.RunAntiEntropyRound(network.Cancellation); + } + + await network.Flush(network.Cancellation); + AssertFakeState(sourceNamespace, (keys[0], 4), (keys[1], 1), (keys[2], 1)); + Assert.Equal(sourceNamespace.GetState(), receiverNamespace.GetState()); + Assert.Equal(6, network.RequestCount); + Assert.Equal(5, network.Replies.Sum(static response => GetAntiEntropyResponseValues(response).Count())); + Assert.Empty(network.Replies.Last().Values); + Assert.Single(network.Broadcasts); + } + finally + { + releaseFirstSend.TrySetResult(); + } + } + + private static void AssertFakeState(FakeNamespace ns, params (DisseminationKey Key, long Version)[] expected) => + Assert.Equal( + expected.OrderBy(static entry => entry.Key.ToString(), StringComparer.Ordinal) + .Select(static entry => (entry.Key, entry.Version, entry.Version)), + ns.GetState()); + + private static void AssertMembershipState(MembershipTableSnapshot expected, MembershipTableSnapshot actual) + { + Assert.Equal(expected.Version, actual.Version); + Assert.Equal(expected.Entries.Keys.Order(), actual.Entries.Keys.Order()); + foreach (var (silo, entry) in expected.Entries) + { + var observed = actual.Entries[silo]; + Assert.Equal( + (entry.SiloAddress, entry.Status, entry.StartTime, entry.IAmAliveTime, entry.ProxyPort, entry.HostName, entry.SiloName, entry.RoleName), + (observed.SiloAddress, observed.Status, observed.StartTime, observed.IAmAliveTime, observed.ProxyPort, observed.HostName, observed.SiloName, observed.RoleName)); + Assert.Equal((entry.UpdateZone, entry.FaultZone), (observed.UpdateZone, observed.FaultZone)); + Assert.Equal(entry.SuspectTimes, observed.SuspectTimes); + } + } + + // Every delivery enters a real protocol. There is deliberately no legacy transport or direct namespace-copy path. + private sealed class RecoveryNetwork : IAsyncDisposable + { + private readonly Dictionary _nodes = []; + private readonly CancellationTokenSource _cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + private SiloAddress[] _participants; + + public RecoveryNetwork( + SiloAddress[] members, + Func createNamespace, + Action? configure = null) + { + _participants = members; + foreach (var member in members) + { + var ns = createNamespace(member); + ns.Options.Priority = DisseminationPriority.Normal; + ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); + ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(1); + var transport = new FakeTransport(member, members.Where(peer => !peer.Equals(member)).ToArray()); + var protocol = CreateProtocol(transport, [ns], options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 8; + options.Overlay.AntiEntropyPeerCount = members.Length; + options.Overlay.AntiEntropyInterval = TimeSpan.FromHours(1); + configure?.Invoke(options); + }, Clock); + var node = new RecoveryNode(member, ns, transport, protocol); + _nodes.Add(member, node); + transport.SendBroadcastResponseHandler = async (peer, batch, cancellation) => + { + var deliver = !DropTreeBroadcasts && !Partitioned; + Broadcasts.Enqueue((member, peer, deliver)); + if (!deliver) + { + throw new InvalidOperationException("The tree link drops every delivery in this phase."); + } + + var response = await _nodes[peer].Protocol.ReceiveBroadcast(batch, cancellation); + return LoseAcknowledgments ? new DisseminationBroadcastResponse() : response; + }; + transport.ExchangeAntiEntropyHandler = async (peer, request, cancellation) => + { + if (Partitioned) + { + throw new InvalidOperationException("The repair link is partitioned."); + } + + var response = await _nodes[peer].Protocol.ReceiveAntiEntropy(request, cancellation); + Replies.Enqueue(response); + return response; + }; + } + } + + public RecordingFakeTimeProvider Clock { get; } = new(); + public CancellationToken Cancellation => _cancellation.Token; + public IEnumerable Nodes => _nodes.Values; + public RecoveryNode this[SiloAddress silo] => _nodes[silo]; + public ConcurrentQueue<(SiloAddress Sender, SiloAddress Peer, bool Delivered)> Broadcasts { get; } = new(); + public ConcurrentQueue Replies { get; } = new(); + public bool DropTreeBroadcasts { get; set; } = true; + public bool LoseAcknowledgments { get; set; } + public bool Partitioned { get; set; } + public int Sweeps { get; private set; } + public int RequestCount => _nodes.Values.Sum(static node => node.Transport.AntiEntropyRequests.Count); + + public void SetTopology(params (SiloAddress Silo, SiloStatus Status)[] members) + { + _participants = members.Select(static member => member.Silo).ToArray(); + foreach (var node in _nodes.Values) + { + node.Transport.Peers.Clear(); + node.Transport.Peers.AddRange(_participants.Where(peer => !peer.Equals(node.Address))); + node.Transport.PeerStatuses.Clear(); + node.Transport.PeerStatuses[node.Address] = SiloStatus.Dead; + foreach (var (silo, status) in members) + { + node.Transport.PeerStatuses[silo] = status; + } + } + } + + public async Task RunSweep(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Clock.Advance(TimeSpan.FromSeconds(1)); + foreach (var member in _participants) + { + await _nodes[member].Protocol.RunAntiEntropyRound(cancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + + Sweeps++; + } + + public async Task Flush(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + foreach (var node in _nodes.Values) + { + await node.Protocol.FlushPendingBroadcast(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + } + + public async ValueTask DisposeAsync() + { + _cancellation.Cancel(); + foreach (var node in _nodes.Values) + { + node.Namespace.Options.Enabled = false; + } + + try + { + foreach (var node in _nodes.Values) + { + await node.Protocol.StopAsync(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + finally + { + _cancellation.Dispose(); + } + } + } + + private sealed record RecoveryNode( + SiloAddress Address, + IDisseminationNamespace Namespace, + FakeTransport Transport, + DisseminationProtocol Protocol); + + private sealed class SendGateObserver : IObserver>, IDisposable + { + private readonly SiloAddress _local; + private readonly string _kind; + private readonly object _lock = new(); + private readonly List<(SiloAddress Peer, string Stage)> _events = []; + private readonly List<(SiloAddress Peer, string Stage, TaskCompletionSource Completion)> _waiters = []; + private readonly IDisposable _subscription; + + public SendGateObserver(SiloAddress local, string kind) + { + _local = local; + _kind = kind; + _subscription = DisseminationEvents.Listener.Subscribe( + this, static name => name == DisseminationEvents.SendGateEventName); + } + + public Task Wait(SiloAddress peer, string stage, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_lock) + { + var index = _events.FindIndex(entry => entry.Peer.Equals(peer) && entry.Stage == stage); + if (index >= 0) + { + _events.RemoveAt(index); + return Task.CompletedTask; + } + + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Add((peer, stage, completion)); + return completion.Task.WaitAsync(cancellationToken); + } + } + + public void OnNext(KeyValuePair value) + { + if (value.Value is DisseminationSendGateEvent transition && transition.Kind == _kind) + { + OnTransition(transition.LocalSilo, transition.Peer, transition.Stage); + } + } + + private void OnTransition(SiloAddress local, SiloAddress peer, string stage) + { + if (!local.Equals(_local)) + { + return; + } + + TaskCompletionSource? completion = null; + lock (_lock) + { + var index = _waiters.FindIndex(waiter => waiter.Peer.Equals(peer) && waiter.Stage == stage); + if (index >= 0) + { + completion = _waiters[index].Completion; + _waiters.RemoveAt(index); + } + else + { + _events.Add((peer, stage)); + } + } + + completion?.TrySetResult(); + } + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void Dispose() => _subscription.Dispose(); + } + + private sealed class QueueReentryObserver(SiloAddress localSilo, Action callback) : IObserver> + { + public void OnNext(KeyValuePair value) + { + if (value.Value is DisseminationBroadcastScheduledEvent scheduled && Equals(scheduled.LocalSilo, localSilo)) + { + callback(); + } + } + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + } + + private static DisseminationProtocol CreatePhase6Protocol( + FakeTransport transport, + IDisseminationNamespace disseminationNamespace, + Microsoft.Extensions.Logging.ILogger? logger = null, + Action? configure = null) + { + var options = new DisseminationOptions { Enabled = true }; + configure?.Invoke(options); + var localSiloDetails = new FakeLocalSiloDetails(transport.LocalSilo); + return new DisseminationProtocol( + localSiloDetails, + transport.GrainFactory, + new DisseminationMembership( + transport.MembershipManager, + localSiloDetails, + Options.Create(options)), + new TestOptionsMonitor(options), + [disseminationNamespace], + TimeProvider.System, + logger ?? NullLogger.Instance, + NullLogger.Instance); + } + + private sealed class Phase6ThrowingNamespace( + SiloAddress localSilo, + DisseminationKey failingKey) : IDisseminationNamespace + { + private readonly FakeNamespace _inner = new(localSilo); + + public Func> Failure { get; init; } = + static (_, _) => throw new InvalidOperationException("A failure must be configured."); + + public List ApplyAttempts { get; } = []; + + public DisseminationNamespace Name => _inner.Name; + + public DisseminationNamespaceOptions Options => _inner.Options; + + public IEnumerable Digests => _inner.Digests; + + public DisseminationValue CreateValue(DisseminationKey key, long sequence) => + _inner.CreateValue(key, sequence); + + public DisseminationBroadcastValue CreateItem( + SiloAddress originator, + DisseminationKey key, + long sequence) => + _inner.CreateItem(originator, key, sequence); + + public long GetVersion(DisseminationKey key) => _inner.GetVersion(key); + + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) => + _inner.CreateRepair(request); + + public ValueTask ApplyValueAsync( + DisseminationValue value, + CancellationToken cancellationToken) + { + ApplyAttempts.Add(value.Key); + return value.Key == failingKey + ? Failure(value, cancellationToken) + : _inner.ApplyValueAsync(value, cancellationToken); + } + } + + private sealed class Phase6ApplyObserver : IObserver>, IDisposable + { + private readonly DisseminationNamespace _namespace; + private readonly DisseminationKey _key; + private readonly SiloAddress _localSilo; + private readonly SiloAddress _sender; + private readonly IDisposable _subscription; + + public Phase6ApplyObserver( + DisseminationNamespace @namespace, + DisseminationKey key, + SiloAddress localSilo, + SiloAddress sender) + { + _namespace = @namespace; + _key = key; + _localSilo = localSilo; + _sender = sender; + _subscription = DisseminationEvents.Listener.Subscribe( + this, + static name => name == "Dissemination.ValueApply"); + } + + public List Events { get; } = []; + + public void OnNext(KeyValuePair value) + { + if (value.Value is DisseminationValueEvent apply + && apply.Namespace == _namespace + && apply.Key == _key + && Equals(apply.LocalSilo, _localSilo) + && Equals(apply.Peer, _sender) + && apply.Result == nameof(DisseminationApplyResult.Rejected)) + { + Events.Add(apply); + } + } + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void Dispose() => _subscription.Dispose(); + } + + private sealed class Phase6ProtocolLogger : Microsoft.Extensions.Logging.ILogger + { + public List Entries { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true; + + public void Log( + Microsoft.Extensions.Logging.LogLevel logLevel, + Microsoft.Extensions.Logging.EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (exception is null) + { + return; + } + + var properties = ((IEnumerable>)(object)state!) + .ToDictionary(static pair => pair.Key, static pair => pair.Value); + Entries.Add(new(logLevel, eventId, exception, properties)); + } + } + + private sealed record Phase6LogEntry( + Microsoft.Extensions.Logging.LogLevel Level, + Microsoft.Extensions.Logging.EventId EventId, + Exception Exception, + Dictionary State); + + private sealed class ThrowingApplyObserver( + DisseminationNamespace namespaceName, + DisseminationKey key, + SiloAddress localSilo, + SiloAddress sender) : IObserver> + { + public void OnNext(KeyValuePair value) + { + if (value.Value is DisseminationValueEvent apply + && apply.Namespace == namespaceName + && apply.Key == key + && Equals(apply.LocalSilo, localSilo) + && Equals(apply.Peer, sender)) + { + throw new InvalidOperationException("Diagnostic observer failure."); + } + } + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + } +} + +#if NET10_0_OR_GREATER +[State] +public partial class MonotonicDisseminationState +{ + public long Version { get; set; } +} + +public enum ModelApplyResult +{ + Applied, + Duplicate, + Obsolete, + Rejected, +} + +public sealed record ModelApplyResponse(ModelApplyResult Result, long Version); + +public sealed record ModelRepairResponse(bool HasValue, long Version); +#endif diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs new file mode 100644 index 00000000000..c91e33a4d07 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs @@ -0,0 +1,237 @@ +#nullable enable + +using Microsoft.Extensions.Time.Testing; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +[TestCategory("BVT"), TestCategory("Dissemination")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Dissemination")] +public class WakeTimerTests +{ + [Fact] + public async Task ChangeCompletesWaitWhenDueTimeElapses() + { + var timeProvider = new FakeTimeProvider(); + using var timer = new WakeTimer(timeProvider); + var wait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + + timer.Change(TimeSpan.FromSeconds(1)); + timeProvider.Advance(TimeSpan.FromMilliseconds(999)); + Assert.False(wait.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + + Assert.True(await wait); + } + + [Fact] + public async Task WakeBeforeWaitCompletesNextWait() + { + using var timer = new WakeTimer(TimeProvider.System); + + timer.Wake(); + + Assert.True(await timer.WaitAsync(TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task ChangeRearmsCurrentWait() + { + var timeProvider = new FakeTimeProvider(); + using var timer = new WakeTimer(timeProvider); + var wait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + + timer.Change(TimeSpan.FromSeconds(1)); + timeProvider.Advance(TimeSpan.FromMilliseconds(500)); + timer.Change(TimeSpan.FromSeconds(1)); + timeProvider.Advance(TimeSpan.FromMilliseconds(999)); + Assert.False(wait.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + + Assert.True(await wait); + } + + [Fact] + public async Task StaleCallbackDoesNotWakeRearmedTimer() + { + var timeProvider = new ControllableTimeProvider(); + using var timer = new WakeTimer(timeProvider); + var wait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + + timer.Change(TimeSpan.FromSeconds(1)); + timeProvider.Advance(TimeSpan.FromMilliseconds(500)); + timer.Change(TimeSpan.FromSeconds(1)); + timeProvider.FireTimer(); + + Assert.False(wait.IsCompleted); + + timeProvider.Advance(TimeSpan.FromSeconds(1)); + timeProvider.FireTimer(); + + Assert.True(await wait); + } + + [Fact] + public async Task CancellingWaitDoesNotDisarmTimer() + { + var timeProvider = new FakeTimeProvider(); + using var timer = new WakeTimer(timeProvider); + using var cancellation = new CancellationTokenSource(); + timer.Change(TimeSpan.FromSeconds(1)); + var canceledWait = timer.WaitAsync(cancellation.Token).AsTask(); + + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => canceledWait); + + var nextWait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + timeProvider.Advance(TimeSpan.FromSeconds(1)); + + Assert.True(await nextWait); + } + + [Fact] + public async Task WakeRacingCancelledWaitIsObservedByNextWaiter() + { + using var timer = new WakeTimer(TimeProvider.System); + using var cancellation = new CancellationTokenSource(); + var synchronizationContext = new QueuedSynchronizationContext(); + Task canceledWait; + var previousSynchronizationContext = SynchronizationContext.Current; + try + { + SynchronizationContext.SetSynchronizationContext(synchronizationContext); + canceledWait = timer.WaitAsync(cancellation.Token).AsTask(); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previousSynchronizationContext); + } + + cancellation.Cancel(); + timer.Wake(); + synchronizationContext.RunAll(); + + await Assert.ThrowsAnyAsync(() => canceledWait); + var nextWait = timer.WaitAsync(TestContext.Current.CancellationToken); + Assert.True(nextWait.IsCompletedSuccessfully); + Assert.True(await nextWait); + } + + [Fact] + public async Task DisposeCompletesCurrentAndFutureWaitsWithFalse() + { + var timer = new WakeTimer(TimeProvider.System); + var wait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + + timer.Dispose(); + + Assert.False(await wait); + Assert.False(await timer.WaitAsync(TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task DisposeCompletesWaitAndDisposesTimerWithoutChangingIt() + { + var timeProvider = new ThrowingChangeTimeProvider(); + var timer = new WakeTimer(timeProvider); + var wait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + + timer.Dispose(); + + Assert.False(await wait); + Assert.True(timeProvider.Timer.IsDisposed); + } + + private sealed class QueuedSynchronizationContext : SynchronizationContext + { + private readonly Queue<(SendOrPostCallback Callback, object? State)> _workItems = []; + + public override void Post(SendOrPostCallback callback, object? state) => _workItems.Enqueue((callback, state)); + + public void RunAll() + { + while (_workItems.TryDequeue(out var workItem)) + { + workItem.Callback(workItem.State); + } + } + } + + private sealed class ControllableTimeProvider : TimeProvider + { + private DateTimeOffset _utcNow; + private long _timestamp; + private ControllableTimer? _timer; + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public override long GetTimestamp() => _timestamp; + + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + _timer = new ControllableTimer(callback, state); + _timer.Change(dueTime, period); + return _timer; + } + + public void Advance(TimeSpan duration) + { + _utcNow += duration; + _timestamp += duration.Ticks; + } + + public void FireTimer() => _timer!.Fire(); + + private sealed class ControllableTimer(TimerCallback callback, object? state) : ITimer + { + private bool _disposed; + + public bool Change(TimeSpan dueTime, TimeSpan period) => !_disposed; + + public void Dispose() => _disposed = true; + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + + public void Fire() + { + if (!_disposed) + { + callback(state); + } + } + } + } + + private sealed class ThrowingChangeTimeProvider : TimeProvider + { + public ThrowingChangeTimer Timer { get; } = new(); + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => + Timer; + } + + private sealed class ThrowingChangeTimer : ITimer + { + public bool IsDisposed { get; private set; } + + public bool Change(TimeSpan dueTime, TimeSpan period) => + throw new InvalidOperationException("Changing this timer is not supported."); + + public void Dispose() => IsDisposed = true; + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Orleans.Runtime.Internal.Tests.csproj b/test/Orleans.Runtime.Internal.Tests/Orleans.Runtime.Internal.Tests.csproj index 51393eff87c..83cd3c21549 100644 --- a/test/Orleans.Runtime.Internal.Tests/Orleans.Runtime.Internal.Tests.csproj +++ b/test/Orleans.Runtime.Internal.Tests/Orleans.Runtime.Internal.Tests.csproj @@ -8,6 +8,7 @@ + From 9bf3fb3cd68526dbeb4de80ab822e7dca6a4669e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 11 Sep 2026 15:16:18 -0700 Subject: [PATCH 04/50] chore(api): capture generated dissemination surfaces --- .github/workflows/dissemination-api.yml | 59 ------------------- .../Orleans.Core.Abstractions.cs | 4 +- src/api/Orleans.Core/Orleans.Core.cs | 53 +++++++++++++++++ src/api/Orleans.Runtime/Orleans.Runtime.cs | 2 + 4 files changed, 58 insertions(+), 60 deletions(-) delete mode 100644 .github/workflows/dissemination-api.yml diff --git a/.github/workflows/dissemination-api.yml b/.github/workflows/dissemination-api.yml deleted file mode 100644 index a7efecf93b9..00000000000 --- a/.github/workflows/dissemination-api.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Dissemination API generation - -on: - workflow_dispatch: - pull_request: - branches: [main] - paths: - - .github/workflows/dissemination-api.yml - - src/Orleans.Core.Abstractions/IDs/SiloAddress.cs - - src/Orleans.Core/Configuration/Options/DisseminationOptions.cs - - src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs - - src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs - -permissions: - contents: read - -concurrency: - group: dissemination-api-${{ github.ref }} - cancel-in-progress: true - -jobs: - generate-api: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} - persist-credentials: false - - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 - with: - global-json-file: global.json - - name: Generate affected public API surfaces - shell: bash - run: | - set -euo pipefail - export CI=false - for name in Orleans.Core.Abstractions Orleans.Core Orleans.Runtime; do - project="src/$name/$name.csproj" - dotnet restore "$project" --configfile .github/NuGet.GenAPI.Config -p:GenerateOrleansApiSource=true - dotnet build "$project" --framework net8.0 --configuration Release --no-incremental --no-restore -p:GenerateOrleansApiSource=true '/t:Build;GenAPIGenerateReferenceAssemblySource' "-bl:genapi-$name.binlog" - done - - name: Upload exact generated surfaces - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: dissemination-generated-api - path: | - src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs - src/api/Orleans.Core/Orleans.Core.cs - src/api/Orleans.Runtime/Orleans.Runtime.cs - if-no-files-found: error - retention-days: 7 - - name: Upload generation diagnostics - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: dissemination-api-binlogs - path: genapi-*.binlog - retention-days: 7 diff --git a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs index 72f59d1cc80..4673ca09681 100644 --- a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs +++ b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs @@ -3003,7 +3003,7 @@ public sealed partial class ResourceOptimizedPlacement : PlacementStrategy [System.Text.Json.Serialization.JsonConverter(typeof(SiloAddressConverter))] [System.Diagnostics.DebuggerDisplay("SiloAddress {ToString()}")] [SuppressReferenceTracking] - public sealed partial class SiloAddress : System.IEquatable, System.IComparable, System.ISpanFormattable, System.IFormattable, System.IParsable, System.IUtf8SpanParsable + public sealed partial class SiloAddress : System.IEquatable, System.IComparable, System.IComparable, System.ISpanFormattable, System.IFormattable, System.IParsable, System.IUtf8SpanParsable { internal SiloAddress() { } @@ -3051,6 +3051,8 @@ internal SiloAddress() { } public static SiloAddress Parse(string value) { throw null; } + int System.IComparable.CompareTo(object? obj) { throw null; } + string System.IFormattable.ToString(string? format, System.IFormatProvider? formatProvider) { throw null; } bool System.ISpanFormattable.TryFormat(System.Span destination, out int charsWritten, System.ReadOnlySpan format, System.IFormatProvider? provider) { throw null; } diff --git a/src/api/Orleans.Core/Orleans.Core.cs b/src/api/Orleans.Core/Orleans.Core.cs index bc9c86ea2a8..ebfc08f7886 100644 --- a/src/api/Orleans.Core/Orleans.Core.cs +++ b/src/api/Orleans.Core/Orleans.Core.cs @@ -467,6 +467,8 @@ public partial class ClusterMembershipOptions public System.TimeSpan DefunctSiloExpiration { get { throw null; } set { } } + public DisseminationNamespaceOptions Dissemination { get { throw null; } set { } } + public bool EnableConnectionLivenessCheck { get { throw null; } set { } } public bool EnableIndirectProbes { get { throw null; } set { } } @@ -532,6 +534,57 @@ public partial class ConnectionOptions public Runtime.Messaging.NetworkProtocolVersion ProtocolVersion { get { throw null; } set { } } } + public sealed partial class DisseminationNamespaceOptions + { + public bool Enabled { get { throw null; } set { } } + + public System.TimeSpan ExpectedUpdateCadence { get { throw null; } set { } } + + public System.TimeSpan MaxCoalescingDelay { get { throw null; } set { } } + + public int MaxPayloadBytes { get { throw null; } set { } } + + public int MaxPendingItemCount { get { throw null; } set { } } + + public DisseminationPriority Priority { get { throw null; } set { } } + + public System.TimeSpan StaleItemTtl { get { throw null; } set { } } + } + + public sealed partial class DisseminationOptions + { + public bool Enabled { get { throw null; } set { } } + + public int MaxBatchBytes { get { throw null; } set { } } + + public int MaxBatchItems { get { throw null; } set { } } + + public int MaxConcurrentSends { get { throw null; } set { } } + + public DisseminationOverlayOptions Overlay { get { throw null; } set { } } + } + + public sealed partial class DisseminationOverlayOptions + { + public System.TimeSpan AntiEntropyInterval { get { throw null; } set { } } + + public int AntiEntropyPeerCount { get { throw null; } set { } } + + public System.Func? FanOutFactor { get { throw null; } set { } } + + public int MaxFanOutFactor { get { throw null; } set { } } + + public int MinFanOutFactor { get { throw null; } set { } } + + public int TargetHopCount { get { throw null; } set { } } + } + + public enum DisseminationPriority + { + Normal = 0, + High = 1 + } + public partial class GatewayOptions { public const int DEFAULT_PREFERED_GATEWAY_INDEX = -1; diff --git a/src/api/Orleans.Runtime/Orleans.Runtime.cs b/src/api/Orleans.Runtime/Orleans.Runtime.cs index 66d0945dad8..5cb045453b6 100644 --- a/src/api/Orleans.Runtime/Orleans.Runtime.cs +++ b/src/api/Orleans.Runtime/Orleans.Runtime.cs @@ -125,6 +125,8 @@ public partial class DeploymentLoadPublisherOptions { public static readonly System.TimeSpan DEFAULT_DEPLOYMENT_LOAD_PUBLISHER_REFRESH_TIME; public System.TimeSpan DeploymentLoadPublisherRefreshTime { get { throw null; } set { } } + + public DisseminationNamespaceOptions Dissemination { get { throw null; } set { } } } public partial class DevelopmentClusterMembershipOptions From 7b3eac91c74f18fc027f6a2b93fab8cd9602f081 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 13 Sep 2026 02:40:44 -0700 Subject: [PATCH 05/50] fix(dissemination): preserve publication and lifecycle guarantees --- .../implementation/runtime-dissemination.md | 8 +- .../IDisseminationSystemTarget.cs | 2 + .../DisseminationBroadcastQueue.cs | 96 ++++- .../DisseminationSystemTarget.cs | 20 +- .../MembershipDisseminationNamespace.cs | 18 +- ...eminationProtocolTests.MembershipReview.cs | 391 ++++++++++++++++++ .../DisseminationProtocolTests.QueueReview.cs | 362 ++++++++++++++++ .../DisseminationProtocolTests.cs | 2 +- 8 files changed, 870 insertions(+), 29 deletions(-) create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index 7f29f95e223..d14bcb86fb0 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -49,7 +49,7 @@ A `DisseminationValue` names one key and a `[FromVersion, ToVersion]` transition The namespace reports whether a repair is current, produced, unavailable, or unable to fit the supplied item and byte budgets. Produced repairs can be complete or a valid prefix. Prefix acknowledgments establish the receiver's new baseline and let the next batch continue the chain. -Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. It retains 32 snapshots, prefers a smaller diff when the peer baseline is present, and falls back to a full snapshot when history or capacity makes the diff unsuitable. Apply accepts same-version snapshots only when they advance liveness and preserves newer local liveness when applying a diff. +Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. It retains 32 snapshots, prefers a smaller diff when the peer baseline is present, and falls back to a full snapshot when history or capacity makes the diff unsuitable. Current diffs identify their entries as the complete target inventory, so table cleanup is reflected even when a receiver retained a different same-version baseline. Older partial-diff payloads retain their incremental semantics. Apply accepts same-version snapshots only when they advance liveness. The membership manager merges maximum per-entry liveness when publishing full snapshots, and diff construction preserves newer local liveness before handing state to that same manager. Deployment-load versions are sample timestamps. Each repair is a full latest value. Application runs on the deployment-load publisher's scheduler and accepts samples only for Active silo generations in the membership oracle. Terminal membership transitions remove placement statistics, and the oracle's monotonic membership state rejects later samples for departed generations. @@ -89,6 +89,8 @@ The subsystem and each namespace default to disabled. Enable . Operators can enable and tune membership and deployment-load dissemination independently while retaining the local concurrency and per-message bounds. +The anti-entropy loop waits without periodic timer wakeups while the subsystem is disabled. An options-change notification wakes the loop when enablement changes; disabling it returns the loop to the dormant wait. Shutdown removes the options subscription and observes the loop's completion. + Enablement requires representative measurements of convergence, RPC volume, serialized bytes, allocation, CPU, and tail latency across stable membership, churn, and partition recovery. Tree delivery can reduce RPC counts while increasing serialized bytes through forwarding and repair. Compare the original runtime, the updated default-off path, and the enabled path at the intended cluster sizes and publication rates, including skewed peer speeds and rolling upgrades. ## Broadcast pumps and backpressure @@ -123,7 +125,7 @@ Repairs from each sender remain ordered. Competing sender chains are ranked befo ## Mixed-version behavior -Peer namespace support is evidence-driven. Inbound traffic, broadcast acknowledgments, and authoritative anti-entropy responses confirm support for one silo generation. An explicit unsupported response revokes it, and membership pruning removes evidence for departed generations. +Peer namespace support is evidence-driven. Inbound traffic, broadcast acknowledgments, and authoritative anti-entropy responses confirm support for one silo generation. An explicit unsupported response revokes it and retires the publication generations covered by that flush. Publications admitted during the send remain queued with a fresh peer baseline. Membership pruning removes evidence for departed generations. Deployment-load publication uses dissemination for confirmed peers and the direct system target for unconfirmed peers. If dissemination is disabled, unavailable, rejected, or unable to accept the update within one refresh interval, direct fanout covers all Active peers. Membership direct gossip begins before optional dissemination and keeps the caller's cancellation and shutdown deadline. @@ -133,7 +135,7 @@ Protocol dictionaries use focused locks for membership projections, response cur Caller cancellation owns public operation lifetime and is checked before each received value is applied. Per-hop value TTL bounds the local broadcast RPC wait and anti-entropy round lifetime, independently of the runtime's cancellation-acknowledgment setting. Shared admission waits are pump-cancellable and retain their FIFO position while capacity is unavailable. Explicit flush and drain observers use their caller's cancellation token, with no separate observer TTL; supply a bounded token when waiting indefinitely for admission is unacceptable. -Stopping rejects new notifications and repair admissions. Accepted broadcast work can drain within the shutdown caller's budget; cancellation stops its queued admissions, local RPC waits, and pump timers. Each local attempt releases its managed admission lease during cleanup. Once pumps stop, the broadcast gate closes to further admissions. Repair cancellation is explicitly forwarded before disposing the round's linked sources, even when the local wait completes cancellation first. +Stopping rejects new notifications and repair admissions. Accepted broadcast work drains through acknowledgments, including retries with the usual backoff, within the shutdown caller's budget. Cancellation reports an incomplete drain and stops its queued admissions, local RPC waits, and pump timers. Each local attempt releases its managed admission lease during cleanup. Once pumps stop, the broadcast gate closes to further admissions. Repair cancellation is explicitly forwarded before disposing the round's linked sources, even when the local wait completes cancellation first. A stalled destination occupies at most one active local broadcast slot and one independent local repair slot. Hop deadlines release those slots and signal cancellation, allowing queued broadcasts and future repair rounds to continue. Retry backoff, rotating peer selection, per-message budgets, and local admission together bound the pace and size of new attempts. diff --git a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs index 5d4177cb2c7..e50c563a47e 100644 --- a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs +++ b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs @@ -165,6 +165,8 @@ private static int Compare(object? left, object? right) { string leftString when right is string rightString => StringComparer.Ordinal.Compare(leftString, rightString), SiloAddress leftSilo when right is SiloAddress rightSilo => leftSilo.CompareTo(rightSilo), + string when right is SiloAddress => -1, + SiloAddress when right is string => 1, _ => Comparer.Default.Compare(left, right), }; } diff --git a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs index e57085c6aed..e537a509725 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs @@ -267,6 +267,7 @@ private sealed class PeerQueuePump private int _retryAttempt; private bool _wakeScheduled; private bool _stopping; + private bool _draining; public PeerQueuePump(SiloAddress peer, DisseminationBroadcastQueue owner) { @@ -330,8 +331,7 @@ public bool Notify(IDisseminationNamespace disseminationNamespace, Dissemination keyState ??= namespaceState.AddKey(key); // A notification is only a wake-up. The namespace will choose the latest repair when this key is drained. keyState.NotificationVersion = version; - keyState.NotificationGeneration++; - _notificationEpoch++; + keyState.NotificationGeneration = ++_notificationEpoch; var wasRetrying = _retryAttempt > 0; _retryAttempt = 0; var wasEmpty = DirtyCount == 0; @@ -547,6 +547,7 @@ public async ValueTask StopAsync(bool drain, CancellationToken cancellationToken else { _stopping = true; + _draining = drain; if (drain) { if (_pumpFailure is { } pumpFailure) @@ -588,19 +589,29 @@ public async ValueTask StopAsync(bool drain, CancellationToken cancellationToken _flushTimer.Wake(); } - if (flushCompletion is not null) + while (flushCompletion is not null) { flushCompletion.Ignore(); await flushCompletion.WaitAsync(cancellationToken); + lock (_lock) + { + if (_pumpFailure is { } pumpFailure) + { + throw new InvalidOperationException($"The dissemination broadcast pump for {Peer} has failed.", pumpFailure); + } + + flushCompletion = DirtyCount > 0 ? _nextFlushCompletion.Task : _activeFlushCompletion?.Task; + } } } finally { - if (!drain || cancellationToken.IsCancellationRequested) + lock (_lock) { - await _shutdownCts.CancelAsync(); + _draining = false; } + await _shutdownCts.CancelAsync(); _flushTimer.Dispose(); try { @@ -689,7 +700,7 @@ private async Task RunScheduledFlushIteration(CancellationToken cancellationToke _retryAttempt = 0; } - if (DirtyCount > 0 && !_stopping) + if (DirtyCount > 0 && (!_stopping || _draining)) { // Do not overwrite a timer which a newer notification already chose. if (notificationEpoch == _notificationEpoch) @@ -753,7 +764,7 @@ private bool TryRecoverFromUnexpectedIterationFailure(Exception exception) TaskCompletionSource? activeFlushCompletion; lock (_lock) { - if (DirtyCount > 0 && !_stopping) + if (DirtyCount > 0 && (!_stopping || _draining)) { _retryAttempt++; var delay = _owner.GetRetryDelay(_retryAttempt); @@ -826,7 +837,7 @@ private async ValueTask SendValues( else if (!enabled || !nextWork.Namespace.Options.Enabled) { pending.Dequeue(); - CompleteUnsupported(nextWork); + CompleteUnsupported(nextWork.Namespace.Name, initialWork); } else { @@ -863,7 +874,7 @@ private async ValueTask SendValues( if (!currentOptions.Enabled || !work.Namespace.Options.Enabled) { pending.Dequeue(); - CompleteUnsupported(work); + CompleteUnsupported(work.Namespace.Name, initialWork); continue; } @@ -965,7 +976,7 @@ private async ValueTask SendValues( { if (unsupportedNamespaces.Contains(sent.Work.Namespace.Name)) { - CompleteUnsupported(sent.Work); + CompleteUnsupported(sent.Work.Namespace.Name, initialWork); continue; } @@ -1099,7 +1110,8 @@ private List DrainDirtyUnsafe() namespaceState.Namespace, key, keyState.NotificationGeneration, - keyState.KnownVersion)); + keyState.KnownVersion, + namespaceState)); } } @@ -1221,18 +1233,64 @@ private bool CompleteFromExistingEvidence(SentKey sent) } } - private void CompleteUnsupported(PendingKeyWork work) + private void CompleteUnsupported(DisseminationNamespace namespaceName, List work) { - // Treat this as peer capability evidence and forget the namespace until a new publication recreates it. + // Capability evidence completes this flush's generations, not publications made during the send. lock (_lock) { - if (!_statesByNamespace.TryGetValue(work.Namespace.Name, out var namespaceState)) + if (!_statesByNamespace.TryGetValue(namespaceName, out var namespaceState)) + { + return; + } + + var matchesState = false; + foreach (var item in work) + { + if (item.Namespace.Name != namespaceName + || !ReferenceEquals(item.NamespaceState, namespaceState)) + { + continue; + } + + matchesState = true; + if (!namespaceState.TryGetKey(item.Key, out var keyState) + || keyState is null) + { + continue; + } + + keyState.InFlight = false; + if (keyState.NotificationGeneration != item.NotificationGeneration) + { + MarkDirtyUnsafe(namespaceState, keyState); + continue; + } + + if (keyState.Dirty) + { + namespaceState.DirtyCount--; + DirtyCount--; + } + + namespaceState.Keys.Remove(item.Key); + } + + if (!matchesState) { return; } - DirtyCount -= namespaceState.DirtyCount; - _statesByNamespace.Remove(work.Namespace.Name); + namespaceState.KnownVersions.Clear(); + foreach (var keyState in namespaceState.Keys.Values) + { + keyState.KnownVersion = null; + } + + if (namespaceState.Keys.Count == 0) + { + _statesByNamespace.Remove(namespaceName); + } + if (DirtyCount == 0) { var completion = _nextFlushCompletion; @@ -1286,7 +1344,7 @@ private void Requeue(IEnumerable work) { lock (_lock) { - if (_stopping) + if (_stopping && !_draining) { foreach (var item in work) { @@ -1335,6 +1393,7 @@ private bool TryGetKeyStateUnsafe( out PeerKeyState keyState) { if (_statesByNamespace.TryGetValue(work.Namespace.Name, out namespaceState!) + && ReferenceEquals(namespaceState, work.NamespaceState) && namespaceState.Keys.TryGetValue(work.Key, out keyState!)) { return true; @@ -1523,7 +1582,8 @@ private readonly record struct PendingKeyWork( IDisseminationNamespace Namespace, DisseminationKey Key, long NotificationGeneration, - long? KnownVersion); + long? KnownVersion, + PeerNamespaceState NamespaceState); private readonly record struct SentKey( PendingKeyWork Work, diff --git a/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs b/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs index 82052c59bba..654211a4ee9 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs @@ -77,6 +77,7 @@ private Task StartAsync(CancellationToken cancellationToken) var shutdownToken = _shutdownCts.Token; _antiEntropyTask = this.RunOrQueueTask(() => RunAntiEntropyLoop(shutdownToken)); + _antiEntropyTask.Ignore(); return Task.CompletedTask; } @@ -114,14 +115,25 @@ private async Task StopAsync(CancellationToken cancellationToken) private async Task RunAntiEntropyLoop(CancellationToken cancellationToken) { + using var wakeTimer = new WakeTimer(_timeProvider); + using var subscription = _options.OnChange((_, _) => wakeTimer.Wake()); try { while (true) { - await Task.Delay( - _options.CurrentValue.Overlay.AntiEntropyInterval, - _timeProvider, - cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + var options = _options.CurrentValue; + wakeTimer.Change(options.Enabled ? options.Overlay.AntiEntropyInterval : Timeout.InfiniteTimeSpan); + if (!await wakeTimer.WaitAsync(cancellationToken)) + { + return; + } + + if (!_options.CurrentValue.Enabled) + { + continue; + } + try { await _protocol.RunAntiEntropyRound(cancellationToken); diff --git a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs index d3374798532..3552c03933f 100644 --- a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs @@ -163,6 +163,7 @@ public async ValueTask ApplyValueAsync( return DisseminationApplyResult.Rejected; } + // The membership manager merges maximum per-entry IAmAliveTime before publishing a full snapshot. var currentSnapshot = membershipManager.CurrentSnapshot; if (snapshot.Version < currentSnapshot.Version) { @@ -216,7 +217,11 @@ private async ValueTask ApplyDiff(MembershipTableSnaps return DisseminationApplyResult.Rejected; } - var entries = current.Entries.ToBuilder(); + // Table cleanup can change the inventory without changing its version, so a receiver's + // baseline can contain entries which the sender never saw and therefore could not remove. + var entries = diff.IncludesAllEntries + ? ImmutableDictionary.CreateBuilder() + : current.Entries.ToBuilder(); foreach (var silo in diff.RemovedSilos) { entries.Remove(silo); @@ -335,7 +340,8 @@ private static MembershipTableSnapshotDiff CreateDiff(MembershipTableSnapshot ba baseSnapshot.Version, snapshot.Version, updated.ToImmutable(), - removed.ToImmutable()); + removed.ToImmutable(), + includesAllEntries: true); } private static long GetFingerprint(MembershipTableSnapshot snapshot) @@ -460,12 +466,14 @@ public MembershipTableSnapshotDiff( MembershipVersion baseVersion, MembershipVersion version, ImmutableArray updatedEntries, - ImmutableArray removedSilos) + ImmutableArray removedSilos, + bool includesAllEntries = false) { BaseVersion = baseVersion; Version = version; UpdatedEntries = updatedEntries; RemovedSilos = removedSilos; + IncludesAllEntries = includesAllEntries; } [Id(0)] @@ -479,4 +487,8 @@ public MembershipTableSnapshotDiff( [Id(3)] public ImmutableArray RemovedSilos { get; } + + // Absent on older payloads, where UpdatedEntries can be only a partial update. + [Id(4)] + public bool IncludesAllEntries { get; } } diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs new file mode 100644 index 00000000000..68b71c3850c --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs @@ -0,0 +1,391 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans; +using Orleans.Configuration; +using Orleans.Core.Diagnostics; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.MembershipService; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData(1)] + [InlineData(2)] + public async Task MembershipNamespaceFullSnapshotPreservesMixedLivenessInRealManager(long incomingVersion) + { + var cancellationToken = TestContext.Current.CancellationToken; + var newer = CreateSilo(39501); + var stale = CreateSilo(39502); + var previous = CreateMembershipSnapshot( + 1, + CreateMembershipEntry(newer, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(10)), + CreateMembershipEntry(stale, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(20))); + var incoming = CreateMembershipSnapshot( + incomingVersion, + previous.Entries[newer].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(30)), + previous.Entries[stale].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(5))); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + using var manager = CreateMembershipReviewManager(CreateSilo(39500), out _); + await ((IMembershipManager)manager).ProcessGossipSnapshot(previous, cancellationToken); + var ns = CreateMembershipNamespace((IMembershipManager)manager, serializer); + var value = new DisseminationValue( + DisseminationKey.Default, + 0, + incomingVersion, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = incoming })); + + Assert.True(incoming.IsSuccessorTo(previous)); + Assert.Equal(DisseminationApplyResult.Applied, await ns.ApplyValueAsync(value, cancellationToken)); + + var expected = CreateMembershipSnapshot( + incomingVersion, + incoming.Entries[newer], + previous.Entries[stale]); + AssertMembershipState(expected, manager.MembershipTableSnapshot); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(5), incoming.Entries[stale].IAmAliveTime); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(10), previous.Entries[newer].IAmAliveTime); + + // Repairs must contain the manager's merged state, not the unmerged incoming snapshot. + var repair = Assert.Single(ns.CreateRepair(MembershipReviewRepairRequest(incomingVersion)).Values); + Assert.Equal(0, repair.FromVersion); + var repaired = Assert.IsType( + serializer.Deserialize(repair.Payload)); + AssertMembershipState(expected, Assert.IsType(repaired.Snapshot)); + Assert.Equal(DisseminationApplyResult.Duplicate, await ns.ApplyValueAsync(value, cancellationToken)); + AssertMembershipState(expected, manager.MembershipTableSnapshot); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MembershipNamespaceCompleteDiffPrunesDivergentInventoryAndRepairsDownstream(bool receiverAlreadyAtTargetVersion) + { + var cancellationToken = TestContext.Current.CancellationToken; + var retired = CreateSilo(39511); + var successor = SiloAddress.New(retired.Endpoint, retired.Generation + 1); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var table = new InMemoryMembershipTable(services.GetRequiredService()); + Assert.True(table.Insert( + CreateMembershipEntry(retired, SiloStatus.Active, DateTime.UnixEpoch), + table.ReadTableVersion().Next())); + var staleGenerationSnapshot = MembershipTableSnapshot.Create(table.ReadAll()); + var retiredRow = Assert.Single(table.ReadAll().Members); + Assert.True(table.Update( + retiredRow.Item1.WithStatus(SiloStatus.Dead), + retiredRow.Item2, + table.ReadTableVersion().Next())); + Assert.True(table.Insert( + CreateMembershipEntry(successor, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(10)), + table.ReadTableVersion().Next())); + var beforeCleanup = MembershipTableSnapshot.Create(table.ReadAll()); + using var sourceManager = CreateMembershipReviewManager(CreateSilo(39510), out _); + using var receiverManager = CreateMembershipReviewManager(CreateSilo(39512), out _); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(beforeCleanup, cancellationToken); + await ((IMembershipManager)receiverManager).ProcessGossipSnapshot(beforeCleanup, cancellationToken); + + var tableVersion = table.ReadTableVersion(); + table.CleanupDefunctSiloEntries(DateTimeOffset.UnixEpoch.AddSeconds(1)); + Assert.Equal(tableVersion, table.ReadTableVersion()); + Assert.DoesNotContain(table.ReadAll().Members, entry => entry.Item1.SiloAddress.Equals(retired)); + table.UpdateIAmAlive(beforeCleanup.Entries[successor].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(20))); + var afterCleanup = MembershipTableSnapshot.Update(beforeCleanup, table.ReadAll()); + Assert.Equal(beforeCleanup.Version, afterCleanup.Version); + Assert.True(afterCleanup.IsSuccessorTo(beforeCleanup)); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(afterCleanup, cancellationToken); + var source = CreateMembershipNamespace((IMembershipManager)sourceManager, serializer); + var receiver = CreateMembershipNamespace((IMembershipManager)receiverManager, serializer); + var sourceBaseline = Assert.Single(source.Digests); + var receiverBaseline = Assert.Single(receiver.Digests); + Assert.Equal(sourceBaseline.Version, receiverBaseline.Version); + Assert.NotEqual(sourceBaseline.Fingerprint, receiverBaseline.Fingerprint); + + var row = Assert.Single(table.ReadAll().Members); + Assert.True(table.Update( + row.Item1.WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(30)), + row.Item2, + table.ReadTableVersion().Next())); + var target = MembershipTableSnapshot.Update(afterCleanup, table.ReadAll()); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(target, cancellationToken); + if (receiverAlreadyAtTargetVersion) + { + await ((IMembershipManager)receiverManager).ProcessGossipSnapshot( + new MembershipTableSnapshot(target.Version, beforeCleanup.Entries), + cancellationToken); + } + + var value = Assert.Single(source.CreateRepair(MembershipReviewRepairRequest(sourceBaseline.Version)).Values); + Assert.Equal((sourceBaseline.Version, target.Version.Value), (value.FromVersion, value.ToVersion)); + var update = Assert.IsType( + serializer.Deserialize(value.Payload)); + var diff = Assert.IsType(update.Diff); + Assert.True(diff.IncludesAllEntries); + Assert.Empty(diff.RemovedSilos); + Assert.Equal(successor, Assert.Single(diff.UpdatedEntries).SiloAddress); + + Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(value, cancellationToken)); + AssertMembershipState(target, receiverManager.MembershipTableSnapshot); + Assert.False(receiverManager.MembershipTableSnapshot.Entries.ContainsKey(retired)); + Assert.Equal(SiloStatus.Dead, receiverManager.MembershipTableSnapshot.GetSiloStatus(retired)); + Assert.Equal(SiloStatus.Active, receiverManager.MembershipTableSnapshot.GetSiloStatus(successor)); + Assert.Equal(Assert.Single(source.Digests), Assert.Single(receiver.Digests)); + Assert.Equal(DisseminationApplyResult.Duplicate, await receiver.ApplyValueAsync(value, cancellationToken)); + + var obsolete = new DisseminationValue( + DisseminationKey.Default, + 0, + staleGenerationSnapshot.Version.Value, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = staleGenerationSnapshot })); + Assert.Equal(DisseminationApplyResult.Obsolete, await receiver.ApplyValueAsync(obsolete, cancellationToken)); + AssertMembershipState(target, receiverManager.MembershipTableSnapshot); + + using var downstreamManager = CreateMembershipReviewManager(CreateSilo(39513), out _); + await ((IMembershipManager)downstreamManager).ProcessGossipSnapshot(beforeCleanup, cancellationToken); + var downstream = CreateMembershipNamespace((IMembershipManager)downstreamManager, serializer); + var forwarded = Assert.Single(receiver.CreateRepair(MembershipReviewRepairRequest(sourceBaseline.Version)).Values); + Assert.True(forwarded.FromVersion == 0 || forwarded.FromVersion == sourceBaseline.Version); + Assert.Equal(target.Version.Value, forwarded.ToVersion); + Assert.Equal(DisseminationApplyResult.Applied, await downstream.ApplyValueAsync(forwarded, cancellationToken)); + AssertMembershipState(target, downstreamManager.MembershipTableSnapshot); + } + + [Fact] + public async Task MembershipNamespaceCompleteDiffPreservesRealManagerLocalDeathEntry() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(39520); + var peer = CreateSilo(39521); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + using var manager = CreateMembershipReviewManager(local, out var fatalErrorHandler); + var localEntry = manager.MembershipTableSnapshot.Entries[local].WithStatus(SiloStatus.Active); + var previous = CreateMembershipSnapshot( + 1, + localEntry, + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + await ((IMembershipManager)manager).ProcessGossipSnapshot(previous, cancellationToken); + var ns = CreateMembershipNamespace((IMembershipManager)manager, serializer); + var update = new MembershipTableSnapshotUpdate + { + Diff = new MembershipTableSnapshotDiff( + previous.Version, + new MembershipVersion(2), + [previous.Entries[peer]], + [], + includesAllEntries: true), + }; + var value = new DisseminationValue(DisseminationKey.Default, 1, 2, serializer.SerializeToArray(update)); + + Assert.Equal(DisseminationApplyResult.Applied, await ns.ApplyValueAsync(value, cancellationToken)); + + var expected = CreateMembershipSnapshot(2, localEntry.WithStatus(SiloStatus.Dead), previous.Entries[peer]); + AssertMembershipState(expected, manager.MembershipTableSnapshot); + Assert.Equal(SiloStatus.Dead, manager.CurrentStatus); + fatalErrorHandler.Received(1).OnFatalException(manager, Arg.Any(), null); + var repair = Assert.Single(ns.CreateRepair(MembershipReviewRepairRequest(2)).Values); + Assert.Equal(0, repair.FromVersion); + var repaired = Assert.IsType( + serializer.Deserialize(repair.Payload)); + AssertMembershipState(expected, Assert.IsType(repaired.Snapshot)); + } + + [Fact] + public async Task MembershipNamespaceDiffWireCompatibilityPreservesLegacyPartialUpdates() + { + var cancellationToken = TestContext.Current.CancellationToken; + var retained = CreateSilo(39531); + var updated = CreateSilo(39532); + var removed = CreateSilo(39533); + var previous = CreateMembershipSnapshot( + 1, + CreateMembershipEntry(retained, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(updated, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(removed, SiloStatus.Dead, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var legacy = new MembershipReviewLegacyDiff + { + BaseVersion = previous.Version, + Version = new MembershipVersion(2), + UpdatedEntries = [previous.Entries[updated].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))], + RemovedSilos = [removed], + }; + + // Root codecs omit the type name when the expected and actual types agree, allowing + // these two schemas to exercise both missing-field and unknown-field wire compatibility. + var decoded = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(legacy))); + Assert.False(decoded.IncludesAllEntries); + Assert.Equal(legacy.BaseVersion, decoded.BaseVersion); + Assert.Equal(legacy.Version, decoded.Version); + Assert.Equal(legacy.RemovedSilos, decoded.RemovedSilos); + Assert.Equal(updated, Assert.Single(decoded.UpdatedEntries).SiloAddress); + using var manager = CreateMembershipReviewManager(CreateSilo(39530), out _); + await ((IMembershipManager)manager).ProcessGossipSnapshot(previous, cancellationToken); + var ns = CreateMembershipNamespace((IMembershipManager)manager, serializer); + var value = new DisseminationValue( + DisseminationKey.Default, + 1, + 2, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Diff = decoded })); + + Assert.Equal(DisseminationApplyResult.Applied, await ns.ApplyValueAsync(value, cancellationToken)); + var expected = CreateMembershipSnapshot(2, previous.Entries[retained], legacy.UpdatedEntries[0]); + AssertMembershipState(expected, manager.MembershipTableSnapshot); + + var complete = new MembershipTableSnapshotDiff( + legacy.BaseVersion, + legacy.Version, + [.. expected.Entries.Values], + legacy.RemovedSilos, + includesAllEntries: true); + var oldPeer = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(complete))); + Assert.Equal(complete.BaseVersion, oldPeer.BaseVersion); + Assert.Equal(complete.Version, oldPeer.Version); + Assert.Equal(complete.RemovedSilos, oldPeer.RemovedSilos); + AssertMembershipState( + expected, + CreateMembershipSnapshot(oldPeer.Version.Value, [.. oldPeer.UpdatedEntries])); + } + + [Fact] + public async Task MembershipNamespaceLegacyInventoryDivergenceConvergesAfterHeartbeatWithFullRepair() + { + var cancellationToken = TestContext.Current.CancellationToken; + var retired = CreateSilo(39541); + var successor = SiloAddress.New(retired.Endpoint, retired.Generation + 1); + var receiverBaseline = CreateMembershipSnapshot( + 1, + CreateMembershipEntry(retired, SiloStatus.Dead, DateTime.UnixEpoch), + CreateMembershipEntry(successor, SiloStatus.Active, DateTime.UnixEpoch)); + var sourceBaseline = CreateMembershipSnapshot(1, receiverBaseline.Entries[successor]); + var target = CreateMembershipSnapshot( + 2, + sourceBaseline.Entries[successor].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + using var sourceManager = CreateMembershipReviewManager(CreateSilo(39540), out _); + using var receiverManager = CreateMembershipReviewManager(CreateSilo(39542), out _); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(sourceBaseline, cancellationToken); + await ((IMembershipManager)receiverManager).ProcessGossipSnapshot(receiverBaseline, cancellationToken); + var source = CreateMembershipNamespace((IMembershipManager)sourceManager, serializer); + var receiver = CreateMembershipNamespace((IMembershipManager)receiverManager, serializer); + _ = Assert.Single(source.Digests); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(target, cancellationToken); + var delta = Assert.Single(source.CreateRepair(MembershipReviewRepairRequest(1)).Values); + Assert.Equal((1L, 2L), (delta.FromVersion, delta.ToVersion)); + var update = Assert.IsType( + serializer.Deserialize(delta.Payload)); + var legacy = Assert.IsType( + serializer.Deserialize( + serializer.SerializeToArray(Assert.IsType(update.Diff)))); + var legacyUpdate = new MembershipTableSnapshotUpdate + { + Diff = new MembershipTableSnapshotDiff(legacy.BaseVersion, legacy.Version, legacy.UpdatedEntries, legacy.RemovedSilos), + }; + var legacyValue = new DisseminationValue(DisseminationKey.Default, 1, 2, serializer.SerializeToArray(legacyUpdate)); + Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(legacyValue, cancellationToken)); + Assert.Equal(SiloStatus.Dead, receiverManager.MembershipTableSnapshot.GetSiloStatus(retired)); + Assert.True(receiverManager.MembershipTableSnapshot.Entries.ContainsKey(retired)); + Assert.Equal(target.Version, receiverManager.MembershipTableSnapshot.Version); + + var transport = new FakeTransport(CreateSilo(39540), CreateSilo(39542)); + var protocol = CreateProtocol(transport, [source], timeProvider: new FakeTimeProvider()); + try + { + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = transport.Peers[0], + Digests = new() { [source.Name] = [Assert.Single(receiver.Digests)] }, + }, cancellationToken); + var full = Assert.Single(GetAntiEntropyResponseValues(response)).Value; + Assert.Equal((0L, 2L), (full.FromVersion, full.ToVersion)); + // Inventory-only cleanup need not be published by the manager: retired entries are already dead. + Assert.Equal(DisseminationApplyResult.Duplicate, await receiver.ApplyValueAsync(full, cancellationToken)); + + var heartbeatTarget = CreateMembershipSnapshot( + 2, + target.Entries[successor].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(20))); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(heartbeatTarget, cancellationToken); + response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = transport.Peers[0], + Digests = new() { [source.Name] = [Assert.Single(receiver.Digests)] }, + }, cancellationToken); + full = Assert.Single(GetAntiEntropyResponseValues(response)).Value; + Assert.Equal((0L, 2L), (full.FromVersion, full.ToVersion)); + Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(full, cancellationToken)); + AssertMembershipState(heartbeatTarget, receiverManager.MembershipTableSnapshot); + Assert.Equal(SiloStatus.Dead, receiverManager.MembershipTableSnapshot.GetSiloStatus(retired)); + Assert.Equal(Assert.Single(source.Digests), Assert.Single(receiver.Digests)); + + response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = transport.Peers[0], + Digests = new() { [source.Name] = [Assert.Single(receiver.Digests)] }, + }, cancellationToken); + Assert.Empty(GetAntiEntropyResponseValues(response)); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + + private static MembershipTableManager CreateMembershipReviewManager( + SiloAddress local, + out IFatalErrorHandler fatalErrorHandler) + { + var timer = Substitute.For(); + var timers = Substitute.For(); + timers.Create(default, default!, default!).ReturnsForAnyArgs(timer); + fatalErrorHandler = Substitute.For(); + return new MembershipTableManager( + new FakeLocalSiloDetails(local), + Options.Create(new ClusterMembershipOptions()), + Substitute.For(), + fatalErrorHandler, + Substitute.For(), + NullLogger.Instance, + timers, + new SiloLifecycleSubject(NullLogger.Instance), + new FakeTimeProvider()); + } + + private static MembershipDisseminationNamespace CreateMembershipNamespace(IMembershipManager manager, Serializer serializer) + { + var options = new ClusterMembershipOptions(); + options.Dissemination.Enabled = true; + return new(manager, new TestOptionsMonitor(options), serializer); + } + + private static DisseminationRepairRequest MembershipReviewRepairRequest(long? fromVersion) => + new(DisseminationKey.Default, fromVersion, toVersion: null, maxItemCount: 1, maxBatchBytes: 1024 * 1024, maxPayloadBytes: 1024 * 1024); + + [GenerateSerializer] + internal sealed class MembershipReviewLegacyDiff + { + [Id(0)] + public MembershipVersion BaseVersion { get; init; } + + [Id(1)] + public MembershipVersion Version { get; init; } + + [Id(2)] + public ImmutableArray UpdatedEntries { get; init; } + + [Id(3)] + public ImmutableArray RemovedSilos { get; init; } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs new file mode 100644 index 00000000000..2815eb6fda8 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs @@ -0,0 +1,362 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.Scheduler; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public void DisseminationKeysSortAcrossSupportedKinds() + { + var firstSilo = CreateSilo(40101); + var secondSilo = CreateSilo(40102); + DisseminationKey[] keys = [secondSilo, "b", DisseminationKey.Default, firstSilo, string.Empty, "a"]; + DisseminationKey[] expected = [DisseminationKey.Default, string.Empty, "a", "b", firstSilo, secondSilo]; + + Array.Sort(keys); + + Assert.Equal(expected, keys); + for (var left = 0; left < expected.Length; left++) + { + for (var right = 0; right < expected.Length; right++) + { + Assert.Equal(Math.Sign(left - right), Math.Sign(expected[left].CompareTo(expected[right]))); + Assert.Equal(Math.Sign(left - right), Math.Sign(((IComparable)expected[left]).CompareTo(expected[right]))); + } + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ShutdownDrainRetriesUntilAcceptedWorkIsAcknowledged(bool transportFailure) + { + var local = CreateSilo(40111); + var peer = CreateSilo(40112); + var transport = new FakeTransport(local, peer); + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local); + ns.SetValue(FakeNamespace.DefaultKey, 2); + var sentVersions = new List(); + transport.SendBroadcastResponseHandler = (_, batch, _) => + { + sentVersions.Add(Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); + return sentVersions.Count == 1 + ? transportFailure + ? Task.FromException(new InvalidOperationException("Transient send failure.")) + : Task.FromResult(new DisseminationBroadcastResponse()) + : Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + value => value.LocalSilo.Equals(local) && value.Peer.Equals(peer) + && value.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + Assert.True(queue.Notify(peer, ns, FakeNamespace.DefaultKey)); + var stop = queue.StopAsync(cancellation.Token); + try + { + var scheduled = await retry; + Assert.False(stop.IsCompleted); + Assert.False(queue.Notify(peer, ns, "after-stop")); + Assert.Equal(new[] { 2L }, sentVersions); + + clock.Advance(scheduled.DueTime); + await stop.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(new[] { 2L, 2L }, sentVersions); + } + finally + { + cancellation.Cancel(); + try + { + await stop.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + } + } + } + + [Fact] + public async Task ShutdownDrainCancellationSurfacesIncompleteDelivery() + { + var local = CreateSilo(40121); + var peer = CreateSilo(40122); + var transport = new FakeTransport(local, peer); + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local); + ns.SetValue(FakeNamespace.DefaultKey, 1); + var sends = 0; + transport.SendBroadcastResponseHandler = (_, _, _) => + { + Interlocked.Increment(ref sends); + return Task.FromResult(new DisseminationBroadcastResponse()); + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + value => value.LocalSilo.Equals(local) && value.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + Assert.True(queue.Notify(peer, ns, FakeNamespace.DefaultKey)); + var stop = queue.StopAsync(cancellation.Token); + try + { + await retry; + Assert.False(stop.IsCompleted); + } + finally + { + cancellation.Cancel(); + } + + var exception = await Assert.ThrowsAnyAsync( + () => stop.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(1, Volatile.Read(ref sends)); + } + + [Fact] + public async Task UnsupportedResponsePreservesPublicationsMadeDuringItsSend() + { + var local = CreateSilo(40131); + var peer = CreateSilo(40132); + var transport = new FakeTransport(local, peer); + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local); + ns.SetValue("updated", 1); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var batches = new List(); + transport.SendBroadcastResponseHandler = async (_, batch, cancellationToken) => + { + batches.Add(batch); + if (batches.Count == 1) + { + entered.SetResult(); + await release.Task.WaitAsync(cancellationToken); + return new DisseminationBroadcastResponse { UnsupportedNamespaces = [ns.Name] }; + } + + return FakeTransport.CreateAcknowledgment(batch); + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); + try + { + Assert.True(queue.Notify(peer, ns, "updated")); + var firstFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + ns.SetValue("updated", 2); + ns.SetValue("new", 7); + Assert.True(queue.Notify(peer, ns, "updated")); + Assert.True(queue.Notify(peer, ns, "new")); + var newerFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + release.SetResult(); + await Task.WhenAll(firstFlush, newerFlush).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(2, batches.Count); + Assert.Equal(1, Assert.Single(GetBroadcastValues(batches[0])).Value.ToVersion); + var delivered = GetBroadcastValues(batches[1]).ToDictionary(value => value.Value.Key, value => value.Value.ToVersion); + Assert.Equal(2, delivered.Count); + Assert.Equal(2, delivered["updated"]); + Assert.Equal(7, delivered["new"]); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(2, batches.Count); + } + finally + { + release.TrySetResult(); + await queue.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task DisabledAntiEntropySleepsUntilOptionsChangeAndUnsubscribesOnStop() + { + var local = CreateSilo(40141); + var peer = CreateSilo(40142); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var clock = new ReviewTimeProvider(); + var options = new ReviewOptionsMonitor(new DisseminationOptions()); + var exchanges = Channel.CreateUnbounded(); + var exchangeCount = 0; + transport.ExchangeAntiEntropyHandler = (destination, _, _) => + { + exchanges.Writer.TryWrite(Interlocked.Increment(ref exchangeCount)); + return ValueTask.FromResult(new DisseminationAntiEntropyResponse { Sender = destination }); + }; + var localDetails = new FakeLocalSiloDetails(local); + var target = new DisseminationSystemTarget( + localDetails, + transport.GrainFactory, + new DisseminationMembership(transport.MembershipManager, localDetails, Options.Create(options.CurrentValue)), + options, + [ns], + clock, + NullLogger.Instance, + NullLogger.Instance, + CreatePhase4SystemTargetShared(local)); + var lifecycle = Substitute.For(); + ILifecycleObserver? observer = null; + lifecycle.Subscribe(Arg.Any(), Arg.Any(), Arg.Any()).Returns(call => + { + observer = call.ArgAt(2); + return new ReviewSubscription(static () => { }); + }); + ((ILifecycleParticipant)target).Participate(lifecycle); + Assert.NotNull(observer); + await observer.OnStart(TestContext.Current.CancellationToken); + try + { + await clock.WaitForChange(Timeout.InfiniteTimeSpan, TestContext.Current.CancellationToken); + await clock.WaitForChange(Timeout.InfiniteTimeSpan, TestContext.Current.CancellationToken); + clock.Advance(TimeSpan.FromDays(1)); + await target.RunOrQueueTask(_ => Task.FromResult(true), TestContext.Current.CancellationToken); + Assert.Equal(0, Volatile.Read(ref exchangeCount)); + + options.Set(new DisseminationOptions { Enabled = true }); + Assert.Equal(1, await exchanges.Reader.ReadAsync(TestContext.Current.CancellationToken)); + await clock.WaitForChange(options.CurrentValue.Overlay.AntiEntropyInterval, TestContext.Current.CancellationToken); + + options.Set(new DisseminationOptions()); + await clock.WaitForChange(Timeout.InfiniteTimeSpan, TestContext.Current.CancellationToken); + clock.Advance(TimeSpan.FromDays(1)); + await target.RunOrQueueTask(_ => Task.FromResult(true), TestContext.Current.CancellationToken); + Assert.Equal(1, Volatile.Read(ref exchangeCount)); + + options.Set(new DisseminationOptions { Enabled = true }); + Assert.Equal(2, await exchanges.Reader.ReadAsync(TestContext.Current.CancellationToken)); + } + finally + { + using var shutdown = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + shutdown.CancelAfter(TimeSpan.FromSeconds(5)); + await observer.OnStop(shutdown.Token); + } + + Assert.Equal(0, options.SubscriptionCount); + } + + private sealed class ReviewOptionsMonitor(DisseminationOptions initial) : IOptionsMonitor + { + private readonly object _lock = new(); + private readonly List> _listeners = []; + private DisseminationOptions _current = initial; + + public DisseminationOptions CurrentValue => Volatile.Read(ref _current); + public DisseminationOptions Get(string? name) => CurrentValue; + public int SubscriptionCount + { + get + { + lock (_lock) + { + return _listeners.Count; + } + } + } + + public IDisposable OnChange(Action listener) + { + lock (_lock) + { + _listeners.Add(listener); + } + + return new ReviewSubscription(() => + { + lock (_lock) + { + _listeners.Remove(listener); + } + }); + } + + public void Set(DisseminationOptions value) + { + Volatile.Write(ref _current, value); + Action[] listeners; + lock (_lock) + { + listeners = [.. _listeners]; + } + + foreach (var listener in listeners) + { + listener(value, null); + } + } + } + + private sealed class ReviewSubscription(Action unsubscribe) : IDisposable + { + private Action? _unsubscribe = unsubscribe; + public void Dispose() => Interlocked.Exchange(ref _unsubscribe, null)?.Invoke(); + } + + private sealed class ReviewTimeProvider : TimeProvider + { + private readonly FakeTimeProvider _inner = new(); + private readonly Channel _changes = Channel.CreateUnbounded(); + + public override long TimestampFrequency => _inner.TimestampFrequency; + public override long GetTimestamp() => _inner.GetTimestamp(); + public override DateTimeOffset GetUtcNow() => _inner.GetUtcNow(); + public void Advance(TimeSpan duration) => _inner.Advance(duration); + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = new ReviewTimer(this, _inner.CreateTimer(callback, state, dueTime, period)); + _changes.Writer.TryWrite(dueTime); + return timer; + } + + public async Task WaitForChange(TimeSpan expected, CancellationToken cancellationToken) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(5)); + while (await _changes.Reader.ReadAsync(deadline.Token) != expected) + { + } + } + + private sealed class ReviewTimer(ReviewTimeProvider owner, ITimer inner) : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) + { + var changed = inner.Change(dueTime, period); + owner._changes.Writer.TryWrite(dueTime); + return changed; + } + + public void Dispose() => inner.Dispose(); + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index 60d499127ec..6178be6b429 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -33,7 +33,7 @@ namespace UnitTests.Dissemination; [TestSuite("BVT")] [TestProvider("None")] [TestArea("Dissemination")] -public class DisseminationProtocolTests +public partial class DisseminationProtocolTests { [Fact] public async Task PublishPropagatesCancellationBeforeQueueing() From 0e87e3bd58a8d0d865f33d12a37a54ac2be401e7 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 13 Sep 2026 03:23:10 -0700 Subject: [PATCH 06/50] fix(dissemination): bound receive work and application lifetimes --- .../implementation/runtime-dissemination.md | 10 +- .../Options/DisseminationOptions.cs | 15 +- .../IDisseminationSystemTarget.cs | 7 + .../Dissemination/DisseminationProtocol.cs | 258 ++++- ...eminationProtocolTests.MembershipReview.cs | 5 +- ...onProtocolTests.OwnerCancellationReview.cs | 73 ++ ...sseminationProtocolTests.ProtocolReview.cs | 912 ++++++++++++++++++ 7 files changed, 1242 insertions(+), 38 deletions(-) create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index d14bcb86fb0..5313b1ee650 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -75,15 +75,15 @@ The subsystem and each namespace default to disabled. Enable | 32 | Per-silo active local broadcast attempts. | -| | 8,192 | Values materialized in one batch. | -| | 1 MiB | Serialized payload bytes in one batch. | +| | 8,192 | Values materialized in an outgoing batch or examined in an incoming batch or repair response. | +| | 1 MiB | Serialized payload bytes in an outgoing batch or admitted from an incoming batch or repair response. | | | 2 | Target depth used to derive fanout. | | / | 4 / 32 | Bounds for derived fanout. | | | 5 seconds | Repair-round cadence and retry-delay ceiling. | | | 3 | Maximum peers selected in one repair round and independent per-silo active local repair attempt limit. | | | 1,024 | Distinct retained keys per namespace and peer. | | | 100 ms | Normal-priority batching window. | -| | 30 seconds | Per-hop transport and application lifetime. | +| | 30 seconds | Independent local transport and application budgets for each hop. | | | 10 seconds | Quiet period before a digest is offered for repair. | | | 1 MiB | Maximum serialized value size for the namespace. | @@ -121,6 +121,8 @@ Repair has its own admission gate, independent of broadcast slots. Across overla Digests identify namespace-owned keys, including missing state at version zero, so repair can discover work rejected during queue admission. Recently advanced streams suppress redundant probes until `ExpectedUpdateCadence` elapses. Responses obey item, batch-byte, payload-byte, and hop-lifetime bounds. Persistent per-requester cursors rotate truncated responses so hot early keys cannot starve later candidates. Cursor state for non-members is least-recently-used and bounded to 64 entries. +Requests advertise the receiver's item and payload-byte limits. Responders use the smaller local and advertised limits, allowing their response cursor to provide fair service even when peers use different batch sizes. Older requests use the responder's own limits. Receivers independently cap incoming broadcasts and repair responses before grouping or applying values. Broadcast acknowledgments cover only the admitted keys at their actual local versions. Receive cursors advance through bounded ordered intervals for oversized deliveries while preserving the order of each repair chain. Cursor state is scoped to peer and direction, with at most 64 retained non-member cursors. Request digest inventory remains separate from these response admission budgets. + Repairs from each sender remain ordered. Competing sender chains are ranked before application, and the namespace's monotonic apply contract rejects an obsolete or incompatible chain. Responses which completed successfully within their local attempt are still applied when another peer exceeds the round deadline. Late RPC faults are observed, and subsequent rounds repair responses which missed the application window. During a partition, local attempts remain bounded and retries follow the configured cadence; after connectivity and membership views recover, rotating peer selection and cursors continue convergence. ## Mixed-version behavior @@ -133,7 +135,7 @@ Deployment-load publication uses dissemination for confirmed peers and the direc Protocol dictionaries use focused locks for membership projections, response cursors, value-update timestamps, peer capability evidence, and per-peer pump state. Network calls, namespace application, logging, diagnostic callbacks, and waiter continuations run outside those locks. Waiters use asynchronous continuations. -Caller cancellation owns public operation lifetime and is checked before each received value is applied. Per-hop value TTL bounds the local broadcast RPC wait and anti-entropy round lifetime, independently of the runtime's cancellation-acknowledgment setting. Shared admission waits are pump-cancellable and retain their FIFO position while capacity is unavailable. Explicit flush and drain observers use their caller's cancellation token, with no separate observer TTL; supply a bounded token when waiting indefinitely for admission is unacceptable. +Caller cancellation owns public operation lifetime and is checked before each received value is applied. Per-hop value TTL bounds the local broadcast RPC wait and anti-entropy exchange lifetime, independently of the runtime's cancellation-acknowledgment setting. Application has a separate local window capped by the receiving namespace's TTL. Broadcast values age from the start of receiver processing, including delays caused by earlier items; completed anti-entropy responses begin their application window after the exchanges finish. These windows use local elapsed time, preserving operation across clock skew. Namespace owners observe cancellation before committing queued or resumed work, including membership application following a pending refresh. Shared admission waits are pump-cancellable and retain their FIFO position while capacity is unavailable. Explicit flush and drain observers use their caller's cancellation token, with no separate observer TTL; supply a bounded token when waiting indefinitely for admission is unacceptable. Stopping rejects new notifications and repair admissions. Accepted broadcast work drains through acknowledgments, including retries with the usual backoff, within the shutdown caller's budget. Cancellation reports an incomplete drain and stops its queued admissions, local RPC waits, and pump timers. Each local attempt releases its managed admission lease during cleanup. Once pumps stop, the broadcast gate closes to further admissions. Repair cancellation is explicitly forwarded before disposing the round's linked sources, even when the local wait completes cancellation first. diff --git a/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs index d0b44a35a82..d953baf516d 100644 --- a/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs +++ b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs @@ -24,11 +24,17 @@ public sealed class DisseminationOptions /// /// Gets or sets the maximum total payload bytes in one dissemination batch. /// + /// + /// Bounds outgoing batches and the values admitted from each incoming broadcast or repair response. + /// public int MaxBatchBytes { get; set; } = 1024 * 1024; /// /// Gets or sets the maximum number of items in one dissemination batch. /// + /// + /// Bounds outgoing batches and the values examined in each incoming broadcast or repair response. + /// public int MaxBatchItems { get; set; } = 8 * 1024; /// @@ -152,11 +158,14 @@ public sealed class DisseminationNamespaceOptions public TimeSpan MaxCoalescingDelay { get; set; } = TimeSpan.FromMilliseconds(100); /// - /// Gets or sets how long a namespace value remains useful. + /// Gets or sets the local transport and application budgets for a namespace value. /// /// - /// The lifetime bounds transport and application independently for each dissemination hop. Forwarding - /// re-materializes the value with a new hop lifetime. + /// Each hop has independent local transport and application windows. Broadcast application ages from + /// the start of receiver processing, including earlier items in the batch. Anti-entropy application + /// starts a new window after exchanges finish. Received lifetimes are capped by the local namespace + /// setting, and owners observe cancellation before applying queued state. Forwarding re-materializes + /// the current value with a new hop lifetime. These local windows use each silo's own clock. /// /// The lifetime is 30 seconds by default and must be greater than zero. public TimeSpan StaleItemTtl { get; set; } = TimeSpan.FromSeconds(30); diff --git a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs index e50c563a47e..4b4a465e734 100644 --- a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs +++ b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs @@ -262,6 +262,13 @@ internal sealed class DisseminationAntiEntropyRequest [Id(2)] public List SupportedNamespaces { get; init; } = []; + + // Older requests use the responder's own limits. + [Id(3)] + public int? MaxResponseItems { get; init; } + + [Id(4)] + public int? MaxResponseBytes { get; init; } } [GenerateSerializer, Immutable] diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index 7c5b2d44a00..9384614b3e8 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -23,6 +23,9 @@ internal sealed partial class DisseminationProtocol private readonly object _antiEntropyResponseCursorLock = new(); private readonly Dictionary _antiEntropyResponseCursors = []; private long _antiEntropyResponseCursorAccess; + private readonly object _receivedBatchCursorLock = new(); + private readonly Dictionary<(SiloAddress Peer, bool AntiEntropy), ReceivedBatchCursor> _receivedBatchCursors = []; + private long _receivedBatchCursorAccess; private readonly object _valueUpdateLock = new(); private readonly Dictionary _lastValueUpdates = []; private readonly object _peerSupportLock = new(); @@ -124,6 +127,7 @@ public async Task ReceiveBroadcast( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + var receivedTimestamp = _timeProvider.GetTimestamp(); var options = _options.CurrentValue; if (!options.Enabled) { @@ -135,14 +139,17 @@ public async Task ReceiveBroadcast( var receivedKeys = new Dictionary>(); var unsupportedNamespaces = new List(); - foreach (var (namespaceName, values) in batch.Values) + foreach (var namespaceName in batch.Values.Keys) { - if (!TryGetEnabledNamespace(namespaceName, out var disseminationNamespace)) + if (!TryGetEnabledNamespace(namespaceName, out _)) { unsupportedNamespaces.Add(namespaceName); - continue; } + } + foreach (var (namespaceName, values) in SelectReceivedValues(batch.Sender, batch.Values, options, antiEntropy: false)) + { + var disseminationNamespace = _namespaces[namespaceName]; DisseminationInstruments.OnBroadcastReceived(disseminationNamespace.Name, "tree", values.Count); ConfirmPeerNamespaces(batch.Sender, [namespaceName]); var namespaceKeys = new Dictionary(); @@ -157,12 +164,23 @@ public async Task ReceiveBroadcast( namespaceName, item.Value.Key, item.Value.ToVersion); - var result = await ApplyReceivedValue( - disseminationNamespace, - item, - batch.Sender, - options, - cancellationToken); + DisseminationApplyResult result; + try + { + result = await ApplyReceivedValue( + disseminationNamespace, + item, + batch.Sender, + options, + receivedTimestamp, + cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + cancellationToken.ThrowIfCancellationRequested(); + throw; + } + if (result is DisseminationApplyResult.Applied) { namespaceKeys[item.Value.Key] = true; @@ -245,7 +263,7 @@ public async Task RunAntiEntropyRound(CancellationToken cancellationToken) roundCancellationToken.ThrowIfCancellationRequested(); // Push traffic suppresses redundant checks; admitted peers share this round's digest snapshot. var requestDigests = CreateAntiEntropyRequestDigests(_timeProvider.GetTimestamp()); - var requests = CreateAntiEntropyRequests(membershipSnapshots, requestDigests, leases.Keys); + var requests = CreateAntiEntropyRequests(membershipSnapshots, requestDigests, leases.Keys, options); if (requests.Count == 0) { return; @@ -320,7 +338,8 @@ public async Task RunAntiEntropyRound(CancellationToken cancellationToken) private Dictionary CreateAntiEntropyRequests( DisseminationMembershipSnapshots membershipSnapshots, Dictionary> requestDigests, - IEnumerable peers) + IEnumerable peers, + DisseminationOptions options) { var result = new Dictionary(); var namespacesByScope = new Dictionary(); @@ -369,6 +388,8 @@ private Dictionary CreateAntiEntro Sender = _localSilo, Digests = peerDigests, SupportedNamespaces = supportedNamespaces, + MaxResponseItems = options.MaxBatchItems, + MaxResponseBytes = options.MaxBatchBytes, }); } } @@ -522,6 +543,9 @@ private async Task ApplyAntiEntropyResponses( DisseminationOptions options, CancellationToken cancellationToken) { + // Completed exchanges get a separate local application window, even if another peer used its whole + // transport budget. No remote clock or transmission delay is inferred from a relative wire lifetime. + var receivedTimestamp = _timeProvider.GetTimestamp(); // Keep each sender's chain intact and rank all repairs which completed within the round's hop lifetime. Dictionary>? repairs = null; foreach (var response in responses) @@ -536,12 +560,10 @@ private async Task ApplyAntiEntropyResponses( // A response only includes namespaces which produced repairs. Absence is not evidence that an // up-to-date or unrelated namespace is unsupported, so confirmations are additive here. ConfirmPeerNamespaces(response.Sender, response.Values.Keys); - foreach (var (namespaceName, values) in response.Values) + foreach (var (namespaceName, values) in SelectReceivedValues(response.Sender, response.Values, options, antiEntropy: true)) { - if (!TryGetEnabledNamespace(namespaceName, out var disseminationNamespace)) - { - continue; - } + cancellationToken.ThrowIfCancellationRequested(); + var disseminationNamespace = _namespaces[namespaceName]; foreach (var stream in values.GroupBy(static item => item.Value.Key)) { @@ -584,17 +606,132 @@ await ApplyReceivedValue( item, candidate.Sender, options, + receivedTimestamp, cancellationToken); } } } } + private Dictionary> SelectReceivedValues( + SiloAddress peer, + Dictionary> values, + DisseminationOptions options, + bool antiEntropy) + { + var result = new Dictionary>(); + var totalCount = values.Values.Sum(static entries => (long)entries.Count); + var cursorKey = (peer, antiEntropy); + long position; + lock (_receivedBatchCursorLock) + { + position = _receivedBatchCursors.TryGetValue(cursorKey, out var cursor) + && cursor.TotalCount == totalCount && cursor.Position < totalCount + ? cursor.Position + : 0; + } + + var skip = position; + var examined = 0; + var byteCount = 0; + foreach (var (namespaceName, entries) in values) + { + if (skip >= entries.Count) + { + skip -= entries.Count; + continue; + } + + if (!TryGetEnabledNamespace(namespaceName, out var disseminationNamespace)) + { + position += entries.Count - skip; + skip = 0; + continue; + } + + for (var index = (int)skip; index < entries.Count; index++) + { + if (examined >= options.MaxBatchItems || byteCount >= options.MaxBatchBytes) + { + goto Complete; + } + + var item = entries[index]; + var payloadBytes = item.Value.Payload.Length; + if (!ValidatePayloadSize(disseminationNamespace, item.Value, options)) + { + // Oversized entries consume inspection capacity, but must not pin the receive cursor. + examined++; + position++; + continue; + } + + if (payloadBytes > options.MaxBatchBytes - byteCount) + { + // Preserve this candidate for a fresh budget instead of discarding a repair-chain suffix. + goto Complete; + } + + if (!result.TryGetValue(namespaceName, out var selected)) + { + selected = []; + result.Add(namespaceName, selected); + } + + selected.Add(item); + examined++; + byteCount += payloadBytes; + position++; + } + + skip = 0; + } + + Complete: + var members = _membership.CurrentSnapshots.AllMembers; + lock (_receivedBatchCursorLock) + { + // Do not wrap within a batch: doing so would reorder a key's delta chain. Subsequent deliveries + // resume an ordered, bounded interval, allowing later keys past a repeatedly rejected/hot prefix. + if (position < totalCount) + { + _receivedBatchCursors[cursorKey] = new(position, totalCount, ++_receivedBatchCursorAccess); + } + else + { + _receivedBatchCursors.Remove(cursorKey); + } + + var nonMembers = _receivedBatchCursors + .Where(entry => !members.ContainsMember(entry.Key.Peer)) + .OrderByDescending(static entry => entry.Value.LastAccess) + .Skip(MaxRetainedNonMemberResponseCursors) + .Select(static entry => entry.Key) + .ToArray(); + foreach (var key in nonMembers) + { + _receivedBatchCursors.Remove(key); + } + } + + return result; + } + public ValueTask ReceiveAntiEntropy( DisseminationAntiEntropyRequest request, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + if (request.MaxResponseItems is { } maxResponseItems) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxResponseItems, nameof(request.MaxResponseItems)); + } + + if (request.MaxResponseBytes is { } maxResponseBytes) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxResponseBytes, nameof(request.MaxResponseBytes)); + } + ConfirmPeerNamespaces(request.Sender, request.SupportedNamespaces); ConfirmPeerNamespaces(request.Sender, request.Digests.Keys); // Incoming digests are passive evidence for existing peer pumps, not a reason to create new ones. @@ -646,6 +783,10 @@ private DisseminationAntiEntropyResponse CreateAntiEntropyResponse( } } + // Honor the recipient's budget at the responder's existing fair cursor. Independently truncating + // rotating responses at the receiver can repeatedly omit the same keys when the limits differ. + var maxResponseItems = Math.Min(options.MaxBatchItems, request.MaxResponseItems ?? int.MaxValue); + var maxResponseBytes = Math.Min(options.MaxBatchBytes, request.MaxResponseBytes ?? int.MaxValue); var valueCount = 0; var byteCount = 0; var truncated = false; @@ -698,8 +839,8 @@ private DisseminationAntiEntropyResponse CreateAntiEntropyResponse( localDigest.Key, peerDigest.Version, toVersion: null, - options.MaxBatchItems - valueCount, - options.MaxBatchBytes - byteCount, + maxResponseItems - valueCount, + maxResponseBytes - byteCount, requestedNamespace.Options.MaxPayloadBytes); var repair = requestedNamespace.CreateRepair(repairRequest); if (repair.Status is DisseminationRepairStatus.InsufficientCapacity) @@ -711,8 +852,8 @@ private DisseminationAntiEntropyResponse CreateAntiEntropyResponse( localDigest.Key, peerDigest.Version, toVersion: null, - options.MaxBatchItems, - options.MaxBatchBytes, + maxResponseItems, + maxResponseBytes, requestedNamespace.Options.MaxPayloadBytes); var emptyBatchRepair = requestedNamespace.CreateRepair(emptyBatchRequest); if (emptyBatchRepair.Status is DisseminationRepairStatus.Produced @@ -870,6 +1011,7 @@ private async ValueTask ApplyReceivedValue( DisseminationBroadcastValue item, SiloAddress sender, DisseminationOptions options, + long receivedTimestamp, CancellationToken cancellationToken) { try @@ -879,6 +1021,7 @@ private async ValueTask ApplyReceivedValue( item, sender, options, + receivedTimestamp, cancellationToken); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -904,6 +1047,7 @@ private async ValueTask ApplyReceivedValueCore( DisseminationBroadcastValue item, SiloAddress sender, DisseminationOptions options, + long receivedTimestamp, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -913,7 +1057,11 @@ private async ValueTask ApplyReceivedValueCore( return DisseminationApplyResult.Rejected; } - if (IsExpired(item)) + var lifetime = item.TimeToLive < disseminationNamespace.Options.StaleItemTtl + ? item.TimeToLive + : disseminationNamespace.Options.StaleItemTtl; + var remainingLifetime = lifetime - _timeProvider.GetElapsedTime(receivedTimestamp); + if (remainingLifetime <= TimeSpan.Zero) { EmitApplyResult(namespaceName, item, sender, DisseminationApplyResult.Obsolete); return DisseminationApplyResult.Obsolete; @@ -926,7 +1074,37 @@ private async ValueTask ApplyReceivedValueCore( return terminalResult; } - var result = await disseminationNamespace.ApplyValueAsync(item.Value, cancellationToken); + using var lifetimeCancellation = new CancellationTokenSource( + remainingLifetime < MaxAntiEntropyRoundLifetime ? remainingLifetime : MaxAntiEntropyRoundLifetime, + _timeProvider); + using var applicationCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + lifetimeCancellation.Token); + DisseminationApplyResult result; + try + { + var application = disseminationNamespace.ApplyValueAsync(item.Value, applicationCancellation.Token); + if (application.IsCompletedSuccessfully) + { + result = application.Result; + } + else + { + // Only this local wait is bounded. Namespace owners must observe cancellation before mutating + // queued state; arbitrary implementations which ignore the token cannot be forcibly stopped. + var applicationTask = application.AsTask(); + applicationTask.Ignore(); + result = await applicationTask.WaitAsync(applicationCancellation.Token); + } + + applicationCancellation.Token.ThrowIfCancellationRequested(); + } + catch (OperationCanceledException) when ( + lifetimeCancellation.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + result = DisseminationApplyResult.Obsolete; + } + EmitApplyResult(namespaceName, item, sender, result); if (result is DisseminationApplyResult.Applied) { @@ -1100,8 +1278,24 @@ private bool ValidatePayloadSize( } var namespaceName = disseminationNamespace.Name; - DisseminationEvents.EmitPayloadDrop(namespaceName, value, _localSilo, "oversize", value.Payload.Length); - DisseminationInstruments.OnPayloadDropped(namespaceName, "oversize"); + try + { + DisseminationEvents.EmitPayloadDrop(namespaceName, value, _localSilo, "oversize", value.Payload.Length); + } + catch (Exception exception) + { + LogDebugDisseminationDiagnosticFailed(_logger, exception, namespaceName, value.Key); + } + + try + { + DisseminationInstruments.OnPayloadDropped(namespaceName, "oversize"); + } + catch (Exception exception) + { + LogDebugDisseminationDiagnosticFailed(_logger, exception, namespaceName, value.Key); + } + return false; } @@ -1237,9 +1431,6 @@ private static bool TryGetTerminalApplyResult( return true; } - private bool IsExpired(DisseminationBroadcastValue item) => - item.TimeToLive <= TimeSpan.Zero; - private DisseminationBroadcastValue CreateBroadcastValue( IDisseminationNamespace disseminationNamespace, DisseminationValue value) => @@ -1260,7 +1451,14 @@ private void EmitApplyResult(DisseminationNamespace namespaceName, Dissemination LogDebugDisseminationDiagnosticFailed(_logger, exception, namespaceName, item.Value.Key); } - DisseminationInstruments.OnValueApplied(namespaceName, result); + try + { + DisseminationInstruments.OnValueApplied(namespaceName, result); + } + catch (Exception exception) + { + LogDebugDisseminationDiagnosticFailed(_logger, exception, namespaceName, item.Value.Key); + } } private static int GetDigestCount(Dictionary> digest) => digest.Values.Sum(entries => entries.Count); @@ -1303,6 +1501,8 @@ private static int CompareAntiEntropyRepairs(AntiEntropyRepair left, AntiEntropy private readonly record struct AntiEntropyResponseCursor(int Position, long LastAccess); + private readonly record struct ReceivedBatchCursor(long Position, long TotalCount, long LastAccess); + private readonly record struct AntiEntropyRepair( IDisseminationNamespace Namespace, List Items, @@ -1330,7 +1530,7 @@ private readonly record struct AntiEntropyResponseCandidate( [LoggerMessage( Level = LogLevel.Debug, - Message = "Dissemination apply diagnostic failed for namespace {Namespace}, key {Key}.")] + Message = "Dissemination value diagnostic failed for namespace {Namespace}, key {Key}.")] private static partial void LogDebugDisseminationDiagnosticFailed( ILogger logger, Exception exception, diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs index 68b71c3850c..a841c2d39d9 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs @@ -345,7 +345,8 @@ public async Task MembershipNamespaceLegacyInventoryDivergenceConvergesAfterHear private static MembershipTableManager CreateMembershipReviewManager( SiloAddress local, - out IFatalErrorHandler fatalErrorHandler) + out IFatalErrorHandler fatalErrorHandler, + IMembershipTable membershipTable = null!) { var timer = Substitute.For(); var timers = Substitute.For(); @@ -354,7 +355,7 @@ private static MembershipTableManager CreateMembershipReviewManager( return new MembershipTableManager( new FakeLocalSiloDetails(local), Options.Create(new ClusterMembershipOptions()), - Substitute.For(), + membershipTable ?? Substitute.For(), fatalErrorHandler, Substitute.For(), NullLogger.Instance, diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs new file mode 100644 index 00000000000..112169db208 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs @@ -0,0 +1,73 @@ +#nullable enable + +using System; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using NSubstitute; +using Orleans; +using Orleans.Runtime; +using Orleans.Runtime.MembershipService; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task MembershipOwnerRechecksCancellationAfterSharedRefreshCompleted() + { + var read = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var table = Substitute.For(); + table.ReadAllAsync(Arg.Any()).Returns(read.Task); + using var manager = CreateMembershipReviewManager(CreateSilo(39601), out _, table); + var refresh = manager.Refresh(cancellationToken: TestContext.Current.CancellationToken); + var context = new MembershipReviewContinuationContext(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var incoming = CreateMembershipSnapshot( + 2, CreateMembershipEntry(CreateSilo(39602), SiloStatus.Active, DateTime.UnixEpoch)); + Task gossip; + var previousContext = SynchronizationContext.Current; + SynchronizationContext.SetSynchronizationContext(context); + try + { + gossip = ((IMembershipManager)manager).ProcessGossipSnapshot(incoming, cancellation.Token); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previousContext); + } + + Assert.False(gossip.IsCompleted); + read.SetResult(new MembershipTableData(new TableVersion(1, "1"))); + var continuation = await context.TakeContinuation(TestContext.Current.CancellationToken); + await refresh.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var committed = manager.MembershipTableSnapshot; + Assert.Equal(new MembershipVersion(1), committed.Version); + + cancellation.Cancel(); + continuation.Callback(continuation.State); + + var exception = await Assert.ThrowsAnyAsync( + () => gossip.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Same(committed, manager.MembershipTableSnapshot); + Assert.DoesNotContain(CreateSilo(39602), manager.MembershipTableSnapshot.Entries.Keys); + } + + private sealed class MembershipReviewContinuationContext : SynchronizationContext + { + private readonly Channel<(SendOrPostCallback Callback, object? State)> _continuations = + Channel.CreateUnbounded<(SendOrPostCallback, object?)>(); + + public override void Post(SendOrPostCallback callback, object? state) => + _continuations.Writer.TryWrite((callback, state)); + + public async Task<(SendOrPostCallback Callback, object? State)> TakeContinuation(CancellationToken cancellationToken) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(5)); + return await _continuations.Reader.ReadAsync(deadline.Token); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs new file mode 100644 index 00000000000..fc0627f8106 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs @@ -0,0 +1,912 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.Scheduler; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task ProtocolReviewPayloadObserverFailureDoesNotBlockLaterRepair() + { + var local = CreateSilo(39501); + var peer = CreateSilo(39502); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.SetValue("oversize", 1); + ns.Inner.SetValue("healthy", 2); + ns.Options.MaxPayloadBytes = 8; + ns.RepairHandler = request => request.Key == new DisseminationKey("oversize") + ? DisseminationRepairResult.Produced(1, [new(request.Key, 0, 1, new byte[9])]) + : ns.Inner.CreateRepair(request); + var logger = new Phase6ProtocolLogger(); + var protocol = CreatePhase6Protocol(new FakeTransport(local, peer), ns, logger); + var observer = new ProtocolReviewPayloadObserver(local, ns.Name); + using var subscription = DisseminationEvents.Listener.Subscribe( + observer, static name => name == "Dissemination.PayloadDrop"); + try + { + var request = new DisseminationAntiEntropyRequest + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = CreateAntiEntropyRequestDigest(ns.Name, ("oversize", 0), ("healthy", 0)), + }; + var response = await protocol.ReceiveAntiEntropy(request, TestContext.Current.CancellationToken); + var value = Assert.Single(GetAntiEntropyResponseValues(response)).Value; + Assert.Equal(new DisseminationKey("healthy"), value.Key); + Assert.Equal(2, value.ToVersion); + Assert.False(response.Truncated); + Assert.Equal(1, observer.Count); + var diagnostic = Assert.Single(logger.Entries); + Assert.Same(observer.Failure, diagnostic.Exception); + Assert.Equal(ns.Name, diagnostic.State["Namespace"]); + Assert.Equal(new DisseminationKey("oversize"), diagnostic.State["Key"]); + + ns.RepairHandler = null; + response = await protocol.ReceiveAntiEntropy(request, TestContext.Current.CancellationToken); + Assert.Equal(new[] { "oversize", "healthy" }, + GetAntiEntropyResponseValues(response).Select(static item => item.Value.Key.ToString())); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task ProtocolReviewValueMetricFailuresDoNotBlockRepairOrApplication() + { + var local = CreateSilo(39503); + var peer = CreateSilo(39504); + var ns = new ProtocolReviewNamespace(local, "protocol-review-metric-isolation"); + ns.Inner.SetValue("oversize", 1); + ns.Inner.SetValue("healthy", 2); + ns.Options.MaxPayloadBytes = 8; + ns.RepairHandler = request => request.Key == new DisseminationKey("oversize") + ? DisseminationRepairResult.Produced(1, [new(request.Key, 0, 1, new byte[9])]) + : ns.Inner.CreateRepair(request); + var logger = new Phase6ProtocolLogger(); + var protocol = CreatePhase6Protocol(new FakeTransport(local, peer), ns, logger); + var failures = new List(); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == DisseminationInstruments.MeterName + && instrument.Name is "orleans-dissemination-payload-dropped" or "orleans-dissemination-values-applied") + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, _, tags, _) => + { + foreach (var tag in tags) + { + if (tag.Key == "namespace" && Equals(tag.Value, ns.Name)) + { + failures.Add(instrument.Name); + throw new InvalidOperationException("Metric observer failure."); + } + } + }); + listener.Start(); + try + { + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = CreateAntiEntropyRequestDigest(ns.Name, ("oversize", 0), ("healthy", 0)), + }, TestContext.Current.CancellationToken); + Assert.Equal(new DisseminationKey("healthy"), Assert.Single(GetAntiEntropyResponseValues(response)).Value.Key); + var acknowledgment = await protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = peer, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(peer, "new", 3)), + }, TestContext.Current.CancellationToken); + Assert.Equal(3, Assert.Single(acknowledgment.Acknowledgments[ns.Name]).Version); + Assert.Equal(3, ns.GetVersion("new")); + Assert.Equal( + new[] { "orleans-dissemination-payload-dropped", "orleans-dissemination-values-applied" }, + failures); + Assert.Equal(2, logger.Entries.Count); + Assert.All(logger.Entries, static entry => Assert.Equal("Metric observer failure.", entry.Exception.Message)); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProtocolReviewMalformedLoadPayloadRejectsAndContinues(bool antiEntropy) + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var options = new DeploymentLoadPublisherOptions(); + options.Dissemination.Enabled = true; + var ns = new DeploymentLoadStatisticsDisseminationNamespace( + harness.Publisher, new TestOptionsMonitor(options), serializer); + var statistics = CreatePhase5Statistics(1); + var malformed = new DisseminationValue(harness.ActiveOne, 0, statistics.DateTime.Ticks, new byte[] { 0xff }); + var valid = ns.CreateValue(harness.ActiveTwo, statistics); + var transport = new FakeTransport(harness.Local, harness.ActiveOne, harness.ActiveTwo); + var logger = new Phase6ProtocolLogger(); + var protocol = CreatePhase6Protocol(transport, ns, logger, settings => settings.Overlay.AntiEntropyPeerCount = 1); + using var observer = new Phase6ApplyObserver(ns.Name, malformed.Key, harness.Local, harness.ActiveOne); + try + { + var values = CreateValueGroups(ns.Name, + CreateDisseminationValue(harness.ActiveOne, malformed), + CreateDisseminationValue(harness.ActiveOne, valid)); + if (antiEntropy) + { + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult( + new DisseminationAntiEntropyResponse { Sender = harness.ActiveOne, Values = values }); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + } + else + { + var acknowledgment = await protocol.ReceiveBroadcast( + new DisseminationBroadcastBatch { Sender = harness.ActiveOne, Values = values }, + TestContext.Current.CancellationToken); + Assert.Equal(new long[] { 0, valid.ToVersion }, + acknowledgment.Acknowledgments[ns.Name].Select(static digest => digest.Version)); + } + + Assert.Equal(0, ns.GetVersion(malformed.Key)); + Assert.Equal(valid.ToVersion, ns.GetVersion(valid.Key)); + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); + Assert.Equal(statistics.ActivationCount, + harness.Publisher.PeriodicStatistics[harness.ActiveTwo].ActivationCount); + Assert.Single(observer.Events); + var failure = Assert.Single(logger.Entries); + Assert.Equal(malformed.Key, failure.State["Key"]); + Assert.Equal(malformed.ToVersion, failure.State["Version"]); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProtocolReviewApplicationWaitIsLocallyBounded(bool cancelCaller) + { + var local = CreateSilo(39511); + var peer = CreateSilo(39512); + var transport = new FakeTransport(local, peer); + var clock = new FakeTimeProvider(); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.ExpectedKeys.Add("blocked"); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + CancellationToken applicationToken = default; + ns.ApplyHandler = (_, token) => + { + applicationToken = token; + entered.TrySetResult(); + return new(release.Task); + }; + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = CreateValueGroups(ns.Inner.CreateItem(peer, "blocked", 1)), + }); + var protocol = CreateProtocol(transport, [ns], timeProvider: clock); + using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + try + { + var round = protocol.RunAntiEntropyRound(caller.Token); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.False(round.IsCompleted); + if (cancelCaller) + { + await caller.CancelAsync(); + await Assert.ThrowsAnyAsync( + () => round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + else + { + clock.Advance(TimeSpan.FromSeconds(1)); + await round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + Assert.True(applicationToken.IsCancellationRequested); + Assert.False(release.Task.IsCompleted); + Assert.Equal(0, ns.GetVersion("blocked")); + ns.ApplyHandler = null; + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.Equal(1, ns.GetVersion("blocked")); + Assert.Equal(2, transport.AntiEntropyRequests.Count); + Assert.Equal(0, Assert.Single(transport.AntiEntropyRequests[1].Request.Digests[ns.Name]).Version); + release.SetResult(DisseminationApplyResult.Applied); + Assert.Equal(1, ns.GetVersion("blocked")); + Assert.Equal(1, ns.Inner.ApplyCounts["blocked"]); + } + finally + { + release.TrySetResult(DisseminationApplyResult.Rejected); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task ProtocolReviewSlowPeerDoesNotExpireCompletedResponseApplication() + { + var local = CreateSilo(39521); + var good = CreateSilo(39522); + var slow = CreateSilo(39523); + var transport = new FakeTransport(local, good, slow); + var clock = new FakeTimeProvider(); + var ns = new ProtocolReviewNamespace(local); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + ns.Inner.ExpectedKeys.Add("value"); + var slowEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var slowRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var applyEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var applyRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + ns.ApplyHandler = async (value, token) => + { + applyEntered.TrySetResult(); + await applyRelease.Task.WaitAsync(token); + return await ns.Inner.ApplyValueAsync(value, token); + }; + transport.ExchangeAntiEntropyHandler = (peer, _, _) => + { + if (peer.Equals(slow)) + { + slowEntered.TrySetResult(); + return new(slowRelease.Task); + } + + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = good, + Values = CreateValueGroups(ns.Inner.CreateItem(good, "value", 1)), + }); + }; + var protocol = CreateProtocol(transport, [ns], options => options.Overlay.AntiEntropyPeerCount = 2, clock); + try + { + var round = protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + await slowEntered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + clock.Advance(TimeSpan.FromSeconds(1)); + await applyEntered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.False(round.IsCompleted); + Assert.False(slowRelease.Task.IsCompleted); + applyRelease.SetResult(); + await round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, ns.GetVersion("value")); + Assert.Equal(1, ns.Inner.ApplyCounts["value"]); + } + finally + { + applyRelease.TrySetResult(); + slowRelease.TrySetResult(new DisseminationAntiEntropyResponse { Sender = slow }); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(1)] + public async Task ProtocolReviewBroadcastUsesLocalReceiptAge(int followingTtlSeconds) + { + var local = CreateSilo(39531); + var peer = CreateSilo(39532); + var clock = new FakeTimeProvider(); + var ns = new ProtocolReviewNamespace(local); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(10); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + ns.ApplyHandler = async (value, token) => + { + entered.TrySetResult(); + await release.Task.WaitAsync(token); + return await ns.Inner.ApplyValueAsync(value, token); + }; + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns], timeProvider: clock); + try + { + var receive = protocol.ReceiveBroadcast(CreateBroadcastBatch(peer, + ns.Inner.CreateItem(peer, "first", 1), + new DisseminationBroadcastValue + { + Value = ns.Inner.CreateValue("later", 1), + TimeToLive = TimeSpan.FromSeconds(followingTtlSeconds), + }), TestContext.Current.CancellationToken); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + clock.Advance(TimeSpan.FromSeconds(1)); + release.SetResult(); + var response = await receive.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(new long[] { 1, 0 }, response.Acknowledgments[ns.Name].Select(static digest => digest.Version)); + Assert.Equal(new[] { new DisseminationKey("first") }, ns.Attempts.Select(static value => value.Key)); + Assert.Equal(1, ns.GetVersion("first")); + Assert.Equal(0, ns.GetVersion("later")); + } + finally + { + release.TrySetResult(); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task ProtocolReviewExpiredQueuedLoadDoesNotMutateOwnerState() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var options = new DeploymentLoadPublisherOptions(); + options.Dissemination.Enabled = true; + options.Dissemination.StaleItemTtl = TimeSpan.FromSeconds(1); + var ns = new DeploymentLoadStatisticsDisseminationNamespace( + harness.Publisher, new TestOptionsMonitor(options), + services.GetRequiredService()); + var clock = new FakeTimeProvider(); + var ownerEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseOwner = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, _) => + { + if (silo.Equals(harness.Local)) + { + ownerEntered.TrySetResult(); + releaseOwner.Task.GetAwaiter().GetResult(); + } + })); + var protocol = CreateProtocol(new FakeTransport(harness.Local, harness.ActiveOne), [ns], timeProvider: clock); + var owner = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); + try + { + await ownerEntered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var value = ns.CreateValue(harness.ActiveOne, CreatePhase5Statistics(2)); + var receive = protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = harness.ActiveOne, + Values = CreateValueGroups(ns.Name, CreateDisseminationValue(harness.ActiveOne, value)), + }, TestContext.Current.CancellationToken); + Assert.False(receive.IsCompleted); + clock.Advance(TimeSpan.FromSeconds(1)); + var response = await receive.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(0, Assert.Single(response.Acknowledgments[ns.Name]).Version); + Assert.False(owner.IsCompleted); + releaseOwner.SetResult(); + await owner.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await harness.Publisher.RunOrQueueTask( + token => + { + token.ThrowIfCancellationRequested(); + return Task.FromResult(true); + }, TestContext.Current.CancellationToken); + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); + Assert.Equal(0, ns.GetVersion(harness.ActiveOne)); + } + finally + { + releaseOwner.TrySetResult(); + await owner.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(2, 1024)] + [InlineData(10, 16)] + [InlineData(2, 16)] + public async Task ProtocolReviewBroadcastCapsAllNamespacesAndAcknowledgesOnlyPrefix(int maxItems, int maxBytes) + { + var local = CreateSilo(39541); + var peer = CreateSilo(39542); + var first = new ProtocolReviewNamespace(local); + var second = new ProtocolReviewNamespace(local, "second"); + var protocol = CreateProtocol(new FakeTransport(local, peer), [first, second], options => + { + options.MaxBatchItems = maxItems; + options.MaxBatchBytes = maxBytes; + }); + var batch = new DisseminationBroadcastBatch + { + Sender = peer, + Values = new() + { + [first.Name] = [first.Inner.CreateItem(peer, "chain", 1), first.Inner.CreateItem(peer, "chain", 2, 1)], + [second.Name] = [second.Inner.CreateItem(peer, "later", 3)], + }, + }; + try + { + var acknowledgment = await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + Assert.Single(acknowledgment.Acknowledgments); + Assert.Equal(2, Assert.Single(acknowledgment.Acknowledgments[first.Name]).Version); + Assert.Equal(new long[] { 1, 2 }, first.Attempts.Select(static value => value.ToVersion)); + Assert.Equal(16, first.Attempts.Sum(static value => value.Payload.Length)); + Assert.Empty(second.Attempts); + Assert.Equal(0, second.GetVersion("later")); + + acknowledgment = await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + Assert.Single(acknowledgment.Acknowledgments); + Assert.Equal(3, Assert.Single(acknowledgment.Acknowledgments[second.Name]).Version); + Assert.Equal(2, first.Attempts.Count); + Assert.Single(second.Attempts); + Assert.Equal(8, second.Attempts.Sum(static value => value.Payload.Length)); + Assert.Equal(3, second.GetVersion("later")); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProtocolReviewReceiveCursorPassesRejectedHeadAndPreservesChain(bool antiEntropy) + { + var local = CreateSilo(39551); + var peer = CreateSilo(39552); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.ExpectedKeys.UnionWith(new DisseminationKey[] { "bad", "chain" }); + ns.Options.ExpectedUpdateCadence = TimeSpan.Zero; + var values = CreateValueGroups( + CreateDisseminationValue(peer, new DisseminationValue("bad", 0, 1, new byte[9])), + ns.Inner.CreateItem(peer, "chain", 1), + ns.Inner.CreateItem(peer, "chain", 2, 1)); + var transport = new FakeTransport(local, peer); + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult( + new DisseminationAntiEntropyResponse { Sender = peer, Values = values }); + var protocol = CreateProtocol(transport, [ns], options => + { + options.MaxBatchItems = 1; + options.MaxBatchBytes = 8; + }); + try + { + for (var round = 0; round < 3; round++) + { + if (antiEntropy) + { + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + } + else + { + var response = await protocol.ReceiveBroadcast( + new DisseminationBroadcastBatch { Sender = peer, Values = values }, + TestContext.Current.CancellationToken); + if (round == 0) + { + Assert.Empty(response.Acknowledgments); + } + else + { + Assert.Equal(round, Assert.Single(response.Acknowledgments[ns.Name]).Version); + } + } + + Assert.Equal(round, ns.GetVersion("chain")); + } + + Assert.Equal(0, ns.GetVersion("bad")); + Assert.Equal(new long[] { 1, 2 }, ns.Attempts.Select(static value => value.ToVersion)); + Assert.Equal(new long[] { 0, 1 }, ns.Attempts.Select(static value => value.FromVersion)); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(2, 1024)] + [InlineData(10, 16)] + [InlineData(2, 16)] + public async Task ProtocolReviewAntiEntropyCapsEachResponseBeforeGrouping(int maxItems, int maxBytes) + { + var local = CreateSilo(39561); + var badPeer = CreateSilo(39562); + var goodPeer = CreateSilo(39563); + var ns = new ProtocolReviewNamespace(local); + var other = new ProtocolReviewNamespace(local, "other"); + var transport = new FakeTransport(local, badPeer, goodPeer); + transport.ExchangeAntiEntropyHandler = (peer, _, _) => + { + var first = peer.Equals(badPeer) + ? CreateDisseminationValue(peer, new DisseminationValue("chain", 0, 1, BitConverter.GetBytes(99L))) + : ns.Inner.CreateItem(peer, "chain", 1); + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = new() + { + [ns.Name] = [first, ns.Inner.CreateItem(peer, "chain", 2, 1)], + [other.Name] = [other.Inner.CreateItem(peer, "omitted", 3)], + }, + }); + }; + var protocol = CreateProtocol(transport, [ns, other], options => + { + options.Overlay.AntiEntropyPeerCount = 2; + options.MaxBatchItems = maxItems; + options.MaxBatchBytes = maxBytes; + }); + try + { + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.Equal(2, transport.AntiEntropyRequests.Count); + Assert.Equal(2, ns.GetVersion("chain")); + Assert.Equal(new long[] { 1, 1, 2 }, ns.Attempts.Select(static value => value.ToVersion)); + Assert.Equal(24, ns.Attempts.Sum(static value => value.Payload.Length)); + Assert.Equal(2, ns.Inner.ApplyCounts["chain"]); + Assert.Empty(other.Attempts); + Assert.Equal(0, other.GetVersion("omitted")); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.Equal(3, other.GetVersion("omitted")); + Assert.Equal(2, other.Attempts.Count); + Assert.Equal(16, other.Attempts.Sum(static value => value.Payload.Length)); + Assert.Equal(3, ns.Attempts.Count); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProtocolReviewLargerResponderOptionsRepairThroughLocalCaps(bool ignoresBudgets) + { + var local = CreateSilo(39571); + var peer = CreateSilo(39572); + var receiver = new ProtocolReviewNamespace(local); + var source = new FakeNamespace(peer); + DisseminationKey[] keys = ["first", "second", "last"]; + receiver.Inner.ExpectedKeys.UnionWith(keys); + foreach (var key in keys) + { + source.SetValue(key, 1); + } + + var responses = new List(); + var remote = CreateProtocol(new FakeTransport(peer, local), source, options => + { + options.MaxBatchItems = 10; + options.MaxBatchBytes = 1024; + }); + var transport = new FakeTransport(local, peer); + transport.ExchangeAntiEntropyHandler = async (_, request, token) => + { + Assert.Equal(2, request.MaxResponseItems); + Assert.Equal(16, request.MaxResponseBytes); + if (ignoresBudgets) + { + request = new DisseminationAntiEntropyRequest + { + Sender = request.Sender, + Digests = request.Digests, + SupportedNamespaces = request.SupportedNamespaces, + }; + } + + var response = await remote.ReceiveAntiEntropy(request, token); + responses.Add(response); + return response; + }; + var protocol = CreateProtocol(transport, [receiver], options => + { + options.MaxBatchItems = 2; + options.MaxBatchBytes = 16; + }); + try + { + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.Equal(ignoresBudgets ? 3 : 2, GetAntiEntropyResponseValues(Assert.Single(responses)).Count()); + Assert.Equal(ignoresBudgets ? 24 : 16, GetAntiEntropyResponseValues(responses[0]).Sum(static item => item.Value.Payload.Length)); + Assert.Equal(keys.Take(2), receiver.Attempts.Select(static value => value.Key)); + Assert.Equal(16, receiver.Attempts.Sum(static value => value.Payload.Length)); + Assert.Equal(0, receiver.GetVersion(keys[2])); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.Equal(keys[2], Assert.Single(GetAntiEntropyResponseValues(responses[1])).Value.Key); + Assert.Equal(keys, receiver.Attempts.Select(static value => value.Key)); + Assert.All(keys, key => Assert.Equal(1, receiver.GetVersion(key))); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + await remote.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task ProtocolReviewAdvancingInventoryDoesNotAliasResponderAndReceiverCursors() + { + var local = CreateSilo(39591); + var peer = CreateSilo(39592); + var clock = new FakeTimeProvider(); + var receiver = new ProtocolReviewNamespace(local); + receiver.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(1); + var source = new FakeNamespace(peer); + DisseminationKey[] keys = ["A", "B", "C", "D"]; + receiver.Inner.ExpectedKeys.UnionWith(keys); + var responses = new List(); + var remote = CreateProtocol(new FakeTransport(peer, local), source, options => + { + options.MaxBatchItems = 2; + options.MaxBatchBytes = 16; + }, clock); + var transport = new FakeTransport(local, peer); + transport.ExchangeAntiEntropyHandler = async (_, request, token) => + { + var response = await remote.ReceiveAntiEntropy(request, token); + responses.Add(response); + return response; + }; + var protocol = CreateProtocol(transport, [receiver], options => + { + options.MaxBatchItems = 1; + options.MaxBatchBytes = 8; + }, clock); + try + { + for (var round = 1; round <= keys.Length; round++) + { + foreach (var key in keys) + { + source.SetValue(key, round); + } + + clock.Advance(TimeSpan.FromSeconds(2)); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + } + + Assert.Equal(keys, receiver.Attempts.Select(static value => value.Key)); + Assert.Equal(new long[] { 1, 2, 3, 4 }, keys.Select(receiver.GetVersion)); + Assert.Equal(4, responses.Count); + Assert.Equal(keys, responses.Select(static response => Assert.Single(GetAntiEntropyResponseValues(response)).Value.Key)); + Assert.All(transport.AntiEntropyRequests, request => + { + Assert.Equal(1, request.Request.MaxResponseItems); + Assert.Equal(8, request.Request.MaxResponseBytes); + Assert.Equal(4, request.Request.Digests[receiver.Name].Count); + }); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + await remote.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(0, null, "MaxResponseItems")] + [InlineData(-1, null, "MaxResponseItems")] + [InlineData(null, 0, "MaxResponseBytes")] + [InlineData(null, -1, "MaxResponseBytes")] + public async Task ProtocolReviewInvalidResponseBudgetsHaveNoReceiveSideEffects( + int? maxResponseItems, + int? maxResponseBytes, + string parameterName) + { + var local = CreateSilo(39601); + var peer = CreateSilo(39602); + var ns = new FakeNamespace(local); + ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); + ns.SetValue("pending", 1); + var transport = new FakeTransport(local, peer); + var protocol = CreateProtocol(transport, ns, timeProvider: new FakeTimeProvider()); + try + { + Assert.True(await protocol.Publish(ns, "pending", 1, TestContext.Current.CancellationToken)); + var repairRequests = ns.RepairRequestCount; + var exception = Assert.Throws(() => + { + _ = protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = CreateAntiEntropyRequestDigest(ns.Name, ("pending", 1)), + MaxResponseItems = maxResponseItems, + MaxResponseBytes = maxResponseBytes, + }, TestContext.Current.CancellationToken); + }); + Assert.Equal(parameterName, exception.ParamName); + Assert.Equal(maxResponseItems ?? maxResponseBytes, Assert.IsType(exception.ActualValue)); + Assert.Equal(repairRequests, ns.RepairRequestCount); + Assert.Equal(peer, Assert.Single(protocol.GetUnconfirmedPeers(ns))); + + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var sent = Assert.Single(transport.BroadcastBatches); + Assert.Equal(1, Assert.Single(GetBroadcastValues(sent.Batch)).Value.ToVersion); + Assert.Equal(peer, sent.Peer); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(null, null)] + [InlineData(1, null)] + [InlineData(null, 8)] + [InlineData(1, 8)] + [InlineData(10, 1024)] + public async Task ProtocolReviewResponseBudgetsRoundTripAndBoundEveryRepairProbe(int? maxResponseItems, int? maxResponseBytes) + { + var local = CreateSilo(39611); + var peer = CreateSilo(39612); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var ns = new ProtocolReviewNamespace(local); + DisseminationKey[] keys = ["A", "B", "C"]; + foreach (var key in keys) + { + ns.Inner.SetValue(key, 1); + } + + var requests = new List(); + ns.RepairHandler = request => + { + requests.Add(request); + return ns.Inner.CreateRepair(request); + }; + var request = new DisseminationAntiEntropyRequest + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = CreateAntiEntropyRequestDigest(ns.Name, ("A", 0), ("B", 0), ("C", 0)), + MaxResponseItems = maxResponseItems, + MaxResponseBytes = maxResponseBytes, + }; + var copy = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(request))); + Assert.Equal(maxResponseItems, copy.MaxResponseItems); + Assert.Equal(maxResponseBytes, copy.MaxResponseBytes); + Assert.Equal(peer, copy.Sender); + Assert.Equal(ns.Name, Assert.Single(copy.SupportedNamespaces)); + Assert.Equal(keys, copy.Digests[ns.Name].Select(static digest => digest.Key)); + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns], options => + { + options.MaxBatchItems = 2; + options.MaxBatchBytes = 16; + }); + try + { + var response = await protocol.ReceiveAntiEntropy(copy, TestContext.Current.CancellationToken); + var maxItems = Math.Min(2, maxResponseItems ?? int.MaxValue); + var maxBytes = Math.Min(16, maxResponseBytes ?? int.MaxValue); + var count = Math.Min(maxItems, maxBytes / sizeof(long)); + Assert.Equal(keys.Take(count), GetAntiEntropyResponseValues(response).Select(static item => item.Value.Key)); + Assert.Equal(count * sizeof(long), GetAntiEntropyResponseValues(response).Sum(static item => item.Value.Payload.Length)); + Assert.True(response.Truncated); + Assert.Equal(count + 2, requests.Count); + for (var index = 0; index <= count; index++) + { + Assert.Equal(keys[index], requests[index].Key); + Assert.Equal(maxItems - index, requests[index].MaxItemCount); + Assert.Equal(maxBytes - index * sizeof(long), requests[index].MaxBatchBytes); + } + + Assert.Equal(keys[count], requests[^1].Key); + Assert.Equal(maxItems, requests[^1].MaxItemCount); + Assert.Equal(maxBytes, requests[^1].MaxBatchBytes); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task ProtocolReviewLargerBroadcastSenderContinuesFromAcknowledgedPrefix() + { + var sourceAddress = CreateSilo(39581); + var receiverAddress = CreateSilo(39582); + var clock = new FakeTimeProvider(); + var source = new FakeNamespace(sourceAddress) { ReturnRepairChain = true }; + source.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); + source.PublishValue(source.CreateValue("chain", 1)); + source.PublishValue(source.CreateValue("chain", 2, 1)); + source.PublishValue(source.CreateValue("chain", 3, 2)); + var receiver = new ProtocolReviewNamespace(receiverAddress); + var receivingProtocol = CreateProtocol(new FakeTransport(receiverAddress, sourceAddress), [receiver], options => + { + options.MaxBatchItems = 1; + options.MaxBatchBytes = 8; + }, clock); + var acknowledgments = new List(); + var sentItemCounts = new List(); + var transport = new FakeTransport(sourceAddress, receiverAddress); + transport.SendBroadcastResponseHandler = async (_, batch, token) => + { + sentItemCounts.Add(GetBroadcastValues(batch).Count()); + var acknowledgment = await receivingProtocol.ReceiveBroadcast(batch, token); + acknowledgments.Add(Assert.Single(acknowledgment.Acknowledgments[source.Name]).Version); + return acknowledgment; + }; + var sendingProtocol = CreateProtocol(transport, source, options => + { + options.MaxBatchItems = 10; + options.MaxBatchBytes = 1024; + }, clock); + try + { + Assert.True(await sendingProtocol.Publish(source, "chain", 3, TestContext.Current.CancellationToken)); + await sendingProtocol.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(new long[] { 1, 2, 3 }, acknowledgments); + Assert.Equal(new[] { 3, 2, 1 }, sentItemCounts); + Assert.Equal(new long[] { 0, 1, 2 }, receiver.Attempts.Select(static value => value.FromVersion)); + Assert.Equal(new long[] { 1, 2, 3 }, receiver.Attempts.Select(static value => value.ToVersion)); + Assert.Equal(3, receiver.GetVersion("chain")); + } + finally + { + await sendingProtocol.StopAsync(TestContext.Current.CancellationToken); + await receivingProtocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + private sealed class ProtocolReviewNamespace(SiloAddress local, DisseminationNamespace? name = null) : IDisseminationNamespace + { + public FakeNamespace Inner { get; } = new(local, name); + public DisseminationNamespace Name => Inner.Name; + public DisseminationNamespaceOptions Options => Inner.Options; + public IEnumerable Digests => Inner.Digests; + public List Attempts { get; } = []; + public Func>? ApplyHandler { get; set; } + public Func? RepairHandler { get; set; } + public long GetVersion(DisseminationKey key) => Inner.GetVersion(key); + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) => + RepairHandler is { } handler ? handler(request) : Inner.CreateRepair(request); + + public ValueTask ApplyValueAsync(DisseminationValue value, CancellationToken cancellationToken) + { + Attempts.Add(value); + return ApplyHandler is { } handler ? handler(value, cancellationToken) : Inner.ApplyValueAsync(value, cancellationToken); + } + } + + private sealed class ProtocolReviewPayloadObserver(SiloAddress local, DisseminationNamespace namespaceName) : IObserver> + { + public Exception Failure { get; } = new InvalidOperationException("Payload observer failure."); + public int Count { get; private set; } + + public void OnNext(KeyValuePair value) + { + if (value.Value is DisseminationValueEvent dropped + && dropped.Namespace == namespaceName + && Equals(dropped.LocalSilo, local)) + { + Count++; + throw Failure; + } + } + + public void OnCompleted() { } + public void OnError(Exception error) { } + } +} From 5c3bc39480689669237ffa6bd7bd533d8d4c110e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 13 Sep 2026 03:35:38 -0700 Subject: [PATCH 07/50] style(dissemination): align receive cursor label formatting --- src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index 9384614b3e8..f31c40ff6a5 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -687,7 +687,7 @@ private Dictionary> Se skip = 0; } - Complete: +Complete: var members = _membership.CurrentSnapshots.AllMembers; lock (_receivedBatchCursorLock) { From b596512e87862f2e168e119a4a381db81f2a1169 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 13 Sep 2026 04:04:31 -0700 Subject: [PATCH 08/50] fix(dissemination): preserve completed application outcomes --- .github/workflows/dissemination-evidence.yml | 5 + .../implementation/runtime-dissemination.md | 2 +- .../Dissemination/DisseminationProtocol.cs | 5 +- ...sseminationProtocolTests.FollowupReview.cs | 171 ++++++++++++++++++ 4 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs diff --git a/.github/workflows/dissemination-evidence.yml b/.github/workflows/dissemination-evidence.yml index 37058d4d27c..7a445b8b5e3 100644 --- a/.github/workflows/dissemination-evidence.yml +++ b/.github/workflows/dissemination-evidence.yml @@ -8,11 +8,16 @@ on: - '.github/workflows/dissemination-evidence.yml' - '.github/scripts/dissemination-evidence.ps1' - 'test/Dissemination.IntegrationHarness/**' + - 'src/Orleans.Core.Abstractions/IDs/SiloAddress.cs' - 'src/Orleans.Runtime/Dissemination/**' - 'src/Orleans.Runtime/MembershipService/**' - 'src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs' - 'src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs' + - 'src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs' - 'src/Orleans.Runtime/Hosting/DefaultSiloServices.cs' + - 'src/Orleans.Runtime/OrleansContracts.txt' + - 'src/Orleans.Core/OrleansContracts.txt' + - 'src/Orleans.Core/Runtime/Constants.cs' - 'src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs' - 'src/Orleans.Core/SystemTargetInterfaces/IDeploymentLoadPublisher.cs' - 'src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs' diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index 5313b1ee650..9e1fb85684a 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -135,7 +135,7 @@ Deployment-load publication uses dissemination for confirmed peers and the direc Protocol dictionaries use focused locks for membership projections, response cursors, value-update timestamps, peer capability evidence, and per-peer pump state. Network calls, namespace application, logging, diagnostic callbacks, and waiter continuations run outside those locks. Waiters use asynchronous continuations. -Caller cancellation owns public operation lifetime and is checked before each received value is applied. Per-hop value TTL bounds the local broadcast RPC wait and anti-entropy exchange lifetime, independently of the runtime's cancellation-acknowledgment setting. Application has a separate local window capped by the receiving namespace's TTL. Broadcast values age from the start of receiver processing, including delays caused by earlier items; completed anti-entropy responses begin their application window after the exchanges finish. These windows use local elapsed time, preserving operation across clock skew. Namespace owners observe cancellation before committing queued or resumed work, including membership application following a pending refresh. Shared admission waits are pump-cancellable and retain their FIFO position while capacity is unavailable. Explicit flush and drain observers use their caller's cancellation token, with no separate observer TTL; supply a bounded token when waiting indefinitely for admission is unacceptable. +Caller cancellation owns public operation lifetime and is checked before each received value is applied. Per-hop value TTL bounds the local broadcast RPC wait and anti-entropy exchange lifetime, independently of the runtime's cancellation-acknowledgment setting. Application has a separate local window capped by the receiving namespace's TTL. Broadcast values age from the start of receiver processing, including delays caused by earlier items; completed anti-entropy responses begin their application window after the exchanges finish. These windows use local elapsed time, preserving operation across clock skew. Namespace owners observe cancellation before committing queued or resumed work, including membership application following a pending refresh. A completed namespace result remains authoritative when local expiry races completion; caller cancellation still aborts remaining work. Shared admission waits are pump-cancellable and retain their FIFO position while capacity is unavailable. Explicit flush and drain observers use their caller's cancellation token, with no separate observer TTL; supply a bounded token when waiting indefinitely for admission is unacceptable. Stopping rejects new notifications and repair admissions. Accepted broadcast work drains through acknowledgments, including retries with the usual backoff, within the shutdown caller's budget. Cancellation reports an incomplete drain and stops its queued admissions, local RPC waits, and pump timers. Each local attempt releases its managed admission lease during cleanup. Once pumps stop, the broadcast gate closes to further admissions. Repair cancellation is explicitly forwarded before disposing the round's linked sources, even when the local wait completes cancellation first. diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index f31c40ff6a5..286bfaf0b84 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -1097,7 +1097,7 @@ private async ValueTask ApplyReceivedValueCore( result = await applicationTask.WaitAsync(applicationCancellation.Token); } - applicationCancellation.Token.ThrowIfCancellationRequested(); + cancellationToken.ThrowIfCancellationRequested(); } catch (OperationCanceledException) when ( lifetimeCancellation.IsCancellationRequested && !cancellationToken.IsCancellationRequested) @@ -1321,7 +1321,8 @@ private bool TryValidatePublish( return false; } - // Publishing is safe only if the namespace can materialize a complete repair for an unknown peer. + // Validate the complete repair chain for an unknown peer. Individual values must fit a batch, + // while the chain can be delivered across multiple acknowledged batches. var request = new DisseminationRepairRequest( key, fromVersion: null, diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs new file mode 100644 index 00000000000..87d774ba3dd --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs @@ -0,0 +1,171 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task CompletedApplicationRemainsAppliedWhenItsLocalDeadlineRacesCompletion() + { + var local = CreateSilo(39651); + var sender = CreateSilo(39652); + var child = CreateSilo(39653); + var clock = new FakeTimeProvider(); + var ns = new ProtocolReviewNamespace(local, "terminal-result-review"); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + CancellationToken applicationToken = default; + ns.ApplyHandler = (value, token) => + { + applicationToken = token; + var result = ns.Inner.ApplyValueAsync(value, token); + Assert.True(result.IsCompletedSuccessfully); + clock.Advance(ns.Options.StaleItemTtl); + return result; + }; + var transport = new FakeTransport(local, sender, child); + var protocol = CreateProtocol(transport, [ns], options => options.Overlay.FanOutFactor = static _ => 2, clock); + try + { + var response = await protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = sender, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(sender, "value", 1)), + }, TestContext.Current.CancellationToken); + + Assert.True(applicationToken.IsCancellationRequested); + Assert.Equal(1, ns.GetVersion("value")); + Assert.Equal(1, Assert.Single(response.Acknowledgments[ns.Name]).Version); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var forwarded = Assert.Single(transport.BroadcastBatches); + Assert.Equal(child, forwarded.Peer); + Assert.Equal(1, Assert.Single(GetBroadcastValues(forwarded.Batch)).Value.ToVersion); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.NotEmpty(transport.AntiEntropyRequests); + Assert.All(transport.AntiEntropyRequests, request => Assert.False(request.Request.Digests.ContainsKey(ns.Name))); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task ChangedOversizedBatchCannotAcknowledgeOrApplyAMissingChainPrefix() + { + var local = CreateSilo(39661); + var peer = CreateSilo(39662); + var first = new FakeNamespace(local, "cursor-first"); + var second = new FakeNamespace(local, "cursor-second"); + second.ExpectedKeys.Add("stream"); + var protocol = CreateProtocol( + new FakeTransport(local, peer), + [first, second], + options => options.MaxBatchItems = 1); + try + { + var firstResponse = await protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = peer, + Values = new Dictionary> + { + [first.Name] = [first.CreateItem(peer, "other", 1)], + [second.Name] = [second.CreateItem(peer, "stream", 1)], + }, + }, TestContext.Current.CancellationToken); + Assert.False(firstResponse.Acknowledgments.ContainsKey(second.Name)); + + var changedBatch = new DisseminationBroadcastBatch + { + Sender = peer, + Values = CreateValueGroups(second.Name, + second.CreateItem(peer, "stream", 1), + second.CreateItem(peer, "stream", 2, fromVersion: 1)), + }; + var skippedPrefix = await protocol.ReceiveBroadcast(changedBatch, TestContext.Current.CancellationToken); + Assert.Equal(0, Assert.Single(skippedPrefix.Acknowledgments[second.Name]).Version); + Assert.Equal(0, second.GetVersion("stream")); + + var prefix = await protocol.ReceiveBroadcast(changedBatch, TestContext.Current.CancellationToken); + Assert.Equal(1, Assert.Single(prefix.Acknowledgments[second.Name]).Version); + var suffix = await protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = peer, + Values = CreateValueGroups(second.Name, second.CreateItem(peer, "stream", 2, fromVersion: 1)), + }, TestContext.Current.CancellationToken); + Assert.Equal(2, Assert.Single(suffix.Acknowledgments[second.Name]).Version); + Assert.Equal(1, first.GetVersion("other")); + Assert.Equal(2, second.GetVersion("stream")); + Assert.Equal(2, second.ApplyCounts["stream"]); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task PublicationRejectsSingleValueBeyondGlobalBatchBudget() + { + var local = CreateSilo(39671); + var peer = CreateSilo(39672); + var ns = new FakeNamespace(local); + ns.Options.MaxPayloadBytes = 16; + var payload = new byte[9]; + BitConverter.GetBytes(1L).CopyTo(payload, 0); + ns.PublishValue(new DisseminationValue("value", 0, 1, payload)); + var transport = new FakeTransport(local, peer); + var protocol = CreateProtocol(transport, ns, options => options.MaxBatchBytes = 8); + try + { + Assert.False(await protocol.Publish(ns, "value", 1, TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Empty(transport.BroadcastBatches); + Assert.Equal(0, transport.GetTargetResolutionCount(peer)); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task PublicationPagesCompleteRepairWithinActualSendBudgets() + { + var local = CreateSilo(39681); + var peer = CreateSilo(39682); + var ns = new FakeNamespace(local) { ReturnRepairChain = true }; + ns.PublishValue(ns.CreateValue("chain", 1)); + ns.PublishValue(ns.CreateValue("chain", 2, fromVersion: 1)); + ns.PublishValue(ns.CreateValue("chain", 3, fromVersion: 2)); + var transport = new FakeTransport(local, peer); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxBatchItems = 1; + options.MaxBatchBytes = 8; + }); + try + { + Assert.True(await protocol.Publish(ns, "chain", 3, TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(new long[] { 1, 2, 3 }, + transport.BroadcastBatches.Select(batch => Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion)); + Assert.All(transport.BroadcastBatches, batch => + Assert.Equal(8, Assert.Single(GetBroadcastValues(batch.Batch)).Value.Payload.Length)); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } +} From 111031bb5db671be92542060e2106ccab3d2ae7c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 13 Sep 2026 04:30:05 -0700 Subject: [PATCH 09/50] fix(dissemination): preserve caller cancellation identity --- src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs | 5 +++++ .../DisseminationProtocolTests.ProtocolReview.cs | 3 ++- .../Dissemination/DisseminationProtocolTests.cs | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index 286bfaf0b84..066fb5e6085 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -304,6 +304,11 @@ public async Task RunAntiEntropyRound(CancellationToken cancellationToken) roundCancellationToken.ThrowIfCancellationRequested(); await ApplyAntiEntropyResponses(responses, options, roundCancellationToken); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + cancellationToken.ThrowIfCancellationRequested(); + throw; + } finally { foreach (var lease in leases.Values) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs index fc0627f8106..e95668306c9 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs @@ -221,8 +221,9 @@ public async Task ProtocolReviewApplicationWaitIsLocallyBounded(bool cancelCalle if (cancelCaller) { await caller.CancelAsync(); - await Assert.ThrowsAnyAsync( + var exception = await Assert.ThrowsAnyAsync( () => round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(caller.Token, exception.CancellationToken); } else { diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index 6178be6b429..cdd5dfcfbb1 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -7398,7 +7398,7 @@ public async Task AntiEntropyStopsBeforeNextValueWhenCallerIsCanceledAfterApply( Assert.True(cancellation.IsCancellationRequested); Assert.True(applyCancellation.IsCancellationRequested); - Assert.Equal(applyCancellation, exception.CancellationToken); + Assert.Equal(cancellation.Token, exception.CancellationToken); Assert.Equal([cancelingKey], ns.ApplyAttempts); Assert.Equal(0, ns.GetVersion(laterKey)); } From 7044bbf72d19556814e8bb34bf63f7f6b37b6e7a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 14 Sep 2026 07:29:31 -0700 Subject: [PATCH 10/50] perf(dissemination): reuse active status lookup for cache pruning --- ...entLoadStatisticsDisseminationNamespace.cs | 14 +++--- .../Placement/DeploymentLoadPublisher.cs | 4 +- ...sseminationProtocolTests.ProtocolReview.cs | 43 +++++++++++++++++++ 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs index b8a24bb0b93..31f4425d614 100644 --- a/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs @@ -44,9 +44,9 @@ public IEnumerable Digests { get { - var activeSilos = deploymentLoadPublisher.GetActiveSilosForStatisticsDigest(); + var activeSilos = deploymentLoadPublisher.GetActiveSiloStatusesForStatisticsDigest(); PruneCache(activeSilos); - foreach (var siloAddress in activeSilos) + foreach (var siloAddress in activeSilos.Keys) { yield return new DigestEntry(siloAddress, GetVersion(siloAddress)); } @@ -114,14 +114,16 @@ public ValueTask ApplyValueAsync( return new(deploymentLoadPublisher.ApplyDisseminatedRuntimeStatisticsAsync(siloAddress, statistics, cancellationToken)); } - private void PruneCache(IReadOnlyCollection activeSilos) + private void PruneCache(Dictionary activeSilos) { - var activeSiloSet = activeSilos.ToHashSet(); lock (_cacheLock) { - foreach (var key in _cachedValues.Keys.Where(key => !activeSiloSet.Contains(key)).ToArray()) + foreach (var key in _cachedValues.Keys) { - _cachedValues.Remove(key); + if (!activeSilos.ContainsKey(key)) + { + _cachedValues.Remove(key); + } } } } diff --git a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs index bc3977f0f3b..b93f0224ae2 100644 --- a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs +++ b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs @@ -181,8 +181,8 @@ internal bool IsRuntimeStatisticsObsolete(SiloAddress siloAddress, long timestam _siloStatusOracle.GetApproximateSiloStatus(siloAddress) != SiloStatus.Active || (_periodicStats.TryGetValue(siloAddress, out var old) && old.DateTime.Ticks > timestampTicks); - internal IReadOnlyCollection GetActiveSilosForStatisticsDigest() => - _siloStatusOracle.GetApproximateSiloStatuses(onlyActive: true).Keys; + internal Dictionary GetActiveSiloStatusesForStatisticsDigest() => + _siloStatusOracle.GetApproximateSiloStatuses(onlyActive: true); internal async Task TryPublishStatisticsViaDissemination( SiloRuntimeStatistics myStats, diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs index e95668306c9..9c2ab26e676 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs @@ -185,6 +185,49 @@ public async Task ProtocolReviewMalformedLoadPayloadRejectsAndContinues(bool ant } } + [Fact] + public void LoadNamespaceDigestsPruneInactiveCacheEntriesAndReuseActivePayloads() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var ns = new DeploymentLoadStatisticsDisseminationNamespace( + harness.Publisher, + new TestOptionsMonitor(new()), + serializer); + var statistics = CreatePhase5Statistics(1); + var cached = new Dictionary(); + foreach (var silo in new[] { harness.Local, harness.ActiveOne, harness.ActiveTwo }) + { + harness.Publisher.PeriodicStatistics[silo] = statistics; + cached.Add(silo, ns.CreateValue(silo, statistics)); + } + + Assert.Equal( + cached.Keys.Order(), + ns.Digests.Select(static digest => Assert.IsType(digest.Key.Value)).Order()); + Assert.All(cached, entry => Assert.True(entry.Value.Payload.Equals(ns.CreateValue(entry.Key, statistics).Payload))); + + harness.StatusOracle.SetStatus(harness.ActiveOne, SiloStatus.Dead); + harness.StatusOracle.SetStatus(harness.ActiveTwo, SiloStatus.Joining); + var restarted = SiloAddress.New(harness.ActiveOne.Endpoint, harness.ActiveOne.Generation + 1); + harness.StatusOracle.SetStatus(restarted, SiloStatus.Active); + var digests = ns.Digests.OrderBy(static digest => (SiloAddress)digest.Key.Value).ToArray(); + Assert.Equal( + new[] { harness.Local, restarted }.Order(), + digests.Select(static digest => Assert.IsType(digest.Key.Value))); + Assert.Equal(statistics.DateTime.Ticks, digests.Single(digest => digest.Key == new DisseminationKey(harness.Local)).Version); + Assert.Equal(0, digests.Single(digest => digest.Key == new DisseminationKey(restarted)).Version); + Assert.True(cached[harness.Local].Payload.Equals(ns.CreateValue(harness.Local, statistics).Payload)); + Assert.False(cached[harness.ActiveOne].Payload.Equals(ns.CreateValue(harness.ActiveOne, statistics).Payload)); + Assert.False(cached[harness.ActiveTwo].Payload.Equals(ns.CreateValue(harness.ActiveTwo, statistics).Payload)); + + harness.StatusOracle.SetStatus(harness.Local, SiloStatus.Stopping); + harness.StatusOracle.SetStatus(restarted, SiloStatus.Dead); + Assert.Empty(ns.Digests); + Assert.False(cached[harness.Local].Payload.Equals(ns.CreateValue(harness.Local, statistics).Payload)); + } + [Theory] [InlineData(false)] [InlineData(true)] From 5efb385f4a33631ee80c37ae81f6278fce8cee53 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 14 Sep 2026 07:35:22 -0700 Subject: [PATCH 11/50] test(dissemination): assert silo key type when sorting digests --- .../Dissemination/DisseminationProtocolTests.ProtocolReview.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs index 9c2ab26e676..7dc13cd16f0 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs @@ -212,7 +212,7 @@ public void LoadNamespaceDigestsPruneInactiveCacheEntriesAndReuseActivePayloads( harness.StatusOracle.SetStatus(harness.ActiveTwo, SiloStatus.Joining); var restarted = SiloAddress.New(harness.ActiveOne.Endpoint, harness.ActiveOne.Generation + 1); harness.StatusOracle.SetStatus(restarted, SiloStatus.Active); - var digests = ns.Digests.OrderBy(static digest => (SiloAddress)digest.Key.Value).ToArray(); + var digests = ns.Digests.OrderBy(static digest => Assert.IsType(digest.Key.Value)).ToArray(); Assert.Equal( new[] { harness.Local, restarted }.Order(), digests.Select(static digest => Assert.IsType(digest.Key.Value))); From 05701520de7184e6cff2a0a901b7ba6c55890335 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 14 Sep 2026 18:20:25 -0700 Subject: [PATCH 12/50] fix(dissemination): converge same-version inventory cleanup --- .../implementation/runtime-dissemination.md | 4 +- .../DisseminationSystemTarget.cs | 6 +- .../MembershipDisseminationNamespace.cs | 25 +-- ...isseminationProtocolTests.FinalComments.cs | 157 ++++++++++++++++++ ...eminationProtocolTests.MembershipReview.cs | 12 +- .../DisseminationProtocolTests.QueueReview.cs | 4 + 6 files changed, 192 insertions(+), 16 deletions(-) create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index 9e1fb85684a..a27c962bfd9 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -49,7 +49,7 @@ A `DisseminationValue` names one key and a `[FromVersion, ToVersion]` transition The namespace reports whether a repair is current, produced, unavailable, or unable to fit the supplied item and byte budgets. Produced repairs can be complete or a valid prefix. Prefix acknowledgments establish the receiver's new baseline and let the next batch continue the chain. -Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. It retains 32 snapshots, prefers a smaller diff when the peer baseline is present, and falls back to a full snapshot when history or capacity makes the diff unsuitable. Current diffs identify their entries as the complete target inventory, so table cleanup is reflected even when a receiver retained a different same-version baseline. Older partial-diff payloads retain their incremental semantics. Apply accepts same-version snapshots only when they advance liveness. The membership manager merges maximum per-entry liveness when publishing full snapshots, and diff construction preserves newer local liveness before handing state to that same manager. +Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. It retains 32 snapshots, prefers a smaller diff when the peer baseline is present, and falls back to a full snapshot when history or capacity makes the diff unsuitable. Current diffs identify their entries as the complete target inventory, so table cleanup is reflected even when a receiver retained a different same-version baseline. Older partial-diff payloads retain their incremental semantics. Same-version snapshots can advance liveness or apply an inventory-only cleanup. The cleanup-only path retains every Active entry and each remaining status, so inactive-row removal converges while heartbeats are unchanged. The membership manager merges maximum per-entry liveness and retains its local-death handling; dissemination reports whether that manager actually published changed state. Deployment-load versions are sample timestamps. Each repair is a full latest value. Application runs on the deployment-load publisher's scheduler and accepts samples only for Active silo generations in the membership oracle. Terminal membership transitions remove placement statistics, and the oracle's monotonic membership state rejects later samples for departed generations. @@ -137,7 +137,7 @@ Protocol dictionaries use focused locks for membership projections, response cur Caller cancellation owns public operation lifetime and is checked before each received value is applied. Per-hop value TTL bounds the local broadcast RPC wait and anti-entropy exchange lifetime, independently of the runtime's cancellation-acknowledgment setting. Application has a separate local window capped by the receiving namespace's TTL. Broadcast values age from the start of receiver processing, including delays caused by earlier items; completed anti-entropy responses begin their application window after the exchanges finish. These windows use local elapsed time, preserving operation across clock skew. Namespace owners observe cancellation before committing queued or resumed work, including membership application following a pending refresh. A completed namespace result remains authoritative when local expiry races completion; caller cancellation still aborts remaining work. Shared admission waits are pump-cancellable and retain their FIFO position while capacity is unavailable. Explicit flush and drain observers use their caller's cancellation token, with no separate observer TTL; supply a bounded token when waiting indefinitely for admission is unacceptable. -Stopping rejects new notifications and repair admissions. Accepted broadcast work drains through acknowledgments, including retries with the usual backoff, within the shutdown caller's budget. Cancellation reports an incomplete drain and stops its queued admissions, local RPC waits, and pump timers. Each local attempt releases its managed admission lease during cleanup. Once pumps stop, the broadcast gate closes to further admissions. Repair cancellation is explicitly forwarded before disposing the round's linked sources, even when the local wait completes cancellation first. +Stopping rejects new notifications and repair admissions. Accepted broadcast work drains through acknowledgments, including retries with the usual backoff, within the shutdown caller's budget. Cancellation reports an incomplete drain and stops its queued admissions, local RPC waits, and pump timers. Owned loop cancellation completes normal cleanup; a canceled shutdown caller receives cancellation with its own token after cleanup. Each local attempt releases its managed admission lease during cleanup. Once pumps stop, the broadcast gate closes to further admissions. Repair cancellation is explicitly forwarded before disposing the round's linked sources, even when the local wait completes cancellation first. A stalled destination occupies at most one active local broadcast slot and one independent local repair slot. Hop deadlines release those slots and signal cancellation, allowing queued broadcasts and future repair rounds to continue. Retry backoff, rotating peer selection, per-message budgets, and local admission together bound the pace and size of new attempts. diff --git a/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs b/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs index 654211a4ee9..571742b8377 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs @@ -97,9 +97,9 @@ private async Task StopAsync(CancellationToken cancellationToken) { await _antiEntropyTask.WaitAsync(cancellationToken); } - catch (OperationCanceledException) + catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested) { - // Expected during silo shutdown. + // The loop observes owned shutdown; caller cancellation is reported after cleanup. } catch (Exception exception) { @@ -111,6 +111,8 @@ private async Task StopAsync(CancellationToken cancellationToken) _shutdownCts.Dispose(); } + + cancellationToken.ThrowIfCancellationRequested(); } private async Task RunAntiEntropyLoop(CancellationToken cancellationToken) diff --git a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs index 3552c03933f..835d188bd83 100644 --- a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs @@ -172,20 +172,14 @@ public async ValueTask ApplyValueAsync( if (snapshot.Version == currentSnapshot.Version) { - // Same-version snapshots can still advance IAmAlive state. + // Same-version snapshots can advance liveness or remove inactive entries. if (!snapshot.IsSuccessorTo(currentSnapshot)) { return DisseminationApplyResult.Duplicate; } - - await membershipManager.ProcessGossipSnapshot(snapshot, cancellationToken); - RememberSnapshot(membershipManager.CurrentSnapshot); - return DisseminationApplyResult.Applied; } - await membershipManager.ProcessGossipSnapshot(snapshot, cancellationToken); - RememberSnapshot(membershipManager.CurrentSnapshot); - return DisseminationApplyResult.Applied; + return await ApplySnapshot(currentSnapshot, snapshot, cancellationToken); } private DisseminationValue CreateSnapshotValue(MembershipTableSnapshot snapshot) => new( @@ -238,9 +232,20 @@ private async ValueTask ApplyDiff(MembershipTableSnaps return DisseminationApplyResult.Duplicate; } + return await ApplySnapshot(current, snapshot, cancellationToken); + } + + private async ValueTask ApplySnapshot( + MembershipTableSnapshot previous, + MembershipTableSnapshot snapshot, + CancellationToken cancellationToken) + { await membershipManager.ProcessGossipSnapshot(snapshot, cancellationToken); - RememberSnapshot(membershipManager.CurrentSnapshot); - return DisseminationApplyResult.Applied; + var current = membershipManager.CurrentSnapshot; + RememberSnapshot(current); + return MembershipSnapshotsEqual(previous, current) + ? DisseminationApplyResult.Duplicate + : DisseminationApplyResult.Applied; } private void RememberSnapshot(MembershipTableSnapshot snapshot) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs new file mode 100644 index 00000000000..3b46bfe52b2 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs @@ -0,0 +1,157 @@ +#nullable enable + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Orleans; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.MembershipService; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MembershipInventoryOnlyRepairConvergesWithoutHeartbeat(bool delta) + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40301); + var sourceAddress = CreateSilo(40302); + var live = CreateMembershipEntry( + CreateSilo(40303), SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(20)); + var retired = CreateMembershipEntry(CreateSilo(40304), SiloStatus.Dead, DateTime.UnixEpoch); + var previous = CreateMembershipSnapshot(2, live, retired); + var incoming = CreateMembershipSnapshot(2, live.WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))); + var expected = CreateMembershipSnapshot(2, live); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + using var receiverManager = CreateMembershipReviewManager(local, out _); + using var sourceManager = CreateMembershipReviewManager(sourceAddress, out _); + await ((IMembershipManager)receiverManager).ProcessGossipSnapshot(previous, cancellationToken); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(expected, cancellationToken); + var receiver = CreateMembershipNamespace((IMembershipManager)receiverManager, serializer); + var source = CreateMembershipNamespace((IMembershipManager)sourceManager, serializer); + var before = Assert.Single(receiver.Digests); + var update = delta + ? new MembershipTableSnapshotUpdate + { + Diff = new MembershipTableSnapshotDiff( + new MembershipVersion(1), incoming.Version, [incoming.Entries[live.SiloAddress]], [], includesAllEntries: true), + } + : new MembershipTableSnapshotUpdate { Snapshot = incoming }; + var value = new DisseminationValue(DisseminationKey.Default, delta ? 1 : 0, 2, serializer.SerializeToArray(update)); + + Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(value, cancellationToken)); + AssertMembershipState(expected, receiverManager.MembershipTableSnapshot); + Assert.NotEqual(before.Fingerprint, Assert.Single(receiver.Digests).Fingerprint); + Assert.Equal(before.Version, Assert.Single(receiver.Digests).Version); + Assert.Equal(Assert.Single(source.Digests), Assert.Single(receiver.Digests)); + Assert.Equal(DisseminationApplyResult.Duplicate, await receiver.ApplyValueAsync(value, cancellationToken)); + var staleInventory = new DisseminationValue( + DisseminationKey.Default, 0, 2, serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = previous })); + Assert.Equal(DisseminationApplyResult.Duplicate, await receiver.ApplyValueAsync(staleInventory, cancellationToken)); + AssertMembershipState(expected, receiverManager.MembershipTableSnapshot); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(10), incoming.Entries[live.SiloAddress].IAmAliveTime); + + var protocol = CreateProtocol(new FakeTransport(sourceAddress, local), [source]); + try + { + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = local, + Digests = new() { [source.Name] = [Assert.Single(receiver.Digests)] }, + }, cancellationToken); + Assert.Empty(GetAntiEntropyResponseValues(response)); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task MembershipInventoryCleanupRespectsManagerLocalEntryRetention() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40311); + var peer = CreateSilo(40312); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + using var manager = CreateMembershipReviewManager(local, out _); + var localEntry = manager.MembershipTableSnapshot.Entries[local].WithStatus(SiloStatus.Dead); + var previous = CreateMembershipSnapshot( + 2, localEntry, CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + await ((IMembershipManager)manager).ProcessGossipSnapshot(previous, cancellationToken); + var current = manager.MembershipTableSnapshot; + var incoming = CreateMembershipSnapshot(2, previous.Entries[peer]); + var ns = CreateMembershipNamespace((IMembershipManager)manager, serializer); + var value = new DisseminationValue( + DisseminationKey.Default, 0, 2, serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = incoming })); + + Assert.True(incoming.IsSuccessorTo(current)); + Assert.Equal(DisseminationApplyResult.Duplicate, await ns.ApplyValueAsync(value, cancellationToken)); + Assert.Same(current, manager.MembershipTableSnapshot); + Assert.Equal(SiloStatus.Dead, manager.MembershipTableSnapshot.Entries[local].Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DisseminationStopReportsCallerCancellationAfterCleanup(bool cancelCaller) + { + var local = CreateSilo(40321); + var transport = new FakeTransport(local, CreateSilo(40322)); + var options = new ReviewOptionsMonitor(new DisseminationOptions()); + var clock = new ReviewTimeProvider(); + var details = new FakeLocalSiloDetails(local); + var target = new DisseminationSystemTarget( + details, + transport.GrainFactory, + new DisseminationMembership(transport.MembershipManager, details, Options.Create(options.CurrentValue)), + options, + [new FakeNamespace(local)], + clock, + NullLogger.Instance, + NullLogger.Instance, + CreatePhase4SystemTargetShared(local)); + var lifecycle = Substitute.For(); + ILifecycleObserver? observer = null; + lifecycle.Subscribe(Arg.Any(), Arg.Any(), Arg.Any()).Returns(call => + { + observer = call.ArgAt(2); + return new ReviewSubscription(static () => { }); + }); + ((ILifecycleParticipant)target).Participate(lifecycle); + Assert.NotNull(observer); + await observer.OnStart(TestContext.Current.CancellationToken); + await clock.WaitForChange(Timeout.InfiniteTimeSpan, TestContext.Current.CancellationToken); + await clock.WaitForChange(Timeout.InfiniteTimeSpan, TestContext.Current.CancellationToken); + using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + if (cancelCaller) + { + caller.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + () => observer.OnStop(caller.Token).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(caller.Token, exception.CancellationToken); + } + else + { + await observer.OnStop(caller.Token).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + await options.Unsubscribed.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(0, options.SubscriptionCount); + Assert.Empty(transport.AntiEntropyRequests); + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs index a841c2d39d9..dd913cd95df 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs @@ -260,7 +260,7 @@ public async Task MembershipNamespaceDiffWireCompatibilityPreservesLegacyPartial } [Fact] - public async Task MembershipNamespaceLegacyInventoryDivergenceConvergesAfterHeartbeatWithFullRepair() + public async Task MembershipNamespaceLegacyInventoryDivergenceConvergesWithoutHeartbeatWithFullRepair() { var cancellationToken = TestContext.Current.CancellationToken; var retired = CreateSilo(39541); @@ -311,8 +311,16 @@ public async Task MembershipNamespaceLegacyInventoryDivergenceConvergesAfterHear }, cancellationToken); var full = Assert.Single(GetAntiEntropyResponseValues(response)).Value; Assert.Equal((0L, 2L), (full.FromVersion, full.ToVersion)); - // Inventory-only cleanup need not be published by the manager: retired entries are already dead. + Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(full, cancellationToken)); + AssertMembershipState(target, receiverManager.MembershipTableSnapshot); + Assert.Equal(Assert.Single(source.Digests), Assert.Single(receiver.Digests)); Assert.Equal(DisseminationApplyResult.Duplicate, await receiver.ApplyValueAsync(full, cancellationToken)); + var converged = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = transport.Peers[0], + Digests = new() { [source.Name] = [Assert.Single(receiver.Digests)] }, + }, cancellationToken); + Assert.Empty(GetAntiEntropyResponseValues(converged)); var heartbeatTarget = CreateMembershipSnapshot( 2, diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs index 2815eb6fda8..aabb25643e2 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs @@ -267,9 +267,11 @@ private sealed class ReviewOptionsMonitor(DisseminationOptions initial) : IOptio { private readonly object _lock = new(); private readonly List> _listeners = []; + private readonly TaskCompletionSource _unsubscribed = new(TaskCreationOptions.RunContinuationsAsynchronously); private DisseminationOptions _current = initial; public DisseminationOptions CurrentValue => Volatile.Read(ref _current); + public Task Unsubscribed => _unsubscribed.Task; public DisseminationOptions Get(string? name) => CurrentValue; public int SubscriptionCount { @@ -295,6 +297,8 @@ public IDisposable OnChange(Action listener) { _listeners.Remove(listener); } + + _unsubscribed.TrySetResult(); }); } From 109ed45c5c2486661b61c371df969a353262f3c8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 14 Sep 2026 18:38:37 -0700 Subject: [PATCH 13/50] test(dissemination): own measurement abort lifetime --- .../Tests/MeasurementTests.cs | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs b/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs index a1184d3036a..1660eeea029 100644 --- a/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs +++ b/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs @@ -26,6 +26,7 @@ public async Task ConnectionDrain_WaitsForInitializationAndCloseBeforeReopening( }); await entered.Task.WaitAsync(TestContext.Current.CancellationToken); observation.Partition(); + Assert.True(context.ConnectionClosed.IsCancellationRequested); var closeCalls = 0; var drain = observation.DrainConnections( static _ => true, @@ -93,6 +94,8 @@ public async Task ConnectionDrain_CapturesUnidentifiedPeersAndPreservesHealthyCo try { Assert.Equal(new[] { "unknown" }, closed); + Assert.True(unknown.ConnectionClosed.IsCancellationRequested); + Assert.False(healthy.ConnectionClosed.IsCancellationRequested); Assert.False(healthyMiddleware.IsCompleted); await cancellation.CancelAsync(); await Assert.ThrowsAnyAsync(() => drain); @@ -292,7 +295,24 @@ private sealed record TestPipe(PipeReader Input, PipeWriter Output) : IDuplexPip private static DefaultConnectionContext CreateConnectionContext(string id) { var pipe = new Pipe(); - return new(id) { Transport = new TestPipe(pipe.Reader, pipe.Writer) }; + return new TestConnectionContext(id) { Transport = new TestPipe(pipe.Reader, pipe.Writer) }; + } + + // DefaultConnectionContext queues Cancel independently of Dispose. These tests own abort synchronously; + // separate barriers continue to control middleware and transport completion. + private sealed class TestConnectionContext : DefaultConnectionContext + { + private readonly CancellationTokenSource _closed = new(); + + public TestConnectionContext(string id) : base(id) => ConnectionClosed = _closed.Token; + + public override void Abort(ConnectionAbortedException abortReason) => _closed.Cancel(); + + public override ValueTask DisposeAsync() + { + _closed.Dispose(); + return base.DisposeAsync(); + } } private readonly struct ReadonlyStatistics(float rawCpuUsagePercentage) From b3cfc156ba0407fa8b09f35aa4874995afc032ef Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 14 Sep 2026 18:52:49 -0700 Subject: [PATCH 14/50] test(dissemination): await deadline requeue before shutdown --- .../Dissemination/DisseminationProtocolTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index cdd5dfcfbb1..257e1acaea3 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -3173,9 +3173,12 @@ public async Task BroadcastTransportCancelsWhenRelativeLifetimeExpires() TestContext.Current.CancellationToken)); timeProvider.Advance(ns.Options.MaxCoalescingDelay); await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var flush = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); timeProvider.Advance(ns.Options.StaleItemTtl); await cancellationObserved.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + // Complete the failed attempt's requeue before shutdown wakes the disabled namespace. + await flush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); ns.Options.Enabled = false; await protocol.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); From 64ed12bca56a5c2dc4a6b442c9fc90e60c763f0c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 14 Sep 2026 19:25:22 -0700 Subject: [PATCH 15/50] fix(dissemination): enforce membership and completion invariants --- .github/scripts/dissemination-evidence.ps1 | 2 +- .github/workflows/dissemination-evidence.yml | 1 + .../implementation/runtime-dissemination.md | 2 +- .../Dissemination/DisseminationProtocol.cs | 10 +-- .../Tests/MeasurementTests.cs | 22 +++++++ .../Tests/ProcessCluster.cs | 10 ++- ...sseminationProtocolTests.FollowupReview.cs | 61 +++++++++++++++++++ 7 files changed, 100 insertions(+), 8 deletions(-) diff --git a/.github/scripts/dissemination-evidence.ps1 b/.github/scripts/dissemination-evidence.ps1 index b311c2e8cc9..c397ab44cf1 100644 --- a/.github/scripts/dissemination-evidence.ps1 +++ b/.github/scripts/dissemination-evidence.ps1 @@ -164,7 +164,7 @@ try { if ($Mode -eq 'Gate') { Invoke-DotNet @( $runner, '--filter-class', 'Orleans.Dissemination.IntegrationHarness.MeasurementTests', - '--minimum-expected-tests', '9', '--report-trx', + '--minimum-expected-tests', '10', '--report-trx', '--results-directory', (Join-Path $results 'instrument-checks') ) } diff --git a/.github/workflows/dissemination-evidence.yml b/.github/workflows/dissemination-evidence.yml index 7a445b8b5e3..4a51acc027d 100644 --- a/.github/workflows/dissemination-evidence.yml +++ b/.github/workflows/dissemination-evidence.yml @@ -18,6 +18,7 @@ on: - 'src/Orleans.Runtime/OrleansContracts.txt' - 'src/Orleans.Core/OrleansContracts.txt' - 'src/Orleans.Core/Runtime/Constants.cs' + - 'src/Orleans.Core/Runtime/MembershipTableSnapshot.cs' - 'src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs' - 'src/Orleans.Core/SystemTargetInterfaces/IDeploymentLoadPublisher.cs' - 'src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs' diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index a27c962bfd9..c728b5725f4 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -49,7 +49,7 @@ A `DisseminationValue` names one key and a `[FromVersion, ToVersion]` transition The namespace reports whether a repair is current, produced, unavailable, or unable to fit the supplied item and byte budgets. Produced repairs can be complete or a valid prefix. Prefix acknowledgments establish the receiver's new baseline and let the next batch continue the chain. -Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. It retains 32 snapshots, prefers a smaller diff when the peer baseline is present, and falls back to a full snapshot when history or capacity makes the diff unsuitable. Current diffs identify their entries as the complete target inventory, so table cleanup is reflected even when a receiver retained a different same-version baseline. Older partial-diff payloads retain their incremental semantics. Same-version snapshots can advance liveness or apply an inventory-only cleanup. The cleanup-only path retains every Active entry and each remaining status, so inactive-row removal converges while heartbeats are unchanged. The membership manager merges maximum per-entry liveness and retains its local-death handling; dissemination reports whether that manager actually published changed state. +Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. It retains 32 snapshots, prefers a smaller diff when the peer baseline is present, and falls back to a full snapshot when history or capacity makes the diff unsuitable. Current diffs identify their entries as the complete target inventory, so table cleanup is reflected even when a receiver retained a different same-version baseline. Older partial-diff payloads retain their incremental semantics. Same-version snapshots can advance liveness or remove inactive entries while preserving every Active entry and each retained status. This allows cleanup to converge while heartbeats are unchanged, and an advancing heartbeat preserves those same membership guards. The membership manager merges maximum per-entry liveness and retains its local-death handling; dissemination reports whether that manager actually published changed state. Deployment-load versions are sample timestamps. Each repair is a full latest value. Application runs on the deployment-load publisher's scheduler and accepts samples only for Active silo generations in the membership oracle. Terminal membership transitions remove placement statistics, and the oracle's monotonic membership state rejects later samples for departed generations. diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index 066fb5e6085..296184aea6a 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -1086,6 +1086,7 @@ private async ValueTask ApplyReceivedValueCore( cancellationToken, lifetimeCancellation.Token); DisseminationApplyResult result; + Task? applicationTask = null; try { var application = disseminationNamespace.ApplyValueAsync(item.Value, applicationCancellation.Token); @@ -1097,19 +1098,20 @@ private async ValueTask ApplyReceivedValueCore( { // Only this local wait is bounded. Namespace owners must observe cancellation before mutating // queued state; arbitrary implementations which ignore the token cannot be forcibly stopped. - var applicationTask = application.AsTask(); + applicationTask = application.AsTask(); applicationTask.Ignore(); result = await applicationTask.WaitAsync(applicationCancellation.Token); } - - cancellationToken.ThrowIfCancellationRequested(); } catch (OperationCanceledException) when ( lifetimeCancellation.IsCancellationRequested && !cancellationToken.IsCancellationRequested) { - result = DisseminationApplyResult.Obsolete; + result = applicationTask is { IsCompletedSuccessfully: true } + ? applicationTask.Result + : DisseminationApplyResult.Obsolete; } + cancellationToken.ThrowIfCancellationRequested(); EmitApplyResult(namespaceName, item, sender, result); if (result is DisseminationApplyResult.Applied) { diff --git a/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs b/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs index 1660eeea029..8be6da8c4b3 100644 --- a/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs +++ b/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs @@ -11,6 +11,28 @@ namespace Orleans.Dissemination.IntegrationHarness; public sealed class MeasurementTests { + [Fact] + public void LoadStateComparisonRequiresExactInventoryAndValues() + { + var expected = new Dictionary + { + ["silo@2"] = "current", + ["peer@1"] = "peer-value", + }; + var actual = new Dictionary(expected); + Assert.True(ProcessCluster.MatchesExpectedLoad(actual, expected)); + + actual["silo@1"] = "retired"; + Assert.False(ProcessCluster.MatchesExpectedLoad(actual, expected)); + actual.Remove("silo@2"); + Assert.False(ProcessCluster.MatchesExpectedLoad(actual, expected)); + actual.Remove("silo@1"); + actual["silo@2"] = "older-value"; + Assert.False(ProcessCluster.MatchesExpectedLoad(actual, expected)); + actual["silo@2"] = "current"; + Assert.True(ProcessCluster.MatchesExpectedLoad(actual, expected)); + } + [Fact] public async Task ConnectionDrain_WaitsForInitializationAndCloseBeforeReopening() { diff --git a/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs b/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs index bcca9a1d1f9..8f689b7ad2d 100644 --- a/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs +++ b/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs @@ -107,12 +107,18 @@ public async Task PublishAndConverge() await Eventually("exact production load state at every active process", async () => { var snapshots = await Task.WhenAll(Active.Select(node => node.Send("snapshot"))); - return snapshots.All(snapshot => expected.All(pair => - snapshot.Load.TryGetValue(pair.Key, out var actual) && StringComparer.Ordinal.Equals(pair.Value, actual))); + return snapshots.All(snapshot => MatchesExpectedLoad(snapshot.Load, expected)); }, expected: expected); return clock.Elapsed.TotalMilliseconds; } + internal static bool MatchesExpectedLoad( + IReadOnlyDictionary actual, + IReadOnlyDictionary expected) => + actual.Count == expected.Count + && expected.All(pair => actual.TryGetValue(pair.Key, out var value) + && StringComparer.Ordinal.Equals(pair.Value, value)); + public async Task HealPartition(SiloProcess partitioned) { using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs index 87d774ba3dd..bc92807fc14 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs @@ -14,6 +14,67 @@ namespace UnitTests.Dissemination; public partial class DisseminationProtocolTests { + [Fact] + public async Task CompletedAsyncApplicationWinsAConcurrentLocalWaitTimeout() + { + var local = CreateSilo(40401); + var sender = CreateSilo(40402); + var child = CreateSilo(40403); + var clock = new FakeTimeProvider(); + var ns = new ProtocolReviewNamespace(local, "async-terminal-result-review"); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + var value = ns.Inner.CreateValue("value", 1); + var application = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + ns.ApplyHandler = (_, _) => new(application.Task); + var transport = new FakeTransport(local, sender, child); + var protocol = CreateProtocol(transport, [ns], options => options.Overlay.FanOutFactor = static _ => 2, clock); + var context = new MembershipReviewContinuationContext(); + var previousContext = SynchronizationContext.Current; + Task receive; + SynchronizationContext.SetSynchronizationContext(context); + try + { + receive = protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = sender, + Values = CreateValueGroups(ns.Name, CreateDisseminationValue(sender, value)), + }, TestContext.Current.CancellationToken); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previousContext); + } + + try + { + clock.Advance(ns.Options.StaleItemTtl); + var timedOutWait = await context.TakeContinuation(TestContext.Current.CancellationToken); + Assert.False(receive.IsCompleted); + ns.Inner.PublishValue(value); + application.SetResult(DisseminationApplyResult.Applied); + timedOutWait.Callback(timedOutWait.State); + for (var index = 0; index < 4 && !receive.IsCompleted; index++) + { + var continuation = await context.TakeContinuation(TestContext.Current.CancellationToken); + continuation.Callback(continuation.State); + } + + var response = await receive.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, Assert.Single(response.Acknowledgments[ns.Name]).Version); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(child, Assert.Single(transport.BroadcastBatches).Peer); + Assert.Equal(1, ns.GetVersion("value")); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.NotEmpty(transport.AntiEntropyRequests); + Assert.All(transport.AntiEntropyRequests, request => Assert.False(request.Request.Digests.ContainsKey(ns.Name))); + } + finally + { + application.TrySetResult(DisseminationApplyResult.Rejected); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + [Fact] public async Task CompletedApplicationRemainsAppliedWhenItsLocalDeadlineRacesCompletion() { From ffd2b82bc84c899dddadbad628a6e37b1d91a9ec Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 14 Sep 2026 19:49:23 -0700 Subject: [PATCH 16/50] test(dissemination): isolate history-miss repair provenance --- .../Tests/VersionSkewTests.cs | 175 ++++++++++-------- .../methodology.json | 4 +- 2 files changed, 101 insertions(+), 78 deletions(-) diff --git a/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs index d28f4d348cd..96fdeeb44c2 100644 --- a/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs +++ b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs @@ -153,89 +153,112 @@ await cluster.Eventually("identical membership before isolation", async () => var snapshots = await Task.WhenAll(cluster.Active.Select(node => node.Send("refresh"))); return snapshots.All(snapshot => SameMembership(snapshots[0], snapshot)); }); - foreach (var node in cluster.Active) + try { - var isolated = await node.Send("isolate-membership", value: true); - Assert.True(isolated.LegacyGossipSuppressed); - Assert.True(isolated.MembershipReadsFrozen); - } + foreach (var node in cluster.Active) + { + var isolated = await node.Send("isolate-membership", value: true); + Assert.True(isolated.LegacyGossipSuppressed); + Assert.True(isolated.MembershipReadsFrozen); + } - Assert.False((await origin.Send("probe-tree", peer: receiver.Last.Address)).TreeProbeRejected); - foreach (var node in cluster.Active) - { - var quiesced = await node.Send("block-tree", value: true); - Assert.NotNull(quiesced.TreeGate); - Assert.True(quiesced.TreeGate.Blocked); - Assert.Equal(0, quiesced.TreeGate.InFlight); - } + Assert.False((await origin.Send("probe-tree", peer: receiver.Last.Address)).TreeProbeRejected); + foreach (var node in cluster.Active) + { + var quiesced = await node.Send("block-tree", value: true); + Assert.NotNull(quiesced.TreeGate); + Assert.True(quiesced.TreeGate.Blocked); + Assert.Equal(0, quiesced.TreeGate.InFlight); + } - // Verify the native PushBroadcast RPC is intercepted, not just that a local flag was set. - Assert.True((await origin.Send("probe-tree", peer: receiver.Last.Address)).TreeProbeRejected); - await Task.WhenAll(cluster.Active.Select(node => node.Send("snapshot"))); - var before = cluster.Active.ToDictionary(node => node.Last.Identity.ProcessId, node => node.Last); - var oldVersion = receiver.Last.MembershipVersion; - await receiver.Send("partition", value: true); - var source = await origin.Send("membership-history", count: 40, version: oldVersion); - Assert.Equal(oldVersion + 40, source.MembershipVersion); - Assert.Equal(0, source.RepairFromVersion); // The actual production namespace evicted the old baseline. - var blocked = await receiver.Send("snapshot"); - Assert.Equal(oldVersion, blocked.MembershipVersion); - Assert.False(SameMembership(source, blocked)); - await receiver.Send("partition", value: false); + // Verify the native PushBroadcast RPC is intercepted, not just that a local flag was set. + Assert.True((await origin.Send("probe-tree", peer: receiver.Last.Address)).TreeProbeRejected); + await Task.WhenAll(cluster.Active.Select(node => node.Send("snapshot"))); + var before = cluster.Active.ToDictionary(node => node.Last.Identity.ProcessId, node => node.Last); + var oldVersion = receiver.Last.MembershipVersion; + await receiver.Send("partition", value: true); + await relay.Send("partition", value: true); + var source = await origin.Send("membership-history", count: 40, version: oldVersion); + Assert.Equal(oldVersion + 40, source.MembershipVersion); + Assert.Equal(0, source.RepairFromVersion); // The actual production namespace evicted the old baseline. + var blocked = await receiver.Send("snapshot"); + Assert.Equal(oldVersion, blocked.MembershipVersion); + Assert.Equal(oldVersion, (await relay.Send("snapshot")).MembershipVersion); + Assert.False(SameMembership(source, blocked)); + await receiver.Send("partition", value: false); - await cluster.Eventually("anti-entropy-only full repair after a real transport partition and retained-history miss", async () => - { - await receiver.Send("snapshot"); - await relay.Send("snapshot"); - return SameMembership(source, receiver.Last) && SameMembership(source, relay.Last) - && receiver.Last.Applies.Any(evidence => evidence.Namespace == "membership" - && evidence.FromVersion == 0 && evidence.ToVersion == source.MembershipVersion - && evidence.Result == "Applied" && evidence.Peer is not null); - }); - await AssertNoTreeAdmissions(cluster, before); - Assert.True(receiver.Last.MembershipReadsFrozen); - Assert.True(receiver.Last.LegacyGossipSuppressed); - await cluster.Save("history-recovery.json", new { Before = before, After = cluster.Active.Select(node => node.Last) }); + // Only the origin can repair this receiver until its full history-miss repair is observed. + // A relay which first learned the target could otherwise serve a valid retained-baseline delta. + await cluster.Eventually("anti-entropy-only full repair after a real transport partition and retained-history miss", async () => + { + await receiver.Send("snapshot"); + return SameMembership(source, receiver.Last) + && receiver.Last.Applies.Any(evidence => evidence.Namespace == "membership" + && evidence.FromVersion == 0 && evidence.ToVersion == source.MembershipVersion + && evidence.Result == "Applied" && evidence.Peer is not null); + }); + await relay.Send("partition", value: false); + await cluster.Eventually("relay convergence after the receiver's full repair", async () => + { + await receiver.Send("snapshot"); + await relay.Send("snapshot"); + return SameMembership(source, receiver.Last) && SameMembership(source, relay.Last); + }); + await AssertNoTreeAdmissions(cluster, before); + Assert.True(receiver.Last.MembershipReadsFrozen); + Assert.True(receiver.Last.LegacyGossipSuppressed); + await cluster.Save("history-recovery.json", new { Before = before, After = cluster.Active.Select(node => node.Last) }); - var priorHeartbeatApplications = receiver.Last.Applies.Count(evidence => - evidence.Namespace == "membership" && evidence.Result == "Applied" && evidence.ToVersion == source.MembershipVersion); - var heartbeat = await origin.Send("membership-heartbeat"); - Assert.Equal(source.MembershipVersion, heartbeat.MembershipVersion); - Assert.NotNull(heartbeat.HeartbeatTicks); - Assert.False(SameMembership(source, heartbeat)); - await cluster.Eventually("same-version heartbeat fingerprint repair without Publish, gossip, or table reads", async () => - { - await receiver.Send("snapshot"); - await relay.Send("snapshot"); - return SameMembership(heartbeat, receiver.Last) && SameMembership(heartbeat, relay.Last) - && receiver.Last.Applies.Count(evidence => evidence.Namespace == "membership" - && evidence.FromVersion == 0 && evidence.ToVersion == heartbeat.MembershipVersion - && evidence.Result == "Applied") > priorHeartbeatApplications; - }); - await AssertNoTreeAdmissions(cluster, before); - await cluster.Save("same-version-heartbeat-recovery.json", cluster.Active.Select(node => node.Last)); + var priorHeartbeatApplications = receiver.Last.Applies.Count(evidence => + evidence.Namespace == "membership" && evidence.Result == "Applied" && evidence.ToVersion == source.MembershipVersion); + var heartbeat = await origin.Send("membership-heartbeat"); + Assert.Equal(source.MembershipVersion, heartbeat.MembershipVersion); + Assert.NotNull(heartbeat.HeartbeatTicks); + Assert.False(SameMembership(source, heartbeat)); + await cluster.Eventually("same-version heartbeat fingerprint repair without Publish, gossip, or table reads", async () => + { + await receiver.Send("snapshot"); + await relay.Send("snapshot"); + return SameMembership(heartbeat, receiver.Last) && SameMembership(heartbeat, relay.Last) + && receiver.Last.Applies.Count(evidence => evidence.Namespace == "membership" + && evidence.FromVersion == 0 && evidence.ToVersion == heartbeat.MembershipVersion + && evidence.Result == "Applied") > priorHeartbeatApplications; + }); + await AssertNoTreeAdmissions(cluster, before); + await cluster.Save("same-version-heartbeat-recovery.json", cluster.Active.Select(node => node.Last)); - var expectedLoad = receiver.Last.Load[origin.Last.Address]; - var expectedVersion = receiver.Last.LoadVersions[origin.Last.Address]; - var priorLoadApplications = receiver.Last.Applies.Count(evidence => - evidence.Namespace == "load" && evidence.Result == "Applied" && evidence.ToVersion == expectedVersion); - await receiver.Send("forget-load", peer: origin.Last.Address); - Assert.DoesNotContain(origin.Last.Address, receiver.Last.Load.Keys); - await cluster.Eventually("load repair reaches exact state with no new publication or direct refresh", async () => - { - await receiver.Send("snapshot"); - return receiver.Last.Load.TryGetValue(origin.Last.Address, out var actual) - && actual == expectedLoad - && receiver.Last.Applies.Count(evidence => evidence.Namespace == "load" - && evidence.FromVersion == 0 && evidence.ToVersion == expectedVersion - && evidence.Result == "Applied") > priorLoadApplications; - }); - await AssertNoTreeAdmissions(cluster, before); - await cluster.Save("load-full-repair.json", receiver.Last); - foreach (var node in cluster.Active) + var expectedLoad = receiver.Last.Load[origin.Last.Address]; + var expectedVersion = receiver.Last.LoadVersions[origin.Last.Address]; + var priorLoadApplications = receiver.Last.Applies.Count(evidence => + evidence.Namespace == "load" && evidence.Result == "Applied" && evidence.ToVersion == expectedVersion); + await receiver.Send("forget-load", peer: origin.Last.Address); + Assert.DoesNotContain(origin.Last.Address, receiver.Last.Load.Keys); + await cluster.Eventually("load repair reaches exact state with no new publication or direct refresh", async () => + { + await receiver.Send("snapshot"); + return receiver.Last.Load.TryGetValue(origin.Last.Address, out var actual) + && actual == expectedLoad + && receiver.Last.Applies.Count(evidence => evidence.Namespace == "load" + && evidence.FromVersion == 0 && evidence.ToVersion == expectedVersion + && evidence.Result == "Applied") > priorLoadApplications; + }); + await AssertNoTreeAdmissions(cluster, before); + await cluster.Save("load-full-repair.json", receiver.Last); + } + finally { - await node.Send("isolate-membership", value: false); - await node.Send("block-tree", value: false); + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + foreach (var node in cluster.Active) + { + await node.Send(cleanup.Token, "isolate-membership", value: false); + await node.Send(cleanup.Token, "block-tree", value: true); + if (node.Last.Partitioned) + { + await node.Send(cleanup.Token, "partition", value: false); + } + + await node.Send(cleanup.Token, "block-tree", value: false); + } } } diff --git a/test/Dissemination.IntegrationHarness/methodology.json b/test/Dissemination.IntegrationHarness/methodology.json index 66f45aa52f7..0e134dfe94c 100644 --- a/test/Dissemination.IntegrationHarness/methodology.json +++ b/test/Dissemination.IntegrationHarness/methodology.json @@ -66,7 +66,7 @@ }, { "test": "PartitionHistoryAndHeartbeat_RequireDisseminationProvenance", - "assertions": "Freeze central-table reads and suppress legacy membership gossip on all nodes. Close the test-only native PushBroadcast incoming-call gate on every node and await all previously admitted PushBroadcast invocations before creating the history gap. Probe the actual RPC to verify rejection. Keep ExchangeAntiEntropy enabled. Isolate one real process while the production membership namespace sees 40 actual table versions. Its 32-entry history must miss the receiver baseline and produce FromVersion=0. Heal transport and require exact full membership state plus Applied provenance while every tree gate stays closed with unchanged admitted-call counts. Keep those gates closed for same-version heartbeat and missing-load full repair. Disabling anti-entropy cannot be rescued by pending/retried/rematerialized tree sends.", + "assertions": "Freeze central-table reads and suppress legacy membership gossip on all nodes. Close the test-only native PushBroadcast incoming-call gate on every node and await all previously admitted PushBroadcast invocations before creating the history gap. Probe the actual RPC to verify rejection. Keep ExchangeAntiEntropy enabled. Isolate both downstream processes while the origin sees 40 actual table versions, evicting the receiver baseline from its 32-entry history. Heal the first receiver while the relay stays isolated, so its repair must come from the history-missing origin and produce FromVersion=0. After observing exact state and full-repair provenance, heal the relay and require every node to converge. Keep tree gates closed with unchanged admitted-call counts for the history, heartbeat and missing-load assertions. Restore table reads, gossip and network/tree gates in bounded cleanup even when an assertion fails.", "nonClaims": "TTL expiry is not queue expiry and is not used as a tree-drain proof. FromVersion=0 alone does not prove anti-entropy. Legacy gossip, central-table reads and all newly admitted PushBroadcast calls are excluded throughout these assertions." }, { @@ -116,7 +116,7 @@ "manualDefaults": { "mode": "Scale", "sizes": [4, 8, 16], "iterations": 10, "pairedRepetitions": 1 }, "controls": { "sizes": "1..6 comma-separated sizes, each 3..32", "iterations": "3..200", "pairedRepetitions": "1..5" }, "scenarios": "Stable; graceful stop/restart at the same endpoint every other round; real connection partition and heal with old pooled connections retired at both endpoints before reopening and confirming transport readiness via bidirectional acknowledged control RPCs. Teardown, readiness probes and retries share a 60-second deadline and remain inside recovery latency and total cost. Each path then offers the same single publication round. Gate artifacts separately cover silent repair, retained-history misses and same-version heartbeat recovery.", - "cadence": "Pause automatic deployment-load timers after startup. Invoke the actual production PublishStatistics method once per active node per offered round, then compare state using cached JsonSerializerOptions with IncludeFields=true, including every public readonly EnvironmentStatistics field. Do not replace it with a formula or synthetic protocol implementation.", + "cadence": "Pause automatic deployment-load timers after startup. Invoke the actual production PublishStatistics method once per active node per offered round, then require exact dictionary inventory and values using cached JsonSerializerOptions with IncludeFields=true, including every public readonly EnvironmentStatistics field. Extra departed or restarted generations fail the same comparison as missing or mismatched values.", "window": "Two warmup publication rounds and complete enabled-peer capability discovery before measuring. Replacements also finish capability discovery before offered rounds; their bootstrap/fallback/discovery costs remain inside the churn window. Two-second OS-counter sampling margins on all paths. Full-GC retained-memory probes are after the CPU/allocation window.", "metrics": [ "All transmitted request AND one-way RPC messages; responses reported in the separate total-message count", From 09ebf2ea26e53b1728fab18a5d9400d0b8ae2a62 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 14 Sep 2026 20:28:58 -0700 Subject: [PATCH 17/50] fix(dissemination): refresh topology after same-version cleanup --- .../implementation/runtime-dissemination.md | 2 + .../Dissemination/DisseminationMembership.cs | 48 +++++++-- .../DisseminationMembershipSnapshotTests.cs | 99 +++++++++++++++++++ 3 files changed, 139 insertions(+), 10 deletions(-) diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index c728b5725f4..62cb46314db 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -68,6 +68,8 @@ Namespaces select a membership scope before topology construction: Fanout is derived from the target hop count and bounded by the configured minimum and maximum, or selected by the code-configured callback. A membership or fanout change creates a new topology from the next snapshot; acknowledged ledgers and anti-entropy repair convergence across the transition. +The projection cache tracks its source snapshot as well as its table version. Same-version cleanup of Joining, ShuttingDown, or Stopping participants refreshes routing and peer selection. Heartbeat-only updates and cleanup of non-participants reuse the existing topology, while repaired projections retain rotation progress. The successor guard preserves the newer inventory when concurrent reads arrive out of order. + ## Configuration and defaults The subsystem and each namespace default to disabled. Enable together with on the selected integration to activate broadcast and repair. Defaults bound concurrency, memory retention, payload size, and repair work: diff --git a/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs b/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs index 332caf0becf..de77e76d163 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs @@ -11,30 +11,33 @@ internal sealed class DisseminationMembership( IOptions options) { private readonly object _membershipLock = new(); - private DisseminationMembershipSnapshots? _currentSnapshots; + private CachedMembership? _current; public DisseminationMembershipSnapshots CurrentSnapshots { get { var membershipSnapshot = membershipManager.CurrentSnapshot; - var current = Volatile.Read(ref _currentSnapshots); - if (current is not null && current.MembershipVersion == membershipSnapshot.Version) + var current = Volatile.Read(ref _current); + if (current is not null && ReferenceEquals(current.Source, membershipSnapshot)) { - return current; + return current.Snapshots; } lock (_membershipLock) { - current = Volatile.Read(ref _currentSnapshots); - if (current is not null && current.MembershipVersion >= membershipSnapshot.Version) + current = Volatile.Read(ref _current); + if (current is not null && !membershipSnapshot.IsSuccessorTo(current.Source)) { - return current; + return current.Snapshots; } - current = ComputeMembership(membershipSnapshot, localSiloDetails.SiloAddress, options.Value.Overlay, current); - Volatile.Write(ref _currentSnapshots, current); - return current; + // Same-version heartbeats and removal of non-participants retain the existing topology. + var snapshots = current is { } previous && HasSameTopology(previous, membershipSnapshot) + ? previous.Snapshots + : ComputeMembership(membershipSnapshot, localSiloDetails.SiloAddress, options.Value.Overlay, current?.Snapshots); + Volatile.Write(ref _current, new(membershipSnapshot, snapshots)); + return snapshots; } } } @@ -63,6 +66,29 @@ public Task RefreshMembership(CancellationToken cancellationToken) => return snapshots.GetSnapshot(scope).ContainsMember(member) ? snapshots : null; } + private static bool HasSameTopology(CachedMembership current, MembershipTableSnapshot source) + { + if (current.Source.Version != source.Version) + { + return false; + } + + if (current.Source.Entries.Count == source.Entries.Count) + { + return true; + } + + foreach (var member in current.Snapshots.AllMembers.Members) + { + if (!source.Entries.ContainsKey(member)) + { + return false; + } + } + + return true; + } + private static DisseminationMembershipSnapshots ComputeMembership( MembershipTableSnapshot snapshot, SiloAddress localSilo, @@ -113,6 +139,8 @@ private static bool IsDisseminationMember(SiloStatus status) => SiloStatus.Stopping => 3, _ => 4, }; + + private sealed record CachedMembership(MembershipTableSnapshot Source, DisseminationMembershipSnapshots Snapshots); } internal sealed class DisseminationMembershipSnapshots( diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs index a002a986794..6db0da8f703 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs @@ -416,6 +416,105 @@ public async Task StaleConcurrentRefreshDoesNotRegressNewerProjection() Assert.Equal(4, manager.SnapshotReadCount); } + [Theory] + [InlineData(SiloStatus.Joining)] + [InlineData(SiloStatus.ShuttingDown)] + [InlineData(SiloStatus.Stopping)] + public void SameVersionCleanupRefreshesEligibleTopologyAndPreservesRotation(SiloStatus removedStatus) + { + var members = CreateSilos(4); + var source = new MembershipTableSnapshot(new MembershipVersion(42), + CreateSourceSnapshot(42, members[0], members[1], members[2]).Entries.Add( + members[3], CreateScopeMembershipEntry(members[3], removedStatus, 3))); + var manager = new MutableMembershipManager(source, TestContext.Current.CancellationToken); + var membership = new DisseminationMembership(manager, new ScopeLocalSiloDetails(members[0]), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + var before = membership.CurrentSnapshots; + Assert.Equal(new[] { members[1] }, before.AllMembers.SelectAntiEntropyPeers(1)); + + manager.SetSnapshot(new(source.Version, source.Entries.Remove(members[3]))); + var after = membership.CurrentSnapshots; + + Assert.NotSame(before, after); + Assert.Equal(source.Version, after.MembershipVersion); + Assert.Equal(members[..3], after.AllMembers.Members); + Assert.Equal(members[..3], after.ActiveMembers.Members); + Assert.DoesNotContain(members[3], after.AllMembers.OriginatorTreeTargets); + Assert.DoesNotContain(members[3], after.AllMembers.ForwardingTreeTargets); + Assert.Equal(new[] { members[2] }, after.AllMembers.SelectAntiEntropyPeers(1)); + Assert.Same(after, membership.CurrentSnapshots); + } + + [Theory] + [InlineData(SiloStatus.Created)] + [InlineData(SiloStatus.Dead)] + public void SameVersionCleanupOfNonParticipantsReusesTopology(SiloStatus removedStatus) + { + var members = CreateSilos(3); + var source = new MembershipTableSnapshot(new MembershipVersion(42), + CreateSourceSnapshot(42, members[0], members[1]).Entries.Add( + members[2], CreateScopeMembershipEntry(members[2], removedStatus, 2))); + var manager = new MutableMembershipManager(source, TestContext.Current.CancellationToken); + var membership = new DisseminationMembership(manager, new ScopeLocalSiloDetails(members[0]), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + var before = membership.CurrentSnapshots; + + manager.SetSnapshot(new(source.Version, source.Entries.Remove(members[2]))); + + Assert.Same(before, membership.CurrentSnapshots); + Assert.Equal(members[..2], membership.CurrentSnapshot.Members); + } + + [Fact] + public void SameVersionHeartbeatReusesTopologyAndRotation() + { + var members = CreateSilos(3); + var source = CreateSourceSnapshot(42, members[0], members[1], members[2]); + var manager = new MutableMembershipManager(source, TestContext.Current.CancellationToken); + var membership = new DisseminationMembership(manager, new ScopeLocalSiloDetails(members[0]), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + var before = membership.CurrentSnapshots; + Assert.Equal(new[] { members[1] }, before.AllMembers.SelectAntiEntropyPeers(1)); + manager.SetSnapshot(new(source.Version, source.Entries.SetItem(members[1], + source.Entries[members[1]].WithIAmAliveTime(DateTime.UnixEpoch.AddMinutes(1))))); + + var after = membership.CurrentSnapshots; + + Assert.Same(before, after); + Assert.Equal(new[] { members[2] }, after.AllMembers.SelectAntiEntropyPeers(1)); + } + + [Fact] + public async Task StaleConcurrentReadCannotRestoreSameVersionRemovedParticipant() + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(3); + var source = new MembershipTableSnapshot(new MembershipVersion(42), + CreateSourceSnapshot(42, members[0], members[1]).Entries.Add( + members[2], CreateScopeMembershipEntry(members[2], SiloStatus.Joining, 2))); + var manager = new MutableMembershipManager(source, cancellationToken) { BlockSecondSnapshotRead = true }; + var membership = new DisseminationMembership(manager, new ScopeLocalSiloDetails(members[0]), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + Assert.Contains(members[2], membership.CurrentSnapshot.Members); + var stale = Task.Run(() => membership.CurrentSnapshots); + DisseminationMembershipSnapshots updated; + try + { + await manager.SecondReadCaptured.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + manager.SetSnapshot(new(source.Version, source.Entries.Remove(members[2]))); + updated = membership.CurrentSnapshots; + Assert.DoesNotContain(members[2], updated.AllMembers.Members); + } + finally + { + manager.ReleaseSecondRead(); + } + + Assert.Same(updated, await stale.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + Assert.Same(updated, membership.CurrentSnapshots); + Assert.Equal(source.Version, updated.MembershipVersion); + } + private static ( DisseminationMembershipSnapshots Snapshots, CountingMembershipManager Manager, From 606c4b7741bfed40c413b29c6e321fe0ece2d2a6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 12:41:23 -0700 Subject: [PATCH 18/50] test(dissemination): temporarily enable defaults for CI exposure --- .github/workflows/dissemination-evidence.yml | 4 ++-- .../implementation/runtime-dissemination.md | 4 ++-- .../Options/DisseminationOptions.cs | 8 ++++---- .../Silo/NewRuntime.cs | 13 +++++++------ .../Tests/ScalingTests.cs | 11 ++++++----- .../Tests/VersionSkewTests.cs | 2 +- .../methodology.json | 16 ++++++++-------- .../Runtime/DeploymentLoadPublisherTests.cs | 19 ++++++++++--------- .../DisseminationClusterTests.cs | 11 +---------- ...isseminationProtocolTests.FinalComments.cs | 2 +- .../DisseminationProtocolTests.QueueReview.cs | 4 ++-- .../DisseminationProtocolTests.cs | 4 ++++ 12 files changed, 48 insertions(+), 50 deletions(-) diff --git a/.github/workflows/dissemination-evidence.yml b/.github/workflows/dissemination-evidence.yml index 4a51acc027d..a81d8ab534f 100644 --- a/.github/workflows/dissemination-evidence.yml +++ b/.github/workflows/dissemination-evidence.yml @@ -96,7 +96,7 @@ jobs: path: Artifacts/DisseminationEvidence/bin bounded-cost: - name: Bounded original, default-off and enabled costs + name: Bounded original, disabled and enabled costs needs: version-skew runs-on: ubuntu-latest timeout-minutes: 25 @@ -111,7 +111,7 @@ jobs: with: name: dissemination-binaries path: Artifacts/DisseminationEvidence/bin - - name: Measure bounded original, default-off and supported-enabled paths + - name: Measure bounded original, explicitly disabled and supported-enabled paths shell: pwsh run: ./.github/scripts/dissemination-evidence.ps1 -Mode Cost -SkipBuild - name: Preserve the automatic PR cost evidence diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index 62cb46314db..c89815590f5 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -72,7 +72,7 @@ The projection cache tracks its source snapshot as well as its table version. Sa ## Configuration and defaults -The subsystem and each namespace default to disabled. Enable together with on the selected integration to activate broadcast and repair. Defaults bound concurrency, memory retention, payload size, and repair work: +The subsystem and both built-in namespaces are temporarily enabled by default for pre-merge testing, so existing cluster tests exercise broadcast and repair. Set to `false` to select the direct-delivery control path, or set to `false` on an individual integration. Restore the opt-in defaults before merging this testing change. Defaults bound concurrency, memory retention, payload size, and repair work: | Option | Default | Effect | |---|---:|---| @@ -93,7 +93,7 @@ Each integration has its own /// Gets or sets a value indicating whether the dissemination subsystem is enabled. /// - /// by default. - public bool Enabled { get; set; } + /// during pre-merge testing. + public bool Enabled { get; set; } = true; /// /// Gets or sets the maximum number of concurrent local dissemination broadcast attempts. @@ -122,8 +122,8 @@ public sealed class DisseminationNamespaceOptions /// /// Gets or sets a value indicating whether this namespace is enabled. /// - /// by default. - public bool Enabled { get; set; } + /// during pre-merge testing. + public bool Enabled { get; set; } = true; /// /// Gets or sets the dissemination priority for this namespace. diff --git a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs index 08c0335ca57..20bf4a4dc3d 100644 --- a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs +++ b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs @@ -13,9 +13,12 @@ internal static class NewRuntime { public static void Configure(IServiceCollection services, NodeConfiguration configuration) { - // The disabled leg must exercise untouched production defaults, not explicit Enabled=false. + // Preserve the opt-out control while the candidate defaults are enabled for pre-merge testing. if (!configuration.Enabled) { + services.Configure(options => options.Enabled = false); + services.Configure(options => options.Dissemination.Enabled = false); + services.Configure(options => options.Dissemination.Enabled = false); return; } @@ -27,7 +30,6 @@ public static void Configure(IServiceCollection services, NodeConfiguration conf services.Configure(options => { - options.Enabled = true; if (configuration.FastRecovery) { options.Overlay.AntiEntropyInterval = TimeSpan.FromMilliseconds(250); @@ -37,13 +39,12 @@ public static void Configure(IServiceCollection services, NodeConfiguration conf options.MaxConcurrentSends = 4; } }); - services.Configure(options => Enable(options.Dissemination, configuration.FastRecovery)); - services.Configure(options => Enable(options.Dissemination, configuration.FastRecovery)); + services.Configure(options => ConfigureRecovery(options.Dissemination, configuration.FastRecovery)); + services.Configure(options => ConfigureRecovery(options.Dissemination, configuration.FastRecovery)); } - private static void Enable(DisseminationNamespaceOptions options, bool fastRecovery) + private static void ConfigureRecovery(DisseminationNamespaceOptions options, bool fastRecovery) { - options.Enabled = true; if (fastRecovery) { options.ExpectedUpdateCadence = TimeSpan.FromMilliseconds(100); diff --git a/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs b/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs index d31b5b18644..2a59e7e3a9a 100644 --- a/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs +++ b/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs @@ -9,7 +9,7 @@ public sealed class ScalingTests { [Fact] [Trait("Category", "DisseminationScale")] - public async Task Scaling_OriginalLegacy_DefaultOff_EnabledSupported() + public async Task Scaling_OriginalLegacy_Disabled_EnabledSupported() { var sizes = (Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_SIZES") ?? "4,8,16") .Split(',').Select(value => int.Parse(value, System.Globalization.CultureInfo.InvariantCulture)).ToArray(); @@ -23,7 +23,7 @@ public async Task Scaling_OriginalLegacy_DefaultOff_EnabledSupported() RuntimeVariant[] variants = [ new("OriginalLegacy", "Old", Enabled: false), - new("CurrentDefaultOff", "New", Enabled: false), + new("CurrentDisabled", "New", Enabled: false), new("CurrentEnabledSupported", "New", Enabled: true), ]; foreach (var size in sizes) @@ -225,7 +225,8 @@ private static async Task Measure(int size, string scenario, RuntimeVari WarmupRounds = 2, SettlingMarginSeconds = 2, AutomaticLoadPublisherPaused = true, - ProductionDisseminationDefaults = variant.Runtime == "New", + ProductionDisseminationDefaults = variant.Runtime == "New" && enabled, + ExplicitDisseminationOptOut = variant.Runtime == "New" && !enabled, LoadNamespaceSupportConfirmedBeforeOfferedRounds = enabled, IncludesUnconfirmedBootstrapTraffic = enabled && scenario == "churn", LivenessFailureDetectionEnabled = false, @@ -241,8 +242,8 @@ private static async Task Measure(int size, string scenario, RuntimeVari Topology = "per-node actual tree at end, or start for a retired node; raw command snapshots preserve intervening topology changes; role is not time-weighted", Excludes = "controller process CPU/allocation, storage disk byte accounting, cross-machine latency, NIC/TCP wire overhead, production storage/placement/grain work", PartitionComparison = "retire partitioned connections at both endpoints, then confirm healed connections with bidirectional acknowledged control RPCs before the same single publication round on all paths; teardown and readiness remain inside partition recovery time and total cost; silent anti-entropy recovery is asserted separately in VersionSkewTests", - OriginalBaselineAttribution = "original-to-current default-off deltas include all intervening source changes, including lifecycle prerequisites and load notification/tombstone/dedup changes; only current off/on isolates enabling the subsystem on the same binary", - Interpretation = "separate original-legacy, current-default-off and current-enabled-supported measurements; no asserted improvement or threshold", + OriginalBaselineAttribution = "original-to-current disabled deltas include all intervening source changes, including lifecycle prerequisites and load notification/tombstone/dedup changes; only current off/on isolates enabling the subsystem on the same binary", + Interpretation = "separate original-legacy, current-explicitly-disabled and current-enabled-supported measurements; enabled defaults are temporary for pre-merge testing", }, }; await cluster.Save("window-start.json", start); diff --git a/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs index 96fdeeb44c2..be2f90fec39 100644 --- a/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs +++ b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs @@ -96,7 +96,7 @@ await cluster.Start("Old", enabled: false), var disabled = replacements[0]; var enabled = replacements.Skip(1).ToArray(); - await cluster.Eventually("enabled peers confirm support, default-disabled peers remain legacy", async () => + await cluster.Eventually("enabled peers confirm support, explicitly disabled peers remain legacy", async () => { foreach (var node in enabled) { diff --git a/test/Dissemination.IntegrationHarness/methodology.json b/test/Dissemination.IntegrationHarness/methodology.json index 0e134dfe94c..285635a3acc 100644 --- a/test/Dissemination.IntegrationHarness/methodology.json +++ b/test/Dissemination.IntegrationHarness/methodology.json @@ -3,7 +3,7 @@ "purpose": "Executable real-binary compatibility/recovery gates and controlled load evidence, not a microbenchmark or a performance claim.", "entrypoint": ".github/scripts/dissemination-evidence.ps1", "workflow": ".github/workflows/dissemination-evidence.yml", - "automaticPrTrigger": "pull_request and merge_group run both the pinned-binary version-skew gate and measured bounded original-legacy, current-default-off and current-enabled-supported costs. The bounded job reuses published binaries, needs no label or workflow_dispatch registration, and has a 25-minute hard limit.", + "automaticPrTrigger": "pull_request and merge_group run both the pinned-binary version-skew gate and measured bounded original-legacy, current-explicitly-disabled and current-enabled-supported costs. The bounded job reuses published binaries, needs no label or workflow_dispatch registration, and has a 25-minute hard limit.", "controlledScaleTrigger": "Larger runs require workflow_dispatch with scale=true after the workflow is registered on the default branch. No pull_request_target or write permission.", "sliceInventory": [ ".github/scripts/dissemination-evidence.ps1", @@ -46,7 +46,7 @@ "old": "Always build the exact pinned pre-feature main commit; do not apply the lifecycle prerequisite or dissemination patches to it.", "manifestHashRpc": "No test uses a cluster-manifest hash RPC. Binary manifest.json files and local DLL SHA256 checks only attest which assemblies each OS process loaded; they are unrelated to the excluded manifest-hash feature.", "compatibility": "Probe the actual IDisseminationSystemTarget RPC on an old process, then require a fresh exact production load sample through legacy delivery while that process remains unconfirmed.", - "sliceOwnership": "All harness and CI files belong to the dissemination/evidence slice. The current dissemination top inherits the lifecycle and scoped-RPC prerequisites and applies load updates on the publisher scheduler while preserving default-off notification behavior. The pinned old source retains its original runtime implementation." + "sliceOwnership": "All harness and CI files belong to the dissemination/evidence slice. The current dissemination top inherits the lifecycle prerequisites and applies load updates on the publisher scheduler while preserving explicitly disabled notification behavior. The pinned old source retains its original runtime implementation." }, "infrastructure": { "hosting": "Existing TestClusterHostFactory, SystemTarget and code generation; OS process orchestration follows StandaloneSiloHandle without its single-binary configuration assumption.", @@ -58,11 +58,11 @@ "gates": [ { "test": "PartitionHealing_ConfirmsTransportBeforeSinglePublicationRound", - "assertions": "Repeat three partition/heal cycles on pinned old, current default-off and current enabled binaries. The first cycle proves an acknowledged control probe fails during the partition; remaining cycles publish and heal immediately. Healing retires old connections at both endpoints while the isolated endpoint rejects new handshakes, then bidirectional control RPCs confirm each exact remote PID. Full load snapshots must converge after one production publication per node with automatic publishers paused." + "assertions": "Repeat three partition/heal cycles on pinned old, current explicitly disabled and current enabled binaries. The first cycle proves an acknowledged control probe fails during the partition; remaining cycles publish and heal immediately. Healing retires old connections at both endpoints while the isolated endpoint rejects new handshakes, then bidirectional control RPCs confirm each exact remote PID. Full load snapshots must converge after one production publication per node with automatic publishers paused." }, { "test": "PinnedBinaries_RollForward_MixedEnablement_AndRollback", - "assertions": "Three pinned old processes are replaced one at a time by one default-disabled and two enabled new processes, then rolled back to pinned old processes. Exact load state equality and stable cross-process RPC PID identity are checked at each stage. Unsupported old IDisseminationSystemTarget RPC failures cannot be inferred from timeouts and must be followed by fresh exact load-state delivery to the still-unconfirmed old process. Disabled new nodes return explicit namespace-unsupported and retain legacy delivery. Enabled peers confirm support. Disabled nodes have no dissemination sends, applies or outgoing anti-entropy." + "assertions": "Three pinned old processes are replaced one at a time by one explicitly disabled and two enabled new processes, then rolled back to pinned old processes. Exact load state equality and stable cross-process RPC PID identity are checked at each stage. Unsupported old IDisseminationSystemTarget RPC failures cannot be inferred from timeouts and must be followed by fresh exact load-state delivery to the still-unconfirmed old process. Disabled new nodes return explicit namespace-unsupported and retain legacy delivery. Enabled peers confirm support. Disabled nodes have no dissemination sends, applies or outgoing anti-entropy." }, { "test": "PartitionHistoryAndHeartbeat_RequireDisseminationProvenance", @@ -105,11 +105,11 @@ } ], "load": { - "test": "Scaling_OriginalLegacy_DefaultOff_EnabledSupported", - "comparison": "Three separately labeled paths: pinned original legacy binary; actual current binary with untouched dissemination-off defaults; the same current binary enabled using production dissemination defaults after peers have confirmed support.", + "test": "Scaling_OriginalLegacy_Disabled_EnabledSupported", + "comparison": "Three separately labeled paths: pinned original legacy binary; actual current binary with dissemination and both namespaces explicitly disabled; the same current binary using temporarily enabled defaults after peers have confirmed support. Enabled defaults provide broader pre-merge test exposure.", "runtimePaths": [ { "name": "OriginalLegacy", "runtime": "Old", "enabled": false }, - { "name": "CurrentDefaultOff", "runtime": "New", "enabled": false }, + { "name": "CurrentDisabled", "runtime": "New", "enabled": false }, { "name": "CurrentEnabledSupported", "runtime": "New", "enabled": true } ], "automaticPrProfile": { "mode": "Cost", "sizes": [4, 8], "iterations": 3, "pairedRepetitions": 1, "overrideable": false }, @@ -134,7 +134,7 @@ "Controller CPU/allocation, disk byte costs, production providers, application grains, cross-machine latency and NIC-level wire costs are not measured. This is a closed-loop control-plane workload with a convergence barrier each round, not a saturation-throughput benchmark.", "Retained managed memory is a post-GC live-size sample on surviving nodes, not peak memory. Retired process snapshots and all-node RSS/private-memory samples remain in artifacts.", "Normal partition cost comparisons include a transport-readiness barrier followed by the same single publication round. The separate anti-entropy-only gate measures silent repair.", - "Original-to-current default-off deltas include the lifecycle and scoped-RPC prerequisites and feature registration costs. The current off/on pair uses the identical current binary and isolates enabling the subsystem.", + "Original-to-current disabled deltas include the lifecycle prerequisites and feature registration costs. The current off/on pair uses the identical current binary and isolates enabling the subsystem.", "Raw samples and repetitions are evidence inputs, not predeclared improvement thresholds or broad-enable approval." ] }, diff --git a/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs b/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs index fb5b6f098de..589dc7217cf 100644 --- a/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs +++ b/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs @@ -26,7 +26,7 @@ namespace NonSilo.Tests.Runtime; public class DeploymentLoadPublisherTests { [Fact] - public async Task PublishStatistics_DefaultDisabled_PreservesDirectPublication() + public async Task PublishStatistics_ExplicitlyDisabled_PreservesDirectPublication() { using var rig = CreateTestRig(TimeSpan.FromSeconds(5)); @@ -35,17 +35,15 @@ public async Task PublishStatistics_DefaultDisabled_PreservesDirectPublication() await rig.DirectTarget.Received(1).UpdateRuntimeStatistics( rig.LocalSilo, rig.Publisher.LocalRuntimeStatistics, TestContext.Current.CancellationToken); Assert.Empty(rig.Dissemination.ReceivedCalls()); - Assert.False(new DisseminationOptions().Enabled); - Assert.False(new DeploymentLoadPublisherOptions().Dissemination.Enabled); - Assert.False(new ClusterMembershipOptions().Dissemination.Enabled); + Assert.False(rig.ServiceProvider.GetRequiredService>().Value.Dissemination.Enabled); } [Theory] [InlineData(false)] [InlineData(true)] - public async Task PublishStatistics_ConfirmedPeersUseDissemination(bool confirmed) + public async Task PublishStatistics_DefaultEnabled_ConfirmedPeersUseDissemination(bool confirmed) { - using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: true); + using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: null); var remoteSilo = SiloAddress.FromParsableString("127.0.0.1:200@100"); rig.Dissemination.GetUnconfirmedPeers(Arg.Any()) .Returns(confirmed ? [] : new[] { remoteSilo }); @@ -157,7 +155,7 @@ public async Task ApplyLoadStatistics_UsesPublisherSchedulerAndSuppressesDissemi } [Fact] - public async Task UpdateRuntimeStatistics_DefaultDisabled_PreservesDuplicateNotifications() + public async Task UpdateRuntimeStatistics_ExplicitlyDisabled_PreservesDuplicateNotifications() { using var rig = CreateTestRig(TimeSpan.Zero); var listener = Substitute.For(); @@ -433,7 +431,7 @@ public async Task StatisticsSubscribers_DeliverToRemainingSubscribersBeforeSurfa private static TestRig CreateTestRig( TimeSpan refreshTime, ITimerRegistry? timerRegistry = null, - bool enableDissemination = false) + bool? enableDissemination = false) { var localSilo = SiloAddress.FromParsableString("127.0.0.1:100@100"); var remoteSilo = SiloAddress.FromParsableString("127.0.0.1:200@100"); @@ -483,7 +481,10 @@ private static TestRig CreateTestRig( services.AddOptions().Configure(options => { options.DeploymentLoadPublisherRefreshTime = refreshTime; - options.Dissemination.Enabled = enableDissemination; + if (enableDissemination is { } enabled) + { + options.Dissemination.Enabled = enabled; + } }); services.AddSingleton(); services.AddSingleton(serviceProvider => new SystemTargetShared( diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs index 27f30168059..65509cbc365 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs @@ -5,8 +5,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; -using Orleans.Configuration; -using Orleans.Hosting; using Orleans.Runtime; using Orleans.Runtime.Dissemination; using Orleans.Runtime.MembershipService; @@ -35,20 +33,13 @@ public sealed class DisseminationDiagnosticCollection public sealed class DisseminationClusterTests { [Fact] - public async Task MembershipUpdatesAreDisseminatedAcrossRealCluster() + public async Task MembershipUpdatesAreDisseminatedAcrossRealClusterWithDefaults() { using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); cancellation.CancelAfter(TimeSpan.FromSeconds(120)); var cancellationToken = cancellation.Token; var builder = new InProcessTestClusterBuilder(3); - builder.ConfigureSilo((_, siloBuilder) => - { - // Dissemination and the membership namespace are disabled by default; enable both. - siloBuilder.Configure(options => options.Enabled = true); - siloBuilder.Configure(options => options.Dissemination.Enabled = true); - }); - await using var cluster = builder.Build(); await cluster.DeployAsync(cancellationToken); await cluster.WaitForLivenessToStabilizeAsync().WaitAsync(cancellationToken); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs index 3b46bfe52b2..876f5d4564b 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs @@ -112,7 +112,7 @@ public async Task DisseminationStopReportsCallerCancellationAfterCleanup(bool ca { var local = CreateSilo(40321); var transport = new FakeTransport(local, CreateSilo(40322)); - var options = new ReviewOptionsMonitor(new DisseminationOptions()); + var options = new ReviewOptionsMonitor(new DisseminationOptions { Enabled = false }); var clock = new ReviewTimeProvider(); var details = new FakeLocalSiloDetails(local); var target = new DisseminationSystemTarget( diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs index aabb25643e2..f3d8a7ab969 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs @@ -203,7 +203,7 @@ public async Task DisabledAntiEntropySleepsUntilOptionsChangeAndUnsubscribesOnSt var transport = new FakeTransport(local, peer); var ns = new FakeNamespace(local); var clock = new ReviewTimeProvider(); - var options = new ReviewOptionsMonitor(new DisseminationOptions()); + var options = new ReviewOptionsMonitor(new DisseminationOptions { Enabled = false }); var exchanges = Channel.CreateUnbounded(); var exchangeCount = 0; transport.ExchangeAntiEntropyHandler = (destination, _, _) => @@ -244,7 +244,7 @@ public async Task DisabledAntiEntropySleepsUntilOptionsChangeAndUnsubscribesOnSt Assert.Equal(1, await exchanges.Reader.ReadAsync(TestContext.Current.CancellationToken)); await clock.WaitForChange(options.CurrentValue.Overlay.AntiEntropyInterval, TestContext.Current.CancellationToken); - options.Set(new DisseminationOptions()); + options.Set(new DisseminationOptions { Enabled = false }); await clock.WaitForChange(Timeout.InfiniteTimeSpan, TestContext.Current.CancellationToken); clock.Advance(TimeSpan.FromDays(1)); await target.RunOrQueueTask(_ => Task.FromResult(true), TestContext.Current.CancellationToken); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index 257e1acaea3..61886269568 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -3206,6 +3206,10 @@ public void DisseminationOptionsUseBatchDefaults() { var options = new DisseminationOptions(); + Assert.True(options.Enabled); + Assert.True(new DisseminationNamespaceOptions().Enabled); + Assert.True(new DeploymentLoadPublisherOptions().Dissemination.Enabled); + Assert.True(new ClusterMembershipOptions().Dissemination.Enabled); Assert.Equal(1024 * 1024, options.MaxBatchBytes); Assert.Equal(8 * 1024, options.MaxBatchItems); Assert.Equal(options.MaxBatchBytes, new DisseminationNamespaceOptions().MaxPayloadBytes); From 97cfeb1b5c90da50fef5ca94b5a11d6450c827d0 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 13:21:40 -0700 Subject: [PATCH 19/50] fix(dissemination): drain protocol operations with shared admission gate --- .../host/monitoring/runtime-dissemination.md | 2 +- .../implementation/runtime-dissemination.md | 4 +- .../Dissemination/DisseminationProtocol.cs | 36 ++- ...seminationProtocolTests.AdmissionReview.cs | 259 ++++++++++++++++++ 4 files changed, 297 insertions(+), 4 deletions(-) create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.AdmissionReview.cs diff --git a/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md b/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md index fa43aa6b655..e5abfa76150 100644 --- a/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md +++ b/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md @@ -75,7 +75,7 @@ Compare returned values with _logger; private readonly DisseminationBroadcastQueue _broadcastQueue; + private readonly AdmissionGate _admission = new(); private readonly DisseminationSendGate _antiEntropySendGate; private readonly CancellationTokenSource _antiEntropyShutdown = new(); private readonly object _antiEntropyResponseCursorLock = new(); @@ -68,6 +70,13 @@ public async ValueTask Publish( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + using var admission = _admission.TryEnter(); + if (!admission.Entered) + { + DisseminationInstruments.OnPublication(disseminationNamespace.Name, accepted: false, reason: "stopping"); + return false; + } + var options = _options.CurrentValue; if (!options.Enabled || !disseminationNamespace.Options.Enabled) { @@ -127,6 +136,9 @@ public async Task ReceiveBroadcast( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + using var admission = _admission.TryEnter(); + ObjectDisposedException.ThrowIf(!admission.Entered, this); + var receivedTimestamp = _timeProvider.GetTimestamp(); var options = _options.CurrentValue; if (!options.Enabled) @@ -237,6 +249,12 @@ public async Task ReceiveBroadcast( public async Task RunAntiEntropyRound(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + using var admission = _admission.TryEnter(); + if (!admission.Entered) + { + return; + } + var options = _options.CurrentValue; if (!options.Enabled) { @@ -727,6 +745,9 @@ public ValueTask ReceiveAntiEntropy( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + using var admission = _admission.TryEnter(); + ObjectDisposedException.ThrowIf(!admission.Entered, this); + if (request.MaxResponseItems is { } maxResponseItems) { ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxResponseItems, nameof(request.MaxResponseItems)); @@ -1126,9 +1147,20 @@ internal Task FlushPendingBroadcast(CancellationToken cancellationToken) => internal async Task StopAsync(CancellationToken cancellationToken) { + var admittedOperations = _admission.CloseAsync(); _antiEntropySendGate.Stop(); - await _antiEntropyShutdown.CancelAsync(); - await _broadcastQueue.StopAsync(cancellationToken); + try + { + await _antiEntropyShutdown.CancelAsync(); + // Admitted publishers and receivers finish enqueueing before the accepted broadcasts drain. + await admittedOperations.WaitAsync(cancellationToken); + } + finally + { + await _broadcastQueue.StopAsync(cancellationToken); + } + + cancellationToken.ThrowIfCancellationRequested(); } internal IReadOnlyList GetUnconfirmedPeers( diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.AdmissionReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.AdmissionReview.cs new file mode 100644 index 00000000000..65a322bb9aa --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.AdmissionReview.cs @@ -0,0 +1,259 @@ +#nullable enable + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task ShutdownDrainsAdmittedPublicationBeforeBroadcastQueue() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40501); + var peer = CreateSilo(40502); + var transport = new FakeTransport(local, peer); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.SetValue("value", 1); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var release = new ManualResetEventSlim(); + var repairs = 0; + ns.RepairHandler = request => + { + if (Interlocked.Increment(ref repairs) == 1) + { + entered.SetResult(); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), cancellationToken)); + } + + return ns.Inner.CreateRepair(request); + }; + var protocol = CreateProtocol(transport, [ns], timeProvider: new FakeTimeProvider()); + var publication = Task.Run(async () => await protocol.Publish(ns, "value", 1, cancellationToken), cancellationToken); + Task stop = Task.CompletedTask; + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + stop = protocol.StopAsync(cancellationToken); + Assert.False(stop.IsCompleted); + Assert.False(await protocol.Publish(ns, "value", 1, cancellationToken)); + Assert.Equal(1, Volatile.Read(ref repairs)); + release.Set(); + + Assert.True(await publication.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(peer, batch.Peer); + Assert.Equal(1, Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion); + } + finally + { + release.Set(); + await publication.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task ShutdownDrainsAdmittedBroadcastApplicationAndForwarding() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40511); + var sender = CreateSilo(40512); + var child = CreateSilo(40513); + var transport = new FakeTransport(local, sender, child); + var ns = new ProtocolReviewNamespace(local); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + ns.ApplyHandler = async (value, token) => + { + await release.Task.WaitAsync(token); + return await ns.Inner.ApplyValueAsync(value, token); + }; + var protocol = CreateProtocol(transport, [ns], options => options.Overlay.FanOutFactor = static _ => 2); + var batch = new DisseminationBroadcastBatch + { + Sender = sender, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(sender, "value", 1)), + }; + var receive = protocol.ReceiveBroadcast(batch, cancellationToken); + Task stop = Task.CompletedTask; + try + { + Assert.Single(ns.Attempts); + stop = protocol.StopAsync(cancellationToken); + Assert.False(stop.IsCompleted); + await Assert.ThrowsAsync(() => protocol.ReceiveBroadcast(batch, cancellationToken)); + Assert.Single(ns.Attempts); + release.SetResult(); + + var response = await receive.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(1, Assert.Single(response.Acknowledgments[ns.Name]).Version); + Assert.Equal(1, ns.GetVersion("value")); + Assert.Equal(child, Assert.Single(transport.BroadcastBatches).Peer); + } + finally + { + release.TrySetResult(); + await receive.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task ShutdownDrainsAdmittedAntiEntropyResponse() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40521); + var peer = CreateSilo(40522); + var transport = new FakeTransport(local, peer); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.SetValue("value", 1); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var release = new ManualResetEventSlim(); + ns.RepairHandler = request => + { + entered.TrySetResult(); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), cancellationToken)); + return ns.Inner.CreateRepair(request); + }; + var protocol = CreateProtocol(transport, [ns]); + var request = new DisseminationAntiEntropyRequest + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = CreateAntiEntropyRequestDigest(ns.Name, ("value", 0)), + }; + var response = Task.Run(async () => await protocol.ReceiveAntiEntropy(request, cancellationToken), cancellationToken); + Task stop = Task.CompletedTask; + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + stop = protocol.StopAsync(cancellationToken); + Assert.False(stop.IsCompleted); + await Assert.ThrowsAsync(async () => await protocol.ReceiveAntiEntropy(request, cancellationToken)); + release.Set(); + + var result = await response.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(1, Assert.Single(result.Values[ns.Name]).Value.ToVersion); + } + finally + { + release.Set(); + await response.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task ShutdownCancelsAndDrainsLocalAntiEntropyRound() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40531); + var peer = CreateSilo(40532); + var transport = new FakeTransport(local, peer); + var raw = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.ExchangeAntiEntropyHandler = (_, _, token) => + { + started.SetResult(token); + return new(raw.Task); + }; + var protocol = CreateProtocol(transport, new FakeNamespace(local)); + var round = protocol.RunAntiEntropyRound(cancellationToken); + try + { + var transportToken = await started.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await protocol.StopAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + + Assert.True(transportToken.IsCancellationRequested); + await Assert.ThrowsAnyAsync( + () => round.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + Assert.False(raw.Task.IsCompleted); + await protocol.RunAntiEntropyRound(cancellationToken); + Assert.Single(transport.AntiEntropyRequests); + } + finally + { + raw.TrySetResult(new DisseminationAntiEntropyResponse { Sender = peer }); + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task ShutdownCancellationKeepsAdmissionClosed() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40541); + var peer = CreateSilo(40542); + var transport = new FakeTransport(local, peer); + var ns = new ProtocolReviewNamespace(local); + using var requestCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + ns.ApplyHandler = async (_, token) => + { + await Task.Delay(Timeout.InfiniteTimeSpan, token); + return DisseminationApplyResult.Applied; + }; + var protocol = CreateProtocol(transport, [ns], timeProvider: new FakeTimeProvider()); + var batch = new DisseminationBroadcastBatch + { + Sender = peer, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(peer, "value", 1)), + }; + var receive = protocol.ReceiveBroadcast(batch, requestCancellation.Token); + using var shutdown = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var stop = protocol.StopAsync(shutdown.Token); + try + { + Assert.Single(ns.Attempts); + Assert.False(stop.IsCompleted); + shutdown.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + () => stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + Assert.Equal(shutdown.Token, exception.CancellationToken); + Assert.False(receive.IsCompleted); + Assert.False(await protocol.Publish(ns, "value", 1, cancellationToken)); + await Assert.ThrowsAsync(() => protocol.ReceiveBroadcast(batch, cancellationToken)); + } + finally + { + requestCancellation.Cancel(); + await Assert.ThrowsAnyAsync( + () => receive.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + await protocol.StopAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + + Assert.Equal(0, ns.GetVersion("value")); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task ShutdownRejectsPublicationWithNoPeersAndPreservesCallerCancellation() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40551); + var transport = new FakeTransport(local); + var ns = new FakeNamespace(local); + ns.SetValue("value", 1); + var protocol = CreateProtocol(transport, ns); + await protocol.StopAsync(cancellationToken); + + Assert.False(await protocol.Publish(ns, "value", 1, cancellationToken)); + using var canceled = new CancellationTokenSource(); + canceled.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + async () => await protocol.Publish(ns, "value", 1, canceled.Token)); + Assert.Equal(canceled.Token, exception.CancellationToken); + Assert.Empty(transport.BroadcastBatches); + } +} From 15a500833559c19024e0dc84c1ec8c8639f95820 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 14:19:15 -0700 Subject: [PATCH 20/50] perf(dissemination): reduce load propagation delays and allocation --- .../implementation/runtime-dissemination.md | 4 + .../Options/DeploymentLoadPublisherOptions.cs | 6 +- ...entLoadStatisticsDisseminationNamespace.cs | 39 ++++- .../DisseminationBroadcastQueue.cs | 32 ++-- .../Dissemination/DisseminationProtocol.cs | 76 ++++++++- .../Dissemination/IDisseminationNamespace.cs | 12 ++ .../MembershipDisseminationNamespace.cs | 3 + ...minationProtocolTests.PerformanceReview.cs | 161 ++++++++++++++++++ .../DisseminationProtocolTests.cs | 1 + 9 files changed, 310 insertions(+), 24 deletions(-) create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PerformanceReview.cs diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index 254be920821..c66b1c2554d 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -53,6 +53,8 @@ Membership uses the membership-table version plus a liveness fingerprint in its Deployment-load versions are sample timestamps. Each repair is a full latest value. Application runs on the deployment-load publisher's scheduler and accepts samples only for Active silo generations in the membership oracle. Terminal membership transitions remove placement statistics, and the oracle's monotonic membership state rejects later samples for departed generations. +After the owner accepts a load sample, its serialized payload is reused for forwarding and repair. Inventory maintenance enumerates namespace keys directly; membership exposes its single key, and deployment load uses the active-silo oracle. Version and fingerprint construction is reserved for operations which use that information. Receive batches which fit all item, byte, and namespace limits are processed directly from their existing collections; truncated batches retain bounded cursor selection. + ## Deterministic topology Every silo derives routing from the same ordered membership projection. Members are ordered by status and silo address. Silo addresses order by generation, port, and IP address, providing stable ordering across membership storage providers with different timestamp precision. Joining, Active, ShuttingDown, and Stopping entries participate. @@ -91,6 +93,8 @@ The subsystem and both built-in namespaces are temporarily enabled by default fo Each integration has its own . Operators can enable and tune membership and deployment-load dissemination independently while retaining the local concurrency and per-message bounds. +Deployment load uses a 10 ms coalescing window and a 5-second expected update cadence. The shorter window batches nearby samples while limiting the delay added at each forwarding hop. Membership retains high priority and bypasses coalescing. + The anti-entropy loop waits without periodic timer wakeups while the subsystem is disabled. An options-change notification wakes the loop when enablement changes; disabling it returns the loop to the dormant wait. Shutdown removes the options subscription and observes the loop's completion. Enablement requires representative measurements of convergence, RPC volume, serialized bytes, allocation, CPU, and tail latency across stable membership, churn, and partition recovery. Tree delivery can reduce RPC counts while increasing serialized bytes through forwarding and repair. Compare the original runtime, the updated explicitly disabled path, and the enabled path at the intended cluster sizes and publication rates, including skewed peer speeds and rolling upgrades. diff --git a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs index f4e08c1efbb..ec935aa4aa9 100644 --- a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs @@ -28,6 +28,10 @@ public class DeploymentLoadPublisherOptions /// unavailable, declines the update, throws, or cannot accept it within the refresh interval, direct /// publication targets all active peers. /// - public DisseminationNamespaceOptions Dissemination { get; set; } = new() { ExpectedUpdateCadence = TimeSpan.FromSeconds(5) }; + public DisseminationNamespaceOptions Dissemination { get; set; } = new() + { + ExpectedUpdateCadence = TimeSpan.FromSeconds(5), + MaxCoalescingDelay = TimeSpan.FromMilliseconds(10), + }; } } diff --git a/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs index 31f4425d614..951c69ea7c6 100644 --- a/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs @@ -40,7 +40,7 @@ public DisseminationValue CreateValue(SiloAddress origin, SiloRuntimeStatistics } } - public IEnumerable Digests + public IEnumerable Keys { get { @@ -48,7 +48,18 @@ public IEnumerable Digests PruneCache(activeSilos); foreach (var siloAddress in activeSilos.Keys) { - yield return new DigestEntry(siloAddress, GetVersion(siloAddress)); + yield return siloAddress; + } + } + } + + public IEnumerable Digests + { + get + { + foreach (var key in Keys) + { + yield return new DigestEntry(key, GetVersion(key)); } } } @@ -111,7 +122,29 @@ public ValueTask ApplyValueAsync( } cancellationToken.ThrowIfCancellationRequested(); - return new(deploymentLoadPublisher.ApplyDisseminatedRuntimeStatisticsAsync(siloAddress, statistics, cancellationToken)); + return ApplyAndCache(siloAddress, statistics, value, cancellationToken); + } + + private async ValueTask ApplyAndCache( + SiloAddress origin, + SiloRuntimeStatistics statistics, + DisseminationValue value, + CancellationToken cancellationToken) + { + var result = await deploymentLoadPublisher.ApplyDisseminatedRuntimeStatisticsAsync(origin, statistics, cancellationToken); + if (result is DisseminationApplyResult.Applied) + { + lock (_cacheLock) + { + // The owner accepted these exact bytes. Forward them without serializing the same sample again. + if (!_cachedValues.TryGetValue(origin, out var cached) || cached.ToVersion < value.ToVersion) + { + _cachedValues[origin] = value; + } + } + } + + return result; } private void PruneCache(Dictionary activeSilos) diff --git a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs index e537a509725..a0cd6c2438b 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs @@ -133,15 +133,13 @@ public async Task Prune( DisseminationMembershipSnapshots membershipSnapshots, CancellationToken cancellationToken) { - // Namespace digests are the authoritative inventory, so clean ledger entries can disappear with their keys. + // Namespace identities are the authoritative inventory for retiring clean ledger entries. var activeKeys = new Dictionary>(); foreach (var disseminationNamespace in _disseminationNamespaces) { if (disseminationNamespace.Options.Enabled) { - activeKeys[disseminationNamespace.Name] = disseminationNamespace.Digests - .Select(static entry => entry.Key) - .ToHashSet(); + activeKeys[disseminationNamespace.Name] = disseminationNamespace.Keys.ToHashSet(); } } @@ -455,7 +453,7 @@ public void PruneKeys( var droppedDirtyCount = 0; lock (_lock) { - foreach (var (namespaceName, namespaceState) in _statesByNamespace.ToArray()) + foreach (var (namespaceName, namespaceState) in _statesByNamespace) { if (!membershipSnapshots.GetSnapshot(namespaceState.Namespace.MembershipScope).ContainsMember(Peer)) { @@ -466,7 +464,7 @@ public void PruneKeys( } activeKeys.TryGetValue(namespaceName, out var namespaceKeys); - foreach (var (key, keyState) in namespaceState.Keys.ToArray()) + foreach (var (key, keyState) in namespaceState.Keys) { if (!keyState.Dirty && !keyState.InFlight @@ -971,10 +969,12 @@ private async ValueTask SendValues( // RPC completion is not application evidence; only the returned receiver versions advance the ledger. _owner._responseObserver?.Invoke(Peer, response); var acknowledgments = CreateAcknowledgmentLookup(response.Acknowledgments); - var unsupportedNamespaces = response.UnsupportedNamespaces.ToHashSet(); + var unsupportedNamespaces = response.UnsupportedNamespaces.Count > 0 + ? response.UnsupportedNamespaces.ToHashSet() + : null; foreach (var sent in sentKeys) { - if (unsupportedNamespaces.Contains(sent.Work.Namespace.Name)) + if (unsupportedNamespaces?.Contains(sent.Work.Namespace.Name) == true) { CompleteUnsupported(sent.Work.Namespace.Name, initialWork); continue; @@ -1016,9 +1016,17 @@ private async ValueTask SendValues( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var transportLifetime = valuesByNamespace.Values - .SelectMany(static values => values) - .Min(static value => value.TimeToLive); + var transportLifetime = TimeSpan.MaxValue; + foreach (var values in valuesByNamespace.Values) + { + foreach (var value in values) + { + if (value.TimeToLive < transportLifetime) + { + transportLifetime = value.TimeToLive; + } + } + } if (transportLifetime <= TimeSpan.Zero) { return null; @@ -1554,7 +1562,7 @@ public void RetireKey(DisseminationKey key, PeerKeyState keyState) public void PruneKnownVersions(HashSet? activeKeys) { - foreach (var key in KnownVersions.Keys.ToArray()) + foreach (var key in KnownVersions.Keys) { if (activeKeys is null || !activeKeys.Contains(key)) { diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index 5a72787080c..d247332cb46 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -642,8 +642,11 @@ private Dictionary> Se DisseminationOptions options, bool antiEntropy) { - var result = new Dictionary>(); - var totalCount = values.Values.Sum(static entries => (long)entries.Count); + long totalCount = 0; + foreach (var entries in values.Values) + { + totalCount += entries.Count; + } var cursorKey = (peer, antiEntropy); long position; lock (_receivedBatchCursorLock) @@ -654,6 +657,19 @@ private Dictionary> Se : 0; } + if (position == 0 && FitsReceiveBudget(values, totalCount, options)) + { + var fastPathMembers = _membership.CurrentSnapshots.AllMembers; + lock (_receivedBatchCursorLock) + { + _receivedBatchCursors.Remove(cursorKey); + PruneReceivedBatchCursors(fastPathMembers); + } + + return values; + } + + var result = new Dictionary>(); var skip = position; var examined = 0; var byteCount = 0; @@ -725,6 +741,51 @@ private Dictionary> Se _receivedBatchCursors.Remove(cursorKey); } + PruneReceivedBatchCursors(members); + } + + return result; + } + + private bool FitsReceiveBudget( + Dictionary> values, + long itemCount, + DisseminationOptions options) + { + if (itemCount > options.MaxBatchItems) + { + return false; + } + + var remainingBytes = options.MaxBatchBytes; + foreach (var (namespaceName, entries) in values) + { + if (entries.Count == 0 || !TryGetEnabledNamespace(namespaceName, out var disseminationNamespace)) + { + return false; + } + + var maxPayloadBytes = disseminationNamespace.Options.MaxPayloadBytes; + foreach (var entry in entries) + { + var length = entry.Value.Payload.Length; + if (remainingBytes == 0 || length > maxPayloadBytes || length > remainingBytes) + { + return false; + } + + remainingBytes -= length; + } + } + + return true; + } + + // Called under _receivedBatchCursorLock. + private void PruneReceivedBatchCursors(DisseminationMembershipSnapshot members) + { + if (_receivedBatchCursors.Count > MaxRetainedNonMemberResponseCursors) + { var nonMembers = _receivedBatchCursors .Where(entry => !members.ContainsMember(entry.Key.Peer)) .OrderByDescending(static entry => entry.Value.LastAccess) @@ -736,8 +797,6 @@ private Dictionary> Se _receivedBatchCursors.Remove(key); } } - - return result; } public ValueTask ReceiveAntiEntropy( @@ -1259,11 +1318,12 @@ private void PrunePeerNamespaceConfirmations(DisseminationMembershipSnapshot mem { lock (_peerSupportLock) { - foreach (var peer in _confirmedPeerNamespaces.Keys - .Where(peer => !membership.ContainsMember(peer)) - .ToArray()) + foreach (var peer in _confirmedPeerNamespaces.Keys) { - _confirmedPeerNamespaces.Remove(peer); + if (!membership.ContainsMember(peer)) + { + _confirmedPeerNamespaces.Remove(peer); + } } } } diff --git a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs index 9016f7f20eb..eda1a73809d 100644 --- a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs @@ -14,6 +14,18 @@ internal interface IDisseminationNamespace IEnumerable Digests { get; } + // Inventory maintenance needs identities, independently of versions and anti-entropy fingerprints. + IEnumerable Keys + { + get + { + foreach (var digest in Digests) + { + yield return digest.Key; + } + } + } + long GetVersion(DisseminationKey key); DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request); diff --git a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs index 835d188bd83..bc5073d2e2a 100644 --- a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs @@ -14,6 +14,7 @@ internal sealed class MembershipDisseminationNamespace( Serializer serializer) : IDisseminationNamespace { private const int MaxSnapshotHistory = 32; + private static readonly DisseminationKey[] MembershipKeys = [DisseminationKey.Default]; private readonly object _historyLock = new(); private readonly SortedDictionary _snapshotHistory = new(); private readonly Dictionary> _snapshotPayloads = []; @@ -25,6 +26,8 @@ internal sealed class MembershipDisseminationNamespace( public DisseminationNamespaceOptions Options => options.CurrentValue.Dissemination; + public IEnumerable Keys => MembershipKeys; + public async ValueTask PublishAsync( IDisseminationService disseminationService, MembershipTableSnapshot snapshot, diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PerformanceReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PerformanceReview.cs new file mode 100644 index 00000000000..178f29b4345 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PerformanceReview.cs @@ -0,0 +1,161 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.MembershipService; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task LoadRepairReusesOnlySuccessfullyAppliedPayloads() + { + var cancellationToken = TestContext.Current.CancellationToken; + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var ns = new DeploymentLoadStatisticsDisseminationNamespace( + harness.Publisher, new TestOptionsMonitor(new()), serializer); + var first = CreatePhase5Statistics(1); + var value = new DisseminationValue(harness.ActiveOne, 0, first.DateTime.Ticks, serializer.SerializeToArray(first)); + + Assert.Equal(DisseminationApplyResult.Applied, await ns.ApplyValueAsync(value, cancellationToken)); + Assert.True(value.Payload.Equals(Repair().Payload)); + + var duplicate = new DisseminationValue(value.Key, 0, value.ToVersion, value.Payload.ToArray()); + Assert.Equal(DisseminationApplyResult.Duplicate, await ns.ApplyValueAsync(duplicate, cancellationToken)); + Assert.True(value.Payload.Equals(Repair().Payload)); + + var second = CreatePhase5Statistics(2); + var replacement = new DisseminationValue(value.Key, 0, second.DateTime.Ticks, serializer.SerializeToArray(second)); + Assert.Equal(DisseminationApplyResult.Applied, await ns.ApplyValueAsync(replacement, cancellationToken)); + Assert.True(replacement.Payload.Equals(Repair().Payload)); + Assert.Equal(DisseminationApplyResult.Obsolete, await ns.ApplyValueAsync(value, cancellationToken)); + Assert.True(replacement.Payload.Equals(Repair().Payload)); + + var malformed = new DisseminationValue(value.Key, 0, replacement.ToVersion + 1, replacement.Payload); + Assert.Equal(DisseminationApplyResult.Rejected, await ns.ApplyValueAsync(malformed, cancellationToken)); + Assert.True(replacement.Payload.Equals(Repair().Payload)); + + DisseminationValue Repair() + { + var repair = ns.CreateRepair(new(value.Key, null, null, 1, 1024, 1024)); + Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); + return Assert.Single(repair.Values); + } + } + + [Fact] + public void MembershipKeyInventoryRequiresNoSnapshotOrFingerprint() + { + var manager = Substitute.For(); + manager.CurrentSnapshot.Returns(_ => throw new InvalidOperationException("Key enumeration must not read membership state.")); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var ns = CreateMembershipNamespace(manager, services.GetRequiredService()); + + Assert.Equal(DisseminationKey.Default, Assert.Single(ns.Keys)); + } + + [Fact] + public async Task QueuePruningUsesKeyInventoryWithoutBuildingDigests() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40601); + var peer = CreateSilo(40602); + var transport = new FakeTransport(local, peer); + var ns = new InventoryOnlyNamespace(local); + ns.Inner.SetValue("value", 1); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + var membership = new DisseminationMembership( + transport.MembershipManager, + new FakeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + try + { + Assert.True(queue.Notify(peer, ns, "value")); + await queue.Prune(membership.CurrentSnapshots, cancellationToken); + await queue.FlushPendingBroadcast(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await queue.Prune(membership.CurrentSnapshots, cancellationToken); + + Assert.Equal(1, Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)).Value.ToVersion); + } + finally + { + await queue.StopAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + } + + [Fact] + public async Task ReceiveBudgetRetainsCursorForZeroByteSuffixAndIgnoresEmptyNamespaces() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40611); + var peer = CreateSilo(40612); + var ns = new ProtocolReviewNamespace(local); + var empty = new FakeNamespace(local, "empty"); + ns.ApplyHandler = (value, _) => + { + ns.Inner.SetValue(value.Key, value.ToVersion); + return new(DisseminationApplyResult.Applied); + }; + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns, empty], options => + { + options.MaxBatchBytes = 8; + options.MaxBatchItems = 2; + }); + var batch = new DisseminationBroadcastBatch + { + Sender = peer, + Values = new() + { + [ns.Name] = + [ + CreateDisseminationValue(peer, new("first", 0, 1, new byte[8])), + CreateDisseminationValue(peer, new("second", 0, 1, ReadOnlyMemory.Empty)), + ], + [empty.Name] = [], + }, + }; + try + { + var first = await protocol.ReceiveBroadcast(batch, cancellationToken); + Assert.Equal("first", Assert.IsType(Assert.Single(first.Acknowledgments[ns.Name]).Key.Value)); + Assert.False(first.Acknowledgments.ContainsKey(empty.Name)); + Assert.Equal(0, ns.GetVersion("second")); + + var second = await protocol.ReceiveBroadcast(batch, cancellationToken); + Assert.Equal("second", Assert.IsType(Assert.Single(second.Acknowledgments[ns.Name]).Key.Value)); + Assert.Equal(new DisseminationKey[] { "first", "second" }, ns.Attempts.Select(static value => value.Key)); + Assert.Equal(1, ns.GetVersion("second")); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + + private sealed class InventoryOnlyNamespace(SiloAddress local) : IDisseminationNamespace + { + public FakeNamespace Inner { get; } = new(local); + public DisseminationNamespace Name => Inner.Name; + public DisseminationNamespaceOptions Options => Inner.Options; + public IEnumerable Keys => Inner.Digests.Select(static digest => digest.Key); + public IEnumerable Digests => throw new InvalidOperationException("Pruning must use key inventory."); + public long GetVersion(DisseminationKey key) => Inner.GetVersion(key); + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) => Inner.CreateRepair(request); + public ValueTask ApplyValueAsync(DisseminationValue value, CancellationToken cancellationToken) => + Inner.ApplyValueAsync(value, cancellationToken); + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index 61886269568..2533513223b 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -3198,6 +3198,7 @@ public void OptionsValidatorRejectsInvalidExpectedUpdateCadence() public void NamespaceOptionsUseExpectedUpdateCadenceDefaults() { Assert.Equal(TimeSpan.FromSeconds(5), new DeploymentLoadPublisherOptions().Dissemination.ExpectedUpdateCadence); + Assert.Equal(TimeSpan.FromMilliseconds(10), new DeploymentLoadPublisherOptions().Dissemination.MaxCoalescingDelay); Assert.Equal(TimeSpan.FromSeconds(10), new ClusterMembershipOptions().Dissemination.ExpectedUpdateCadence); } From 92a231fc1aa7619c1ac6ca5ff023e199c30db526 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 15:14:41 -0700 Subject: [PATCH 21/50] test(dissemination): compare off-on scaling through 100 silos --- .github/scripts/dissemination-evidence.ps1 | 50 ++++++++- .github/workflows/dissemination-evidence.yml | 31 +++-- .../implementation/runtime-dissemination.md | 8 ++ .../Shared/Protocol.cs | 8 +- .../Silo/NewRuntime.cs | 2 + .../Silo/Program.cs | 2 + .../Tests/MeasurementTests.cs | 77 +++++++++++++ .../Tests/ProcessCluster.cs | 34 +++++- .../Tests/ScalingComparison.cs | 106 ++++++++++++++++++ .../Tests/ScalingTests.cs | 96 +++++++++++++--- .../methodology.json | 19 ++-- 11 files changed, 393 insertions(+), 40 deletions(-) create mode 100644 test/Dissemination.IntegrationHarness/Tests/ScalingComparison.cs diff --git a/.github/scripts/dissemination-evidence.ps1 b/.github/scripts/dissemination-evidence.ps1 index c397ab44cf1..e92fc6f7fe2 100644 --- a/.github/scripts/dissemination-evidence.ps1 +++ b/.github/scripts/dissemination-evidence.ps1 @@ -10,6 +10,14 @@ param( [int] $Iterations = 10, [ValidateRange(1, 5)] [int] $Repetitions = 1, + [ValidateSet('All', 'Current')] + [string] $RuntimePaths = 'All', + [ValidatePattern('^(stable|churn|partition)(,(stable|churn|partition)){0,2}$')] + [string] $Scenarios = 'stable,churn,partition', + [ValidateRange(0, 64)] + [int] $SiloProcessorCount = 0, + [ValidateRange(0, 9)] + [int] $GCConserveMemory = 0, [switch] $SkipBuild ) @@ -21,6 +29,10 @@ if ($Mode -eq 'Cost') { $Sizes = '4,8' $Iterations = 3 $Repetitions = 1 + $RuntimePaths = 'All' + $Scenarios = 'stable,churn,partition' + $SiloProcessorCount = 0 + $GCConserveMemory = 0 } $baselineSha = '6739589254b746a8790cf53524e6abe372bb53d4' @@ -31,8 +43,32 @@ $results = Join-Path $artifacts "results/$Mode" New-Item -ItemType Directory -Force $binaries, $results | Out-Null Copy-Item -LiteralPath (Join-Path $root 'test/Dissemination.IntegrationHarness/methodology.json') -Destination $results foreach ($size in $Sizes.Split(',')) { - if ([int]$size -lt 3 -or [int]$size -gt 32) { - throw 'Silo counts must be between 3 and 32.' + if ([int]$size -lt 3 -or [int]$size -gt 128) { + throw 'Silo counts must be between 3 and 128.' + } +} + +if ($Mode -eq 'Scale' -and $IsLinux) { + $memory = Get-Content -LiteralPath '/proc/meminfo' + $availableLine = $memory | Where-Object { $_ -match '^MemAvailable:\s+(\d+)\s+kB$' } + if (!$availableLine -or $availableLine -notmatch '^MemAvailable:\s+(\d+)\s+kB$') { + throw 'Cannot establish available host memory for the scale run.' + } + $availableBytes = [long]$Matches[1] * 1024 + $largestSize = ($Sizes.Split(',') | ForEach-Object { [int]$_ } | Measure-Object -Maximum).Maximum + $requiredBytes = [long]$largestSize * 128MB + 1GB + @{ + AvailableBytes = $availableBytes + RequiredPlanningBytes = $requiredBytes + PlanningBytesPerSilo = 128MB + ControllerHeadroomBytes = 1GB + HostProcessorCount = [Environment]::ProcessorCount + SiloProcessorCount = $SiloProcessorCount + GCConserveMemory = $GCConserveMemory + MemoryInformation = $memory + } | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $results 'host-resources.json') + if ($availableBytes -lt $requiredBytes) { + throw "Insufficient host memory for $largestSize silo processes: available=$availableBytes, planning requirement=$requiredBytes. Use a larger runner." } } @@ -136,6 +172,10 @@ $env:ORLEANS_DISSEMINATION_SIZES = $Sizes $env:ORLEANS_DISSEMINATION_ITERATIONS = $Iterations.ToString([System.Globalization.CultureInfo]::InvariantCulture) $env:ORLEANS_DISSEMINATION_REPETITIONS = $Repetitions.ToString([System.Globalization.CultureInfo]::InvariantCulture) $env:ORLEANS_DISSEMINATION_PROFILE = $Mode +$env:ORLEANS_DISSEMINATION_RUNTIME_PATHS = $RuntimePaths +$env:ORLEANS_DISSEMINATION_SCENARIOS = $Scenarios +$env:ORLEANS_DISSEMINATION_SILO_PROCESSOR_COUNT = $SiloProcessorCount.ToString([System.Globalization.CultureInfo]::InvariantCulture) +$env:ORLEANS_DISSEMINATION_GC_CONSERVE_MEMORY = $GCConserveMemory.ToString([System.Globalization.CultureInfo]::InvariantCulture) $runner = Join-Path $binaries 'Runner/Dissemination.IntegrationHarness.Tests.dll' if (!(Test-Path -LiteralPath $runner)) { throw "Published runner not found: $runner" @@ -154,6 +194,10 @@ $minimum = if ($Mode -eq 'Gate') { '7' } else { '1' } Sizes = $Sizes Iterations = $Iterations Repetitions = $Repetitions + RuntimePaths = $RuntimePaths + Scenarios = $Scenarios + SiloProcessorCount = $SiloProcessorCount + GCConserveMemory = $GCConserveMemory StartUtc = [DateTimeOffset]::UtcNow Command = "dotnet $runner --filter-class $testClass --minimum-expected-tests $minimum --report-trx" Limitation = 'Loopback OS processes; controlled test membership provider; not cross-machine production throughput.' @@ -164,7 +208,7 @@ try { if ($Mode -eq 'Gate') { Invoke-DotNet @( $runner, '--filter-class', 'Orleans.Dissemination.IntegrationHarness.MeasurementTests', - '--minimum-expected-tests', '10', '--report-trx', + '--minimum-expected-tests', '16', '--report-trx', '--results-directory', (Join-Path $results 'instrument-checks') ) } diff --git a/.github/workflows/dissemination-evidence.yml b/.github/workflows/dissemination-evidence.yml index a81d8ab534f..af50619d295 100644 --- a/.github/workflows/dissemination-evidence.yml +++ b/.github/workflows/dissemination-evidence.yml @@ -3,7 +3,7 @@ name: Dissemination evidence on: pull_request: branches: [main] - types: [opened, synchronize, reopened] + types: [opened, synchronize, reopened, labeled] paths: - '.github/workflows/dissemination-evidence.yml' - '.github/scripts/dissemination-evidence.ps1' @@ -33,9 +33,9 @@ on: type: boolean default: false sizes: - description: 'Comma-separated process counts (3..32, maximum six sizes)' + description: 'Comma-separated process counts (3..128, maximum six sizes)' type: string - default: '4,8,16' + default: '8,16,32,64,100' iterations: description: 'Publication rounds per scenario (3..200)' type: number @@ -43,7 +43,7 @@ on: repetitions: description: 'Paired off/on repetitions (1..5)' type: number - default: 1 + default: 2 permissions: contents: read @@ -59,6 +59,7 @@ env: jobs: version-skew: name: Pinned-binary upgrade, rollback and recovery + if: github.event_name != 'pull_request' || github.event.action != 'labeled' || github.event.label.name == 'dissemination-scale' runs-on: ubuntu-latest timeout-minutes: 35 steps: @@ -123,11 +124,15 @@ jobs: path: Artifacts/DisseminationEvidence/results/Cost scaling: - name: Larger controlled dissemination cost comparison - if: github.event_name == 'workflow_dispatch' && inputs.scale + name: Dissemination scaling (${{ matrix.size }} silos) + if: (github.event_name == 'workflow_dispatch' && inputs.scale) || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'dissemination-scale')) needs: [version-skew, bounded-cost] runs-on: ubuntu-latest timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + size: ${{ fromJSON(format('[{0}]', inputs.sizes || '8,16,32,64,100')) }} steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 with: @@ -139,21 +144,23 @@ jobs: with: name: dissemination-binaries path: Artifacts/DisseminationEvidence/bin - - name: Measure stable, churn and partition windows on all three paths + - name: Measure paired current-binary stable scaling shell: pwsh env: - EVIDENCE_SIZES: ${{ inputs.sizes || '4,8,16' }} + EVIDENCE_SIZE: ${{ matrix.size }} EVIDENCE_ITERATIONS: ${{ inputs.iterations || 10 }} - EVIDENCE_REPETITIONS: ${{ inputs.repetitions || 1 }} + EVIDENCE_REPETITIONS: ${{ inputs.repetitions || 2 }} run: | ./.github/scripts/dissemination-evidence.ps1 -Mode Scale -SkipBuild ` - -Sizes $env:EVIDENCE_SIZES ` + -Sizes $env:EVIDENCE_SIZE ` -Iterations ([int]$env:EVIDENCE_ITERATIONS) ` - -Repetitions ([int]$env:EVIDENCE_REPETITIONS) + -Repetitions ([int]$env:EVIDENCE_REPETITIONS) ` + -RuntimePaths Current -Scenarios stable ` + -SiloProcessorCount 1 -GCConserveMemory 9 - name: Preserve raw cost windows, per-node load and failure evidence if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: dissemination-scale-evidence + name: dissemination-scale-${{ matrix.size }}-evidence retention-days: 30 path: Artifacts/DisseminationEvidence/results diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index c66b1c2554d..4d2d269541e 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -99,6 +99,14 @@ The anti-entropy loop waits without periodic timer wakeups while the subsystem i Enablement requires representative measurements of convergence, RPC volume, serialized bytes, allocation, CPU, and tail latency across stable membership, churn, and partition recovery. Tree delivery can reduce RPC counts while increasing serialized bytes through forwarding and repair. Compare the original runtime, the updated explicitly disabled path, and the enabled path at the intended cluster sizes and publication rates, including skewed peer speeds and rolling upgrades. +### Measuring cluster-size scaling + +The evidence workflow pairs the same current binary with dissemination disabled and enabled. Reports retain raw totals and normalize RPCs, serialized bytes, allocations, and CPU by offered publication count. Actual live silo count, binary identity, workload, and resource settings must match within each pair. End-to-end convergence is reported alongside publication duration and the remaining convergence wait, making the controller's sequential publication work and 100 ms polling cadence visible. + +The opt-in `dissemination-scale` PR label or scale dispatch runs separate jobs at 8, 16, 32, 64, and 100 silo processes. Each size uses ten publication rounds and two order-reversed repetitions of the current off/on stable workload. The regular three-path 4/8-silo stable, churn, and partition checks remain in place. Remove the label after collecting a curve to return subsequent pushes to the small profile. + +The large curve uses a runtime processor-count override of one and `GCConserveMemory=9` uniformly across sizes and variants. All silo processes share the runner's physical CPUs. Artifacts record the effective processor count, GC mode, host memory, startup costs, per-node measurements, and paired results. A memory preflight reserves a planning allowance of 128 MiB per silo plus 1 GiB for the controller. These single-host results expose protocol scaling under shared CPU and memory; cross-machine capacity and production tail latency require a separately provisioned deployment. + ## Broadcast pumps and backpressure Each destination has one independent pump. Repeated notifications coalesce by namespace and key, high-priority namespaces bypass the coalescing window, and batches obey global item and byte bounds plus namespace payload bounds. An owner-wide managed admission gate combines the local broadcast limit with one active local broadcast attempt per destination. diff --git a/test/Dissemination.IntegrationHarness/Shared/Protocol.cs b/test/Dissemination.IntegrationHarness/Shared/Protocol.cs index 36eca6675ed..be57b5a9aaf 100644 --- a/test/Dissemination.IntegrationHarness/Shared/Protocol.cs +++ b/test/Dissemination.IntegrationHarness/Shared/Protocol.cs @@ -9,7 +9,9 @@ internal sealed record NodeConfiguration( int SiloPort, int ParentProcessId, bool Enabled, - bool FastRecovery = true); + bool FastRecovery = true, + int SiloProcessorCount = 0, + int GCConserveMemory = 0); internal sealed record Command( int Id, @@ -236,6 +238,8 @@ internal sealed record NodeSnapshot public required string[] ForwardingTargets { get; init; } public string[] TopologyMembers { get; init; } = []; public int Fanout { get; init; } + public int AntiEntropyPeerCount { get; init; } + public double AntiEntropyIntervalMilliseconds { get; init; } public required Dictionary Metrics { get; init; } public required ApplyEvidence[] Applies { get; init; } public long TransportBytesWritten { get; init; } @@ -248,6 +252,8 @@ internal sealed record NodeSnapshot public long ManagedHeapBytes { get; init; } public long WorkingSetBytes { get; init; } public long PrivateBytes { get; init; } + public int ProcessorCount { get; init; } + public bool ServerGC { get; init; } public int PendingControlCalls { get; init; } public int StartedControlCalls { get; init; } public int CancelledControlCalls { get; init; } diff --git a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs index 20bf4a4dc3d..5481a9212ec 100644 --- a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs +++ b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs @@ -76,6 +76,8 @@ public static NodeSnapshot Decorate(IServiceProvider services, NodeSnapshot snap ForwardingTargets = topology.ForwardingTreeTargets.Select(address => address.ToParsableString()).ToArray(), TopologyMembers = topology.Members.Select(address => address.ToParsableString()).ToArray(), Fanout = options.Overlay.GetFanOutFactor(topology.Members.Length), + AntiEntropyPeerCount = options.Overlay.AntiEntropyPeerCount, + AntiEntropyIntervalMilliseconds = options.Overlay.AntiEntropyInterval.TotalMilliseconds, TreeGate = services.GetService()?.Snapshot, }; } diff --git a/test/Dissemination.IntegrationHarness/Silo/Program.cs b/test/Dissemination.IntegrationHarness/Silo/Program.cs index 6e51e3218e2..9873c6fba0e 100644 --- a/test/Dissemination.IntegrationHarness/Silo/Program.cs +++ b/test/Dissemination.IntegrationHarness/Silo/Program.cs @@ -338,6 +338,8 @@ internal static NodeSnapshot Snapshot(IServiceProvider services, bool? detailed ManagedHeapBytes = GC.GetTotalMemory(forceFullCollection: false), WorkingSetBytes = process.WorkingSet64, PrivateBytes = process.PrivateMemorySize64, + ProcessorCount = Environment.ProcessorCount, + ServerGC = System.Runtime.GCSettings.IsServerGC, PendingControlCalls = services.GetRequiredService().Pending, StartedControlCalls = services.GetRequiredService().Started, CancelledControlCalls = services.GetRequiredService().Cancelled, diff --git a/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs b/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs index 8be6da8c4b3..2df171f06bd 100644 --- a/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs +++ b/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs @@ -11,6 +11,83 @@ namespace Orleans.Dissemination.IntegrationHarness; public sealed class MeasurementTests { + [Fact] + public void ScalingComparisonNormalizesEqualOfferedWorkAndReportsLatencyDistribution() + { + var environment = JsonSerializer.SerializeToElement(new { ProcessorCount = 4, SiloProcessorCount = 1, GCConserveMemory = 9 }); + var disabled = new ScalingSample( + "CurrentDisabled", 100, 100, "stable", 0, 4, 400, + new("candidate", "same-binary"), environment, + 39600, 800000, 4000000, 2000, 1000000, [40, 10, 30, 20]); + var enabled = disabled with + { + RuntimePath = "CurrentEnabledSupported", + TotalRpcs = 4000, + SerializedBytesSent = 400000, + AllocatedBytes = 2000000, + CpuMilliseconds = 1000, + ConvergenceMilliseconds = [30, 10, 20, 10], + }; + + var result = ScalingComparison.Create(JsonSerializer.Serialize(new[] { enabled, disabled })); + + Assert.Equal(0, result.IncompletePairs); + var pair = Assert.Single(result.Pairs); + Assert.Equal(100, pair.Size); + Assert.Equal(400, pair.OfferedPublications); + Assert.Equal(99, pair.Disabled.RpcsPerPublication); + Assert.Equal(10, pair.Enabled.RpcsPerPublication); + Assert.Equal(2000, pair.Disabled.SerializedBytesPerPublication); + Assert.Equal(1000, pair.Enabled.SerializedBytesPerPublication); + Assert.Equal(10000, pair.Disabled.AllocatedBytesPerPublication); + Assert.Equal(5000, pair.Enabled.AllocatedBytesPerPublication); + Assert.Equal(5, pair.Disabled.CpuMillisecondsPerPublication); + Assert.Equal(2.5, pair.Enabled.CpuMillisecondsPerPublication); + Assert.Equal(25, pair.Disabled.MedianConvergenceMilliseconds); + Assert.Equal(15, pair.Enabled.MedianConvergenceMilliseconds); + Assert.Equal(40, pair.Disabled.P95ConvergenceMilliseconds); + Assert.Equal(30, pair.Enabled.P95ConvergenceMilliseconds); + Assert.Contains("99.00 -> 10.00", result.ToMarkdown(), StringComparison.Ordinal); + } + + [Theory] + [InlineData("binary")] + [InlineData("workload")] + [InlineData("environment")] + [InlineData("silo-count")] + public void ScalingComparisonRejectsIncomparableMeasurements(string mismatch) + { + var sample = new ScalingSample( + "CurrentDisabled", 100, 100, "stable", 0, 3, 300, + new("candidate", "same-binary"), JsonSerializer.SerializeToElement(new { ProcessorCount = 4 }), + 100, 1000, 10000, 100, 1000, [1, 2, 3]); + var enabled = sample with { RuntimePath = "CurrentEnabledSupported" }; + enabled = mismatch switch + { + "binary" => enabled with { Runtime = new("different", "other-binary") }, + "workload" => enabled with { OfferedPublications = 301 }, + "environment" => enabled with { Environment = JsonSerializer.SerializeToElement(new { ProcessorCount = 1 }) }, + "silo-count" => enabled with { LiveSilos = 99 }, + _ => throw new InvalidOperationException(mismatch), + }; + + Assert.Throws(() => ScalingComparison.Create(JsonSerializer.Serialize(new[] { sample, enabled }))); + } + + [Fact] + public void ScalingComparisonReportsPartialPairsExplicitly() + { + var sample = new ScalingSample( + "CurrentDisabled", 8, 8, "stable", 0, 3, 24, + new("candidate", "same-binary"), JsonSerializer.SerializeToElement(new { ProcessorCount = 4 }), + 100, 1000, 10000, 100, 1000, [1, 2, 3]); + + var result = ScalingComparison.Create(JsonSerializer.Serialize(new[] { sample })); + + Assert.Empty(result.Pairs); + Assert.Equal(1, result.IncompletePairs); + } + [Fact] public void LoadStateComparisonRequiresExactInventoryAndValues() { diff --git a/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs b/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs index 8f689b7ad2d..0ca8f4d622b 100644 --- a/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs +++ b/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs @@ -23,7 +23,7 @@ internal sealed class ProcessCluster : IAsyncDisposable private readonly string _membershipDirectory; private int _sequence; - public ProcessCluster(string scenario, bool fastRecovery = true) + public ProcessCluster(string scenario, bool fastRecovery = true, int siloProcessorCount = 0, int gcConserveMemory = 0) { var artifactRoot = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RESULTS") ?? throw new InvalidOperationException("Run .github/scripts/dissemination-evidence.ps1 first, or set ORLEANS_DISSEMINATION_{OLD,NEW,RESULTS}."); @@ -32,12 +32,20 @@ public ProcessCluster(string scenario, bool fastRecovery = true) _membershipDirectory = Path.Combine(Directory, "membership"); System.IO.Directory.CreateDirectory(_membershipDirectory); FastRecovery = fastRecovery; + Assert.InRange(siloProcessorCount, 0, 64); + Assert.InRange(gcConserveMemory, 0, 9); + SiloProcessorCount = siloProcessorCount; + GCConserveMemory = gcConserveMemory; } public string Directory { get; } public bool FastRecovery { get; } + public int SiloProcessorCount { get; } + public int GCConserveMemory { get; } public IReadOnlyList All => _all; public SiloProcess[] Active => _all.Where(node => !node.Stopped).ToArray(); + public double LastPublicationMilliseconds { get; private set; } + public double LastConvergenceWaitMilliseconds { get; private set; } public async Task Start(string runtime, bool enabled, int? port = null) { @@ -57,7 +65,8 @@ public async Task Start(string runtime, bool enabled, int? port = n var name = $"{runtime}-{_sequence++}-{(enabled ? "enabled" : "default")}"; var node = new SiloProcess( binaryDirectory, - new(name, Path.GetFileName(Directory), _membershipDirectory, port ?? AvailablePort(), Environment.ProcessId, enabled, FastRecovery), + new(name, Path.GetFileName(Directory), _membershipDirectory, port ?? AvailablePort(), Environment.ProcessId, + enabled, FastRecovery, SiloProcessorCount, GCConserveMemory), Directory, manifest); _all.Add(node); @@ -104,12 +113,16 @@ public async Task PublishAndConverge() expected.Add(snapshot.Address, snapshot.Load[snapshot.Address]); } + var publicationMilliseconds = clock.Elapsed.TotalMilliseconds; await Eventually("exact production load state at every active process", async () => { var snapshots = await Task.WhenAll(Active.Select(node => node.Send("snapshot"))); return snapshots.All(snapshot => MatchesExpectedLoad(snapshot.Load, expected)); }, expected: expected); - return clock.Elapsed.TotalMilliseconds; + var elapsed = clock.Elapsed.TotalMilliseconds; + LastPublicationMilliseconds = publicationMilliseconds; + LastConvergenceWaitMilliseconds = elapsed - publicationMilliseconds; + return elapsed; } internal static bool MatchesExpectedLoad( @@ -260,6 +273,17 @@ public SiloProcess(string binaryDirectory, NodeConfiguration configuration, stri }, }; _pending[0] = _ready; + if (configuration.SiloProcessorCount > 0) + { + _process.StartInfo.Environment["DOTNET_PROCESSOR_COUNT"] = + configuration.SiloProcessorCount.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + + if (configuration.GCConserveMemory > 0) + { + _process.StartInfo.Environment["DOTNET_GCConserveMemory"] = + configuration.GCConserveMemory.ToString(System.Globalization.CultureInfo.InvariantCulture); + } } public NodeConfiguration Configuration { get; } @@ -290,6 +314,10 @@ public async Task Start() Assert.Equal(_manifest.Runtime == "New", Last.Identity.HasDissemination); Assert.Equal(Configuration.Enabled, Last.Enabled); Assert.Equal(Configuration.Enabled, Last.NamespaceEnabled); + if (Configuration.SiloProcessorCount > 0) + { + Assert.Equal(Configuration.SiloProcessorCount, Last.ProcessorCount); + } var actualDirectory = Path.GetDirectoryName(Path.GetFullPath(Last.Identity.AssemblyPath)); Assert.Equal(Path.GetFullPath(_process.StartInfo.WorkingDirectory), actualDirectory); foreach (var name in new[] { "Orleans.Runtime", "Orleans.Core", "Orleans.Core.Abstractions", "Orleans.Serialization" }) diff --git a/test/Dissemination.IntegrationHarness/Tests/ScalingComparison.cs b/test/Dissemination.IntegrationHarness/Tests/ScalingComparison.cs new file mode 100644 index 00000000000..2e45e1e63a2 --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Tests/ScalingComparison.cs @@ -0,0 +1,106 @@ +using System.Globalization; +using System.Text; +using System.Text.Json; + +namespace Orleans.Dissemination.IntegrationHarness; + +internal sealed record ScalingComparison(ScalingPair[] Pairs, int IncompletePairs) +{ + public static ScalingComparison Create(string json) + { + var samples = JsonSerializer.Deserialize(json) + ?? throw new InvalidOperationException("Missing scaling samples."); + var pairs = new List(); + var incomplete = 0; + foreach (var group in samples + .Where(sample => sample.RuntimePath is "CurrentDisabled" or "CurrentEnabledSupported") + .GroupBy(sample => (sample.Size, sample.Scenario, sample.Repetition))) + { + if (group.Count() == 1) + { + incomplete++; + continue; + } + + var disabled = group.Single(sample => sample.RuntimePath == "CurrentDisabled"); + var enabled = group.Single(sample => sample.RuntimePath == "CurrentEnabledSupported"); + Validate(disabled); + Validate(enabled); + if (disabled.Iterations != enabled.Iterations + || disabled.OfferedPublications != enabled.OfferedPublications + || disabled.Runtime.SourceRevision != enabled.Runtime.SourceRevision + || disabled.Runtime.Sha256 != enabled.Runtime.Sha256 + || disabled.Environment.GetRawText() != enabled.Environment.GetRawText()) + { + throw new InvalidOperationException($"Mismatched binary, workload, or environment for {group.Key}."); + } + + pairs.Add(new( + disabled.Size, disabled.Scenario, disabled.Repetition, disabled.Iterations, disabled.OfferedPublications, + disabled.Runtime.SourceRevision, disabled.Runtime.Sha256, disabled.Environment, + ScalingCost.From(disabled), ScalingCost.From(enabled))); + } + + return new(pairs.ToArray(), incomplete); + } + + private static void Validate(ScalingSample sample) + { + if (sample.Size < 3 || sample.LiveSilos != sample.Size || sample.OfferedPublications <= 0 + || sample.Runtime is null || string.IsNullOrEmpty(sample.Runtime.SourceRevision) || string.IsNullOrEmpty(sample.Runtime.Sha256) + || sample.Environment.ValueKind != JsonValueKind.Object + || sample.Iterations < 3 || sample.ConvergenceMilliseconds is null || sample.ConvergenceMilliseconds.Length != sample.Iterations + || sample.TotalRpcs <= 0 || sample.SerializedBytesSent <= 0 || sample.AllocatedBytes <= 0 + || sample.CpuMilliseconds < 0 || sample.ConvergenceMilliseconds.Any(value => !double.IsFinite(value) || value < 0)) + { + throw new InvalidOperationException($"Incomplete scaling measurement for {sample.RuntimePath}, {sample.Size} silos."); + } + } + + public string ToMarkdown() + { + var result = new StringBuilder(); + result.AppendLine("### Dissemination disabled vs enabled"); + result.AppendLine("Each cell is disabled -> enabled. Costs per offered publication include background and control traffic. Single-host, resource-shared measurements; raw totals, environment and per-node data are in the artifacts."); + result.AppendLine(); + result.AppendLine("| Silos / scenario / repetition | RPCs / publication | KiB / publication | Allocated KiB / publication | CPU ms / publication | Median ms | P95 ms |"); + result.AppendLine("|---|---:|---:|---:|---:|---:|---:|"); + foreach (var pair in Pairs) + { + result.AppendLine(FormattableString.Invariant( + $"| {pair.Size} / {pair.Scenario} / {pair.Repetition} | {pair.Disabled.RpcsPerPublication:F2} -> {pair.Enabled.RpcsPerPublication:F2} | {pair.Disabled.SerializedBytesPerPublication / 1024:F2} -> {pair.Enabled.SerializedBytesPerPublication / 1024:F2} | {pair.Disabled.AllocatedBytesPerPublication / 1024:F2} -> {pair.Enabled.AllocatedBytesPerPublication / 1024:F2} | {pair.Disabled.CpuMillisecondsPerPublication:F2} -> {pair.Enabled.CpuMillisecondsPerPublication:F2} | {pair.Disabled.MedianConvergenceMilliseconds:F2} -> {pair.Enabled.MedianConvergenceMilliseconds:F2} | {pair.Disabled.P95ConvergenceMilliseconds:F2} -> {pair.Enabled.P95ConvergenceMilliseconds:F2} |")); + } + + result.AppendLine(); + result.AppendLine($"Incomplete pairs: {IncompletePairs.ToString(CultureInfo.InvariantCulture)}."); + return result.ToString(); + } +} + +internal sealed record ScalingSample( + string RuntimePath, int Size, int LiveSilos, string Scenario, int Repetition, int Iterations, int OfferedPublications, + ScalingBinary Runtime, JsonElement Environment, long TotalRpcs, double SerializedBytesSent, + long AllocatedBytes, double CpuMilliseconds, long RetainedManagedBytes, double[] ConvergenceMilliseconds); + +internal sealed record ScalingBinary(string SourceRevision, string Sha256); + +internal sealed record ScalingPair( + int Size, string Scenario, int Repetition, int Iterations, int OfferedPublications, + string SourceRevision, string Sha256, JsonElement Environment, ScalingCost Disabled, ScalingCost Enabled); + +internal sealed record ScalingCost( + long TotalRpcs, double SerializedBytesSent, long AllocatedBytes, double CpuMilliseconds, long RetainedManagedBytes, + double RpcsPerPublication, double SerializedBytesPerPublication, double AllocatedBytesPerPublication, + double CpuMillisecondsPerPublication, double MedianConvergenceMilliseconds, double P95ConvergenceMilliseconds) +{ + public static ScalingCost From(ScalingSample sample) + { + var ordered = sample.ConvergenceMilliseconds.Order().ToArray(); + var median = (ordered[(ordered.Length - 1) / 2] + ordered[ordered.Length / 2]) / 2; + return new( + sample.TotalRpcs, sample.SerializedBytesSent, sample.AllocatedBytes, sample.CpuMilliseconds, sample.RetainedManagedBytes, + (double)sample.TotalRpcs / sample.OfferedPublications, sample.SerializedBytesSent / sample.OfferedPublications, + (double)sample.AllocatedBytes / sample.OfferedPublications, sample.CpuMilliseconds / sample.OfferedPublications, + median, ordered[(int)Math.Ceiling(0.95 * ordered.Length) - 1]); + } +} diff --git a/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs b/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs index 2a59e7e3a9a..ff62a1d41b6 100644 --- a/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs +++ b/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs @@ -9,14 +9,21 @@ public sealed class ScalingTests { [Fact] [Trait("Category", "DisseminationScale")] - public async Task Scaling_OriginalLegacy_Disabled_EnabledSupported() + public async Task Scaling_SelectedRuntimePathsConverge() { var sizes = (Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_SIZES") ?? "4,8,16") .Split(',').Select(value => int.Parse(value, System.Globalization.CultureInfo.InvariantCulture)).ToArray(); var iterations = Setting("ITERATIONS", 10, 3, 200); var repetitions = Setting("REPETITIONS", 1, 1, 5); - Assert.All(sizes, size => Assert.InRange(size, 3, 32)); + Assert.All(sizes, size => Assert.InRange(size, 3, 128)); Assert.InRange(sizes.Length, 1, 6); + Assert.Equal(sizes.Length, sizes.Distinct().Count()); + var scenarios = (Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_SCENARIOS") ?? "stable,churn,partition").Split(','); + Assert.InRange(scenarios.Length, 1, 3); + Assert.Equal(scenarios.Length, scenarios.Distinct().Count()); + Assert.All(scenarios, scenario => Assert.Contains(scenario, new[] { "stable", "churn", "partition" })); + var paths = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RUNTIME_PATHS") ?? "All"; + Assert.Contains(paths, new[] { "All", "Current" }); var records = new List(); var output = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RESULTS")!; @@ -26,31 +33,50 @@ public async Task Scaling_OriginalLegacy_Disabled_EnabledSupported() new("CurrentDisabled", "New", Enabled: false), new("CurrentEnabledSupported", "New", Enabled: true), ]; + if (paths == "Current") + { + variants = variants[1..]; + } + foreach (var size in sizes) { - foreach (var scenario in new[] { "stable", "churn", "partition" }) + foreach (var scenario in scenarios) { for (var repetition = 0; repetition < repetitions; repetition++) { - // Rotate all three paths to expose thermal/cache/order effects across repetitions. + // Rotate paths to expose order effects across repetitions. for (var order = 0; order < variants.Length; order++) { var variant = variants[(repetition + order) % variants.Length]; var record = await Measure(size, scenario, variant, iterations, repetition, order); records.Add(record); - await File.WriteAllTextAsync(Path.Combine(output, "cost-results.json"), - JsonSerializer.Serialize(records, new JsonSerializerOptions { WriteIndented = true }), + var json = JsonSerializer.Serialize(records, new JsonSerializerOptions { WriteIndented = true }); + await File.WriteAllTextAsync(Path.Combine(output, "cost-results.json"), json, TestContext.Current.CancellationToken); + await File.WriteAllTextAsync(Path.Combine(output, "comparisons.json"), + JsonSerializer.Serialize(ScalingComparison.Create(json), new JsonSerializerOptions { WriteIndented = true }), TestContext.Current.CancellationToken); } } } } + + var report = ScalingComparison.Create(JsonSerializer.Serialize(records)); + Assert.Equal(0, report.IncompletePairs); + Assert.Equal(sizes.Length * scenarios.Length * repetitions, report.Pairs.Length); + if (Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY") is { Length: > 0 } summary) + { + await File.AppendAllTextAsync(summary, report.ToMarkdown(), TestContext.Current.CancellationToken); + } } private static async Task Measure(int size, string scenario, RuntimeVariant variant, int iterations, int repetition, int order) { var enabled = variant.Enabled; - await using var cluster = new ProcessCluster($"scale-{size}-{scenario}-{variant.Name}-{repetition}", fastRecovery: false); + var siloProcessorCount = Setting("SILO_PROCESSOR_COUNT", 0, 0, 64); + var gcConserveMemory = Setting("GC_CONSERVE_MEMORY", 0, 0, 9); + var bootstrap = Stopwatch.StartNew(); + await using var cluster = new ProcessCluster( + $"scale-{size}-{scenario}-{variant.Name}-{repetition}", fastRecovery: false, siloProcessorCount, gcConserveMemory); for (var index = 0; index < size; index++) { await cluster.Start(variant.Runtime, enabled); @@ -70,9 +96,14 @@ private static async Task Measure(int size, string scenario, RuntimeVari await Task.Delay(TimeSpan.FromSeconds(2), TestContext.Current.CancellationToken); var start = (await Task.WhenAll(cluster.Active.Select(node => node.Send("measure")))) .ToDictionary(snapshot => snapshot.Identity.ProcessId); + Assert.Equal(size, start.Count); + Assert.All(start.Values, snapshot => Assert.Equal(size, snapshot.ActiveMembers.Length)); + var bootstrapMilliseconds = bootstrap.Elapsed.TotalMilliseconds; var startedAt = DateTimeOffset.UtcNow; var clock = Stopwatch.StartNew(); var latencies = new List(); + var publicationLatencies = new List(); + var convergenceWaits = new List(); var recoveryLatencies = new List(); var offeredPublications = 0; var restarts = 0; @@ -108,13 +139,19 @@ private static async Task Measure(int size, string scenario, RuntimeVari var recovery = Stopwatch.StartNew(); // Establish delivery paths before offering the next sample, including on the one-way legacy path. await cluster.HealPartition(victim); - latencies.Add(await cluster.PublishAndConverge()); - offeredPublications += size; + await PublishRound(); recoveryLatencies.Add(recovery.Elapsed.TotalMilliseconds); continue; } + await PublishRound(); + } + + async Task PublishRound() + { latencies.Add(await cluster.PublishAndConverge()); + publicationLatencies.Add(cluster.LastPublicationMilliseconds); + convergenceWaits.Add(cluster.LastConvergenceWaitMilliseconds); offeredPublications += size; } @@ -197,6 +234,7 @@ private static async Task Measure(int size, string scenario, RuntimeVari RuntimePath = variant.Name, Runtime = finish[0].Identity, Size = size, + LiveSilos = start.Count, Scenario = scenario, Enabled = enabled, Iterations = iterations, @@ -209,6 +247,8 @@ private static async Task Measure(int size, string scenario, RuntimeVari WindowStartedAtUtc = startedAt, WindowFinishedAtUtc = finishedAt, ConvergenceMilliseconds = latencies, + PublicationMilliseconds = publicationLatencies, + ConvergenceAfterPublicationMilliseconds = convergenceWaits, PartitionRecoveryMilliseconds = recoveryLatencies, TotalRpcs = nodes.Sum(node => node.Rpcs), TotalMessages = nodes.Sum(node => node.SentMessages), @@ -218,8 +258,20 @@ private static async Task Measure(int size, string scenario, RuntimeVari CpuMilliseconds = nodes.Sum(node => node.CpuMilliseconds), AllocatedBytes = nodes.Sum(node => node.AllocatedBytes), RetainedManagedBytes = nodes.Sum(node => node.RetainedManagedBytes), + BootstrapMilliseconds = bootstrapMilliseconds, + BootstrapCpuMilliseconds = start.Values.Sum(snapshot => snapshot.CpuMilliseconds), + BootstrapAllocatedBytes = start.Values.Sum(snapshot => snapshot.AllocatedBytes), Nodes = nodes, - Environment = new { RuntimeInformation.OSDescription, RuntimeInformation.FrameworkDescription, System.Environment.ProcessorCount }, + Environment = new + { + RuntimeInformation.OSDescription, + RuntimeInformation.FrameworkDescription, + System.Environment.ProcessorCount, + SiloProcessorCount = siloProcessorCount, + GCConserveMemory = gcConserveMemory, + WorkerProcessorCounts = start.Values.Select(snapshot => snapshot.ProcessorCount).Distinct().Order().ToArray(), + WorkerServerGC = start.Values.Select(snapshot => snapshot.ServerGC).Distinct().Order().ToArray(), + }, Methodology = new { WarmupRounds = 2, @@ -238,7 +290,7 @@ private static async Task Measure(int size, string scenario, RuntimeVari TransportBytes = "bytes advanced at the real connection transport pipe, including handshake; may include buffered bytes lost when a partition aborts a connection", RetainedMemory = "full GC live managed bytes on surviving nodes only; not a peak; RSS/private bytes and retired-node snapshots also retained", Includes = "all silo work during the window, including control/snapshot/instrumentation overhead and legacy fallback; restart startup/shutdown costs included for churn", - Observation = "state polling on all three paths; detailed counters only at window boundaries; per-value DiagnosticListener capture disabled in scale runs", + Observation = "identical state polling on selected paths; detailed counters only at window boundaries; per-value DiagnosticListener capture disabled in scale runs", Topology = "per-node actual tree at end, or start for a retired node; raw command snapshots preserve intervening topology changes; role is not time-weighted", Excludes = "controller process CPU/allocation, storage disk byte accounting, cross-machine latency, NIC/TCP wire overhead, production storage/placement/grain work", PartitionComparison = "retire partitioned connections at both endpoints, then confirm healed connections with bidirectional acknowledged control RPCs before the same single publication round on all paths; teardown and readiness remain inside partition recovery time and total cost; silent anti-entropy recovery is asserted separately in VersionSkewTests", @@ -252,12 +304,28 @@ private static async Task Measure(int size, string scenario, RuntimeVari return record; } - private static Task ConfirmSupport(ProcessCluster cluster) => - cluster.Eventually("all enabled peers confirm support before offered publication rounds", async () => + private static Task ConfirmSupport(ProcessCluster cluster) + { + var timeout = TimeSpan.FromSeconds(90); + foreach (var node in cluster.Active) + { + Assert.True(node.Last.AntiEntropyPeerCount > 0); + Assert.True(node.Last.AntiEntropyIntervalMilliseconds > 0); + var rounds = Math.Ceiling((cluster.Active.Length - 1.0) / node.Last.AntiEntropyPeerCount); + var rotationBudget = TimeSpan.FromMilliseconds(2 * rounds * node.Last.AntiEntropyIntervalMilliseconds) + + TimeSpan.FromSeconds(60); + if (rotationBudget > timeout) + { + timeout = rotationBudget; + } + } + + return cluster.Eventually("all enabled peers confirm support before offered publication rounds", async () => { var snapshots = await Task.WhenAll(cluster.Active.Select(node => node.Send("snapshot"))); return snapshots.All(snapshot => snapshot.UnconfirmedPeers.Length == 0); - }, TimeSpan.FromSeconds(90)); + }, timeout); + } private sealed record RuntimeVariant(string Name, string Runtime, bool Enabled); diff --git a/test/Dissemination.IntegrationHarness/methodology.json b/test/Dissemination.IntegrationHarness/methodology.json index 285635a3acc..3750c7dccc3 100644 --- a/test/Dissemination.IntegrationHarness/methodology.json +++ b/test/Dissemination.IntegrationHarness/methodology.json @@ -4,7 +4,7 @@ "entrypoint": ".github/scripts/dissemination-evidence.ps1", "workflow": ".github/workflows/dissemination-evidence.yml", "automaticPrTrigger": "pull_request and merge_group run both the pinned-binary version-skew gate and measured bounded original-legacy, current-explicitly-disabled and current-enabled-supported costs. The bounded job reuses published binaries, needs no label or workflow_dispatch registration, and has a 25-minute hard limit.", - "controlledScaleTrigger": "Larger runs require workflow_dispatch with scale=true after the workflow is registered on the default branch. No pull_request_target or write permission.", + "controlledScaleTrigger": "The dissemination-scale PR label or workflow_dispatch with scale=true runs an independent job per size, defaulting to 8,16,32,64,100 real silo processes. Remove the label after collecting evidence to keep subsequent pushes on the small profile. Jobs use read-only permissions.", "sliceInventory": [ ".github/scripts/dissemination-evidence.ps1", ".github/workflows/dissemination-evidence.yml", @@ -20,6 +20,7 @@ "test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs", "test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs", "test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs", + "test/Dissemination.IntegrationHarness/Tests/ScalingComparison.cs", "test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs" ], "runtimeDependencies": { @@ -105,7 +106,7 @@ } ], "load": { - "test": "Scaling_OriginalLegacy_Disabled_EnabledSupported", + "test": "Scaling_SelectedRuntimePathsConverge", "comparison": "Three separately labeled paths: pinned original legacy binary; actual current binary with dissemination and both namespaces explicitly disabled; the same current binary using temporarily enabled defaults after peers have confirmed support. Enabled defaults provide broader pre-merge test exposure.", "runtimePaths": [ { "name": "OriginalLegacy", "runtime": "Old", "enabled": false }, @@ -113,17 +114,21 @@ { "name": "CurrentEnabledSupported", "runtime": "New", "enabled": true } ], "automaticPrProfile": { "mode": "Cost", "sizes": [4, 8], "iterations": 3, "pairedRepetitions": 1, "overrideable": false }, - "manualDefaults": { "mode": "Scale", "sizes": [4, 8, 16], "iterations": 10, "pairedRepetitions": 1 }, - "controls": { "sizes": "1..6 comma-separated sizes, each 3..32", "iterations": "3..200", "pairedRepetitions": "1..5" }, + "largeScaleDefaults": { "mode": "Scale", "sizes": [8, 16, 32, 64, 100], "iterations": 10, "pairedRepetitions": 2, "runtimePaths": "Current", "scenarios": ["stable"], "siloProcessorCount": 1, "gcConserveMemory": 9 }, + "controls": { "sizes": "1..6 distinct comma-separated sizes, each 3..128", "iterations": "3..200", "pairedRepetitions": "1..5", "runtimePaths": "All or Current", "scenarios": "distinct stable, churn, partition selections" }, + "largeScaleResources": "All sizes in the large curve use the same one-logical-processor and GCConserveMemory=9 child-process policy on standard shared-host CI capacity. Effective worker processor count and server-GC mode are recorded and processor count is asserted. Linux preflight requires available memory of 128 MiB per requested silo plus 1 GiB controller headroom, and saves host-resources.json. This is a planning guard, not a memory cap or proof against resource exhaustion. An inadequate runner fails before starting the measurement.", + "largeScaleScope": "Large jobs compare the identical current binary with dissemination explicitly off versus enabled defaults, with order reversed on the second repetition. The ordinary 4/8-silo three-path stable/churn/partition profile remains unchanged. Script options support additional scenarios on a suitably provisioned host.", + "comparisonReports": "comparisons.json and GitHub job summaries pair equal-size/scenario/repetition current-off/current-on samples. Pair validation checks actual live silo count, identical binary hash/revision, environment, iterations and offered publications. Reports include raw totals, costs per publication, median/P95 convergence, retained managed memory, and explicit incomplete-pair counts.", "scenarios": "Stable; graceful stop/restart at the same endpoint every other round; real connection partition and heal with old pooled connections retired at both endpoints before reopening and confirming transport readiness via bidirectional acknowledged control RPCs. Teardown, readiness probes and retries share a 60-second deadline and remain inside recovery latency and total cost. Each path then offers the same single publication round. Gate artifacts separately cover silent repair, retained-history misses and same-version heartbeat recovery.", "cadence": "Pause automatic deployment-load timers after startup. Invoke the actual production PublishStatistics method once per active node per offered round, then require exact dictionary inventory and values using cached JsonSerializerOptions with IncludeFields=true, including every public readonly EnvironmentStatistics field. Extra departed or restarted generations fail the same comparison as missing or mismatched values.", - "window": "Two warmup publication rounds and complete enabled-peer capability discovery before measuring. Replacements also finish capability discovery before offered rounds; their bootstrap/fallback/discovery costs remain inside the churn window. Two-second OS-counter sampling margins on all paths. Full-GC retained-memory probes are after the CPU/allocation window.", + "window": "Two warmup publication rounds and complete enabled-peer capability discovery before measuring. Discovery has a minimum 90-second controller budget, expanded to two full peer rotations plus 60 seconds using the actual advertised repair cadence and peer count. Production repair settings stay unchanged. Startup/warmup/preparation duration, CPU and allocations are reported separately. Replacements also finish capability discovery before offered rounds; their bootstrap/fallback/discovery costs remain inside the churn window. Two-second OS-counter sampling margins on all paths. Full-GC retained-memory probes are after the CPU/allocation window.", "metrics": [ "All transmitted request AND one-way RPC messages; responses reported in the separate total-message count", "Complete serialized Orleans message bytes, including headers/body, from message-size observations", "Actual process-wide System.Net.Sockets sent/received bytes, sampled each second; transport-pipe submitted/read bytes separately", "Process CPU time, GC total allocated bytes, post-full-GC retained managed bytes, working set and private bytes", "Per-round exact state-convergence latency and partition recovery latency", + "Separate per-round publication duration and residual convergence wait, retaining the original end-to-end latency and 100 ms polling cadence", "Per-node counters, actual originator/forwarding targets, tree rank and upper-tree role; raw snapshots preserve topology changes", "Raw protocol payload, namespace-batch and anti-entropy counters, never substituted for total RPC or byte counts" ], @@ -144,7 +149,7 @@ "siloCommandSeconds": 60, "controllerCommandSeconds": 70, "convergenceSeconds": 60, - "scaleCapabilityDiscoverySeconds": 90, + "scaleCapabilityDiscovery": "max(90 seconds, two complete peer rotations at actual repair cadence plus 60 seconds)", "membershipLockSeconds": 10, "shutdownSeconds": 35, "individualProcessLifetimeMinutes": 45, @@ -159,6 +164,6 @@ "gate": "dissemination-gate-evidence: TRX, identity/probe snapshots, isolated repair checkpoints, bounded shutdown outcomes, raw logs, methodology and binlogs", "binaries": "dissemination-binaries: pinned old/current/runner publish directories plus SHA manifests for explicit replay using -SkipBuild", "automaticCost": "dissemination-bounded-cost-evidence: cost-results.json and per-window/raw artifacts for the fixed automatic PR profile, including actual restart and partition counts", - "scale": "dissemination-scale-evidence: cost-results.json, per-window cost.json, raw start/finish/warmup/node snapshots and partial results on failure" + "scale": "dissemination-scale--evidence: cost-results.json, comparisons.json, host-resources.json, per-window cost.json, raw start/finish/warmup/node snapshots and partial results on failure" } } From 36ccad1558642ccadf23d73bf3f992904bd3c190 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 16:50:52 -0700 Subject: [PATCH 22/50] perf(dissemination): compact acknowledgments and reuse pruning inventory --- .../implementation/runtime-dissemination.md | 4 +- .../IDisseminationSystemTarget.cs | 7 + .../DisseminationBroadcastQueue.cs | 97 ++++-- .../Dissemination/DisseminationProtocol.cs | 54 +++- ...ionProtocolTests.CompactAcknowledgments.cs | 277 ++++++++++++++++++ ...sseminationProtocolTests.ProtocolReview.cs | 19 +- 6 files changed, 410 insertions(+), 48 deletions(-) create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index 4d2d269541e..ed1586bd370 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -49,11 +49,13 @@ A `DisseminationValue` names one key and a `[FromVersion, ToVersion]` transition The namespace reports whether a repair is current, produced, unavailable, or unable to fit the supplied item and byte budgets. Produced repairs can be complete or a valid prefix. Prefix acknowledgments establish the receiver's new baseline and let the next batch continue the chain. +Broadcast senders advertise support for compact acknowledgments. When the entire batch fits the receiver's limits and each key's current version equals the maximum version transmitted for that key, one response flag certifies those versions. Namespace entries retain capability evidence with empty per-key lists. The sender advances each ledger to its captured transmitted version, including when that batch is only a prefix of a larger repair. Receivers report explicit versions for older callers, truncated batches, and differing receiver versions, preserving precise repair baselines across version skew. + Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. It retains 32 snapshots, prefers a smaller diff when the peer baseline is present, and falls back to a full snapshot when history or capacity makes the diff unsuitable. Current diffs identify their entries as the complete target inventory, so table cleanup is reflected even when a receiver retained a different same-version baseline. Older partial-diff payloads retain their incremental semantics. Same-version snapshots can advance liveness or remove inactive entries while preserving every Active entry and each retained status. This allows cleanup to converge while heartbeats are unchanged, and an advancing heartbeat preserves those same membership guards. The membership manager merges maximum per-entry liveness and retains its local-death handling; dissemination reports whether that manager actually published changed state. Deployment-load versions are sample timestamps. Each repair is a full latest value. Application runs on the deployment-load publisher's scheduler and accepts samples only for Active silo generations in the membership oracle. Terminal membership transitions remove placement statistics, and the oracle's monotonic membership state rejects later samples for departed generations. -After the owner accepts a load sample, its serialized payload is reused for forwarding and repair. Inventory maintenance enumerates namespace keys directly; membership exposes its single key, and deployment load uses the active-silo oracle. Version and fingerprint construction is reserved for operations which use that information. Receive batches which fit all item, byte, and namespace limits are processed directly from their existing collections; truncated batches retain bounded cursor selection. +After the owner accepts a load sample, its serialized payload is reused for forwarding and repair. Inventory maintenance enumerates namespace keys directly; membership exposes its single key, and deployment load uses the active-silo oracle. Each broadcast queue retains one reusable scratch inventory; individual pruning calls have exclusive ownership and clear all keys before returning it, including on failure. Version and fingerprint construction is reserved for operations which use that information. Receive batches which fit all item, byte, and namespace limits are processed directly from their existing collections; truncated batches retain bounded cursor selection. ## Deterministic topology diff --git a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs index 4b4a465e734..caafcc97ca5 100644 --- a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs +++ b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs @@ -238,6 +238,9 @@ internal sealed class DisseminationBroadcastBatch [Id(1)] public Dictionary> Values { get; init; } = []; + + [Id(2)] + public bool SupportsCompactAcknowledgments { get; init; } } // Acknowledgments report the versions the receiver actually holds after processing the batch. @@ -249,6 +252,10 @@ internal sealed class DisseminationBroadcastResponse [Id(1)] public List UnsupportedNamespaces { get; init; } = []; + + // Certifies the maximum transmitted version of every key. Namespace entries retain capability evidence. + [Id(2)] + public bool AllVersionsAcknowledged { get; init; } } [GenerateSerializer, Immutable] diff --git a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs index a0cd6c2438b..e5a7397431b 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs @@ -1,6 +1,7 @@ using System.Collections.Frozen; using System.Runtime.InteropServices; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.ObjectPool; using Microsoft.Extensions.Options; using Orleans.Configuration; using Orleans.Runtime.Internal; @@ -24,6 +25,7 @@ internal sealed partial class DisseminationBroadcastQueue private readonly object _lock = new(); private readonly Dictionary _peers = []; private readonly DisseminationSendGate _sendGate; + private readonly ObjectPool>> _inventoryPool; private bool _stopped; public DisseminationBroadcastQueue( @@ -44,6 +46,8 @@ public DisseminationBroadcastQueue( _logger = logger; _responseObserver = responseObserver; _sendGate = new(Math.Max(1, options.CurrentValue.MaxConcurrentSends)); + _inventoryPool = new DefaultObjectPool>>( + new InventoryPoolPolicy(_disseminationNamespaces), maximumRetained: 1); } public bool Notify( @@ -134,49 +138,81 @@ public async Task Prune( CancellationToken cancellationToken) { // Namespace identities are the authoritative inventory for retiring clean ledger entries. - var activeKeys = new Dictionary>(); - foreach (var disseminationNamespace in _disseminationNamespaces) + var activeKeys = _inventoryPool.Get(); + try { - if (disseminationNamespace.Options.Enabled) + foreach (var disseminationNamespace in _disseminationNamespaces) { - activeKeys[disseminationNamespace.Name] = disseminationNamespace.Keys.ToHashSet(); + if (disseminationNamespace.Options.Enabled) + { + activeKeys[disseminationNamespace.Name].UnionWith(disseminationNamespace.Keys); + } } - } - List? removedPeers = null; - List retainedPeers; - lock (_lock) - { - retainedPeers = new(_peers.Count); - foreach (var (peer, pending) in _peers) + List? removedPeers = null; + List retainedPeers; + lock (_lock) { - if (!_localSilo.Equals(peer) && !membershipSnapshots.AllMembers.ContainsMember(peer)) + retainedPeers = new(_peers.Count); + foreach (var (peer, pending) in _peers) { - (removedPeers ??= []).Add(pending); + if (!_localSilo.Equals(peer) && !membershipSnapshots.AllMembers.ContainsMember(peer)) + { + (removedPeers ??= []).Add(pending); + } + else + { + retainedPeers.Add(pending); + } } - else + + if (removedPeers is not null) { - retainedPeers.Add(pending); + foreach (var pending in removedPeers) + { + _peers.Remove(pending.Peer); + } } } + foreach (var peer in retainedPeers) + { + peer.PruneKeys(activeKeys, membershipSnapshots); + } + if (removedPeers is not null) { - foreach (var pending in removedPeers) - { - _peers.Remove(pending.Peer); - } + await Task.WhenAll(removedPeers.Select(peer => peer.StopAsync(drain: false, cancellationToken).AsTask())); } } + finally + { + _inventoryPool.Return(activeKeys); + } + } - foreach (var peer in retainedPeers) + private sealed class InventoryPoolPolicy(IDisseminationNamespace[] namespaces) + : PooledObjectPolicy>> + { + public override Dictionary> Create() { - peer.PruneKeys(activeKeys, membershipSnapshots); + var result = new Dictionary>(namespaces.Length); + foreach (var ns in namespaces) + { + result.Add(ns.Name, []); + } + + return result; } - if (removedPeers is not null) + public override bool Return(Dictionary> inventory) { - await Task.WhenAll(removedPeers.Select(peer => peer.StopAsync(drain: false, cancellationToken).AsTask())); + foreach (var keys in inventory.Values) + { + keys.Clear(); + } + + return true; } } @@ -932,6 +968,7 @@ private async ValueTask SendValues( work.Namespace.Name, out _); namespaceValues ??= []; + long sentVersion = 0; foreach (var value in repair.Values) { namespaceValues.Add(new DisseminationBroadcastValue @@ -941,9 +978,10 @@ private async ValueTask SendValues( }); itemCount++; byteCount += value.Payload.Length; + sentVersion = Math.Max(sentVersion, value.ToVersion); } - sentKeys.Add(new(work, knownVersion, repair.Version)); + sentKeys.Add(new(work, knownVersion, repair.Version, sentVersion)); if (itemCount >= currentOptions.MaxBatchItems || byteCount >= currentOptions.MaxBatchBytes) { break; @@ -968,7 +1006,7 @@ private async ValueTask SendValues( // RPC completion is not application evidence; only the returned receiver versions advance the ledger. _owner._responseObserver?.Invoke(Peer, response); - var acknowledgments = CreateAcknowledgmentLookup(response.Acknowledgments); + var acknowledgments = response.AllVersionsAcknowledged ? null : CreateAcknowledgmentLookup(response.Acknowledgments); var unsupportedNamespaces = response.UnsupportedNamespaces.Count > 0 ? response.UnsupportedNamespaces.ToHashSet() : null; @@ -980,9 +1018,10 @@ private async ValueTask SendValues( continue; } - if (!acknowledgments.TryGetValue( + var acknowledgedVersion = sent.SentVersion; + if (acknowledgments is not null && !acknowledgments.TryGetValue( new(sent.Work.Namespace.Name, sent.Work.Key), - out var acknowledgedVersion)) + out acknowledgedVersion)) { if (!CompleteFromExistingEvidence(sent)) { @@ -1047,6 +1086,7 @@ private async ValueTask SendValues( { Sender = _owner._localSilo, Values = valuesByNamespace, + SupportsCompactAcknowledgments = true, }; sendCancellation.Token.ThrowIfCancellationRequested(); var sendTask = GetTarget().PushBroadcast(batch, sendCancellation.Token); @@ -1596,7 +1636,8 @@ private readonly record struct PendingKeyWork( private readonly record struct SentKey( PendingKeyWork Work, long? FromVersion, - long ResolvedVersion); + long ResolvedVersion, + long SentVersion); private readonly record struct DigestKey( DisseminationNamespace Namespace, diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index d247332cb46..5ea431ff467 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -149,7 +149,7 @@ public async Task ReceiveBroadcast( }; } - var receivedKeys = new Dictionary>(); + var receivedKeys = new Dictionary>(); var unsupportedNamespaces = new List(); foreach (var namespaceName in batch.Values.Keys) { @@ -159,17 +159,19 @@ public async Task ReceiveBroadcast( } } - foreach (var (namespaceName, values) in SelectReceivedValues(batch.Sender, batch.Values, options, antiEntropy: false)) + var selectedValues = SelectReceivedValues(batch.Sender, batch.Values, options, antiEntropy: false); + foreach (var (namespaceName, values) in selectedValues) { var disseminationNamespace = _namespaces[namespaceName]; DisseminationInstruments.OnBroadcastReceived(disseminationNamespace.Name, "tree", values.Count); ConfirmPeerNamespaces(batch.Sender, [namespaceName]); - var namespaceKeys = new Dictionary(); + var namespaceKeys = new Dictionary(); receivedKeys.Add(disseminationNamespace, namespaceKeys); foreach (var item in values) { cancellationToken.ThrowIfCancellationRequested(); - namespaceKeys.TryAdd(item.Value.Key, false); + var existing = namespaceKeys.TryGetValue(item.Value.Key, out var keyState); + var sentVersion = existing ? Math.Max(keyState.SentVersion, item.Value.ToVersion) : item.Value.ToVersion; // The sender necessarily owns this version; use that fact only if an outbound ledger already exists. _broadcastQueue.ObservePeerVersion( batch.Sender, @@ -193,10 +195,7 @@ public async Task ReceiveBroadcast( throw; } - if (result is DisseminationApplyResult.Applied) - { - namespaceKeys[item.Value.Key] = true; - } + namespaceKeys[item.Value.Key] = new(sentVersion, keyState.Applied || result is DisseminationApplyResult.Applied); } } @@ -209,7 +208,7 @@ public async Task ReceiveBroadcast( foreach (var (disseminationNamespace, keys) in receivedKeys) { var membership = membershipSnapshots.GetSnapshot(disseminationNamespace.MembershipScope); - foreach (var (key, applied) in keys) + foreach (var (key, state) in keys) { if (disseminationNamespace.GetVersion(key) <= 0) { @@ -220,20 +219,26 @@ public async Task ReceiveBroadcast( { if (!Equals(peer, batch.Sender)) { - _broadcastQueue.Notify(peer, disseminationNamespace, key, force: applied); + _broadcastQueue.Notify(peer, disseminationNamespace, key, force: state.Applied); } } } } // Once downstream work is queued, report the versions this receiver actually holds. - var acknowledgments = new Dictionary>(); + var compact = batch.SupportsCompactAcknowledgments + && ReferenceEquals(selectedValues, batch.Values) + && CanAcknowledgeTransmittedVersions(receivedKeys); + var acknowledgments = new Dictionary>(receivedKeys.Count); foreach (var (disseminationNamespace, keys) in receivedKeys) { - var namespaceAcknowledgments = new List(keys.Count); - foreach (var key in keys.Keys) + var namespaceAcknowledgments = new List(compact ? 0 : keys.Count); + if (!compact) { - namespaceAcknowledgments.Add(new DigestEntry(key, disseminationNamespace.GetVersion(key))); + foreach (var key in keys.Keys) + { + namespaceAcknowledgments.Add(new DigestEntry(key, disseminationNamespace.GetVersion(key))); + } } acknowledgments.Add(disseminationNamespace.Name, namespaceAcknowledgments); @@ -243,9 +248,30 @@ public async Task ReceiveBroadcast( { Acknowledgments = acknowledgments, UnsupportedNamespaces = unsupportedNamespaces, + AllVersionsAcknowledged = compact, }; } + private static bool CanAcknowledgeTransmittedVersions( + Dictionary> receivedKeys) + { + foreach (var (disseminationNamespace, keys) in receivedKeys) + { + foreach (var (key, state) in keys) + { + // A different version needs an explicit acknowledgment to preserve the peer's exact repair baseline. + if (disseminationNamespace.GetVersion(key) != state.SentVersion) + { + return false; + } + } + } + + return true; + } + + private readonly record struct ReceivedKeyState(long SentVersion, bool Applied); + public async Task RunAntiEntropyRound(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs new file mode 100644 index 00000000000..82e7b364e12 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs @@ -0,0 +1,277 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using Orleans; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CompactAcknowledgmentsRequireCallerSupport(bool supportsCompact) + { + var local = CreateSilo(40701); + var peer = CreateSilo(40702); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(new FakeTransport(local, peer), ns); + var batch = new DisseminationBroadcastBatch + { + Sender = peer, + SupportsCompactAcknowledgments = supportsCompact, + Values = CreateValueGroups(ns.CreateItem(peer, "first", 1), ns.CreateItem(peer, "second", 2)), + }; + try + { + var response = await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + + Assert.Equal(supportsCompact, response.AllVersionsAcknowledged); + Assert.Empty(response.UnsupportedNamespaces); + Assert.Equal(ns.Name, Assert.Single(response.Acknowledgments.Keys)); + Assert.Equal(1, ns.GetVersion("first")); + Assert.Equal(2, ns.GetVersion("second")); + if (supportsCompact) + { + Assert.Empty(response.Acknowledgments[ns.Name]); + } + else + { + Assert.Equal(new long[] { 1, 2 }, response.Acknowledgments[ns.Name].Select(static value => value.Version)); + } + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task CompactAcknowledgmentsPreserveExactAheadVersions() + { + var local = CreateSilo(40711); + var peer = CreateSilo(40712); + var ns = new FakeNamespace(local); + ns.SetValue("value", 3); + var protocol = CreateProtocol(new FakeTransport(local, peer), ns); + try + { + var response = await protocol.ReceiveBroadcast(new() + { + Sender = peer, + SupportsCompactAcknowledgments = true, + Values = CreateValueGroups(ns.CreateItem(peer, "value", 2)), + }, TestContext.Current.CancellationToken); + + Assert.False(response.AllVersionsAcknowledged); + Assert.Equal(3, Assert.Single(response.Acknowledgments[ns.Name]).Version); + Assert.Equal(3, ns.GetVersion("value")); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData("rejected")] + [InlineData("disabled")] + [InlineData("item-limit")] + public async Task CompactAcknowledgmentsRetainExplicitPartialOutcomes(string scenario) + { + var local = CreateSilo(40721); + var peer = CreateSilo(40722); + var ns = new ProtocolReviewNamespace(local); + var other = new FakeNamespace(local, "other"); + other.Options.Enabled = scenario != "disabled"; + ns.ApplyHandler = (value, token) => value.Key == new DisseminationKey("second") && scenario == "rejected" + ? new(DisseminationApplyResult.Rejected) + : ns.Inner.ApplyValueAsync(value, token); + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns, other], options => + { + options.MaxBatchItems = scenario == "item-limit" ? 1 : 10; + }); + var values = CreateValueGroups(ns.Inner.CreateItem(peer, "first", 1), ns.Inner.CreateItem(peer, "second", 2)); + if (scenario == "disabled") + { + values[other.Name] = [other.CreateItem(peer, "other", 1)]; + } + + try + { + var response = await protocol.ReceiveBroadcast(new() + { + Sender = peer, + SupportsCompactAcknowledgments = true, + Values = values, + }, TestContext.Current.CancellationToken); + + Assert.False(response.AllVersionsAcknowledged); + Assert.Equal(1, response.Acknowledgments[ns.Name].Single(value => value.Key == new DisseminationKey("first")).Version); + if (scenario == "rejected") + { + Assert.Equal(0, response.Acknowledgments[ns.Name].Single(value => value.Key == new DisseminationKey("second")).Version); + } + else if (scenario == "item-limit") + { + Assert.Single(response.Acknowledgments[ns.Name]); + Assert.Equal(0, ns.GetVersion("second")); + } + else + { + Assert.Equal(other.Name, Assert.Single(response.UnsupportedNamespaces)); + Assert.False(response.Acknowledgments.ContainsKey(other.Name)); + } + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public void CompactAcknowledgmentFieldsPreserveLegacyWireSchemas() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var peer = CreateSilo(40731); + var current = new DisseminationBroadcastBatch + { + Sender = peer, + SupportsCompactAcknowledgments = true, + Values = CreateValueGroups(new FakeNamespace(peer).CreateItem(peer, "value", 1)), + }; + var legacyBatch = serializer.Deserialize(serializer.SerializeToArray(current)); + Assert.Equal(peer, legacyBatch.Sender); + Assert.Equal(1, Assert.Single(legacyBatch.Values[FakeNamespace.DefaultName]).Value.ToVersion); + var request = serializer.Deserialize(serializer.SerializeToArray(legacyBatch)); + Assert.False(request.SupportsCompactAcknowledgments); + + var legacyResponse = new CompactReviewLegacyResponse + { + Acknowledgments = new() { [FakeNamespace.DefaultName] = [new("value", 3)] }, + }; + var response = serializer.Deserialize(serializer.SerializeToArray(legacyResponse)); + Assert.False(response.AllVersionsAcknowledged); + Assert.Equal(3, Assert.Single(response.Acknowledgments[FakeNamespace.DefaultName]).Version); + var oldReader = serializer.Deserialize(serializer.SerializeToArray(response)); + Assert.Equal(3, Assert.Single(oldReader.Acknowledgments[FakeNamespace.DefaultName]).Version); + var compact = new DisseminationBroadcastResponse + { + AllVersionsAcknowledged = true, + Acknowledgments = new() { [FakeNamespace.DefaultName] = [] }, + }; + var ignoresNewField = serializer.Deserialize(serializer.SerializeToArray(compact)); + Assert.Equal(FakeNamespace.DefaultName, Assert.Single(ignoresNewField.Acknowledgments.Keys)); + Assert.Empty(ignoresNewField.Acknowledgments[FakeNamespace.DefaultName]); + } + + [Fact] + public void CompactAcknowledgmentBodyIsSmallerAndRoundTrips() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var explicitResponse = new DisseminationBroadcastResponse + { + Acknowledgments = new() + { + [FakeNamespace.DefaultName] = CreateSilos(128).Select(silo => new DigestEntry(silo, 639000000000000000)).ToList(), + }, + }; + var compactResponse = new DisseminationBroadcastResponse + { + AllVersionsAcknowledged = true, + Acknowledgments = new() { [FakeNamespace.DefaultName] = [] }, + }; + + var explicitBytes = serializer.SerializeToArray(explicitResponse); + var compactBytes = serializer.SerializeToArray(compactResponse); + var roundTrip = serializer.Deserialize(compactBytes); + + Assert.True(roundTrip.AllVersionsAcknowledged); + Assert.Empty(roundTrip.Acknowledgments[FakeNamespace.DefaultName]); + Assert.True(compactBytes.Length * 4 < explicitBytes.Length, + $"Compact acknowledgment body={compactBytes.Length} bytes; explicit body={explicitBytes.Length} bytes."); + } + + [Fact] + public async Task PruningScratchInventoryIsClearedAfterEnumerationFailure() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40741); + var peer = CreateSilo(40742); + var ns = new CompactReviewInventoryNamespace(local); + ns.Inner.SetValue("old", 1); + var transport = new FakeTransport(local, peer); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + var membership = new DisseminationMembership( + transport.MembershipManager, new FakeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + try + { + Assert.True(queue.Notify(peer, ns, "old")); + await queue.FlushPendingBroadcast(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Single(transport.BroadcastBatches); + + ns.ReadKeys = FailingKeys; + await Assert.ThrowsAsync(() => queue.Prune(membership.CurrentSnapshots, cancellationToken)); + ns.ReadKeys = static () => new DisseminationKey[] { "new" }; + await queue.Prune(membership.CurrentSnapshots, cancellationToken); + ns.ReadKeys = static () => new DisseminationKey[] { "old", "new" }; + + Assert.True(queue.Notify(peer, ns, "old", force: false)); + await queue.FlushPendingBroadcast(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(2, transport.BroadcastBatches.Count); + Assert.All(transport.BroadcastBatches, batch => Assert.Equal(1, Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion)); + } + finally + { + await queue.StopAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + + static IEnumerable FailingKeys() + { + yield return "old"; + throw new InvalidOperationException("Inventory enumeration failed after producing a key."); + } + } + + private sealed class CompactReviewInventoryNamespace(SiloAddress local) : IDisseminationNamespace + { + public FakeNamespace Inner { get; } = new(local); + public Func> ReadKeys { get; set; } = static () => new DisseminationKey[] { "old" }; + public DisseminationNamespace Name => Inner.Name; + public DisseminationNamespaceOptions Options => Inner.Options; + public IEnumerable Keys => ReadKeys(); + public IEnumerable Digests => Inner.Digests; + public long GetVersion(DisseminationKey key) => Inner.GetVersion(key); + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) => Inner.CreateRepair(request); + public ValueTask ApplyValueAsync(DisseminationValue value, CancellationToken cancellationToken) => + Inner.ApplyValueAsync(value, cancellationToken); + } + + [GenerateSerializer] + internal sealed class CompactReviewLegacyBatch + { + [Id(0)] public SiloAddress Sender { get; init; } = null!; + [Id(1)] public Dictionary> Values { get; init; } = []; + } + + [GenerateSerializer] + internal sealed class CompactReviewLegacyResponse + { + [Id(0)] public Dictionary> Acknowledgments { get; init; } = []; + [Id(1)] public List UnsupportedNamespaces { get; init; } = []; + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs index 7dc13cd16f0..9244e2fb189 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs @@ -864,8 +864,10 @@ public async Task ProtocolReviewResponseBudgetsRoundTripAndBoundEveryRepairProbe } } - [Fact] - public async Task ProtocolReviewLargerBroadcastSenderContinuesFromAcknowledgedPrefix() + [Theory] + [InlineData(1)] + [InlineData(10)] + public async Task ProtocolReviewLargerBroadcastSenderContinuesFromAcknowledgedPrefix(int senderLimit) { var sourceAddress = CreateSilo(39581); var receiverAddress = CreateSilo(39582); @@ -882,18 +884,22 @@ public async Task ProtocolReviewLargerBroadcastSenderContinuesFromAcknowledgedPr options.MaxBatchBytes = 8; }, clock); var acknowledgments = new List(); + var compactResponses = new List(); var sentItemCounts = new List(); var transport = new FakeTransport(sourceAddress, receiverAddress); transport.SendBroadcastResponseHandler = async (_, batch, token) => { sentItemCounts.Add(GetBroadcastValues(batch).Count()); var acknowledgment = await receivingProtocol.ReceiveBroadcast(batch, token); - acknowledgments.Add(Assert.Single(acknowledgment.Acknowledgments[source.Name]).Version); + compactResponses.Add(acknowledgment.AllVersionsAcknowledged); + acknowledgments.Add(acknowledgment.AllVersionsAcknowledged + ? receiver.GetVersion("chain") + : Assert.Single(acknowledgment.Acknowledgments[source.Name]).Version); return acknowledgment; }; var sendingProtocol = CreateProtocol(transport, source, options => { - options.MaxBatchItems = 10; + options.MaxBatchItems = senderLimit; options.MaxBatchBytes = 1024; }, clock); try @@ -902,7 +908,10 @@ public async Task ProtocolReviewLargerBroadcastSenderContinuesFromAcknowledgedPr await sendingProtocol.FlushPendingBroadcast(TestContext.Current.CancellationToken) .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(new long[] { 1, 2, 3 }, acknowledgments); - Assert.Equal(new[] { 3, 2, 1 }, sentItemCounts); + int[] expectedCounts = senderLimit == 1 ? [1, 1, 1] : [3, 2, 1]; + bool[] expectedCompact = senderLimit == 1 ? [true, true, true] : [false, false, true]; + Assert.Equal(expectedCounts, sentItemCounts); + Assert.Equal(expectedCompact, compactResponses); Assert.Equal(new long[] { 0, 1, 2 }, receiver.Attempts.Select(static value => value.FromVersion)); Assert.Equal(new long[] { 1, 2, 3 }, receiver.Attempts.Select(static value => value.ToVersion)); Assert.Equal(3, receiver.GetVersion("chain")); From fa28ac19a88bde8b509e67a666110d9f6939b734 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 17:03:57 -0700 Subject: [PATCH 23/50] test(dissemination): assert decoded acknowledgment types --- ...tionProtocolTests.CompactAcknowledgments.cs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs index 82e7b364e12..e6335c00d97 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs @@ -152,27 +152,32 @@ public void CompactAcknowledgmentFieldsPreserveLegacyWireSchemas() SupportsCompactAcknowledgments = true, Values = CreateValueGroups(new FakeNamespace(peer).CreateItem(peer, "value", 1)), }; - var legacyBatch = serializer.Deserialize(serializer.SerializeToArray(current)); + var legacyBatch = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(current))); Assert.Equal(peer, legacyBatch.Sender); Assert.Equal(1, Assert.Single(legacyBatch.Values[FakeNamespace.DefaultName]).Value.ToVersion); - var request = serializer.Deserialize(serializer.SerializeToArray(legacyBatch)); + var request = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(legacyBatch))); Assert.False(request.SupportsCompactAcknowledgments); var legacyResponse = new CompactReviewLegacyResponse { Acknowledgments = new() { [FakeNamespace.DefaultName] = [new("value", 3)] }, }; - var response = serializer.Deserialize(serializer.SerializeToArray(legacyResponse)); + var response = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(legacyResponse))); Assert.False(response.AllVersionsAcknowledged); Assert.Equal(3, Assert.Single(response.Acknowledgments[FakeNamespace.DefaultName]).Version); - var oldReader = serializer.Deserialize(serializer.SerializeToArray(response)); + var oldReader = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(response))); Assert.Equal(3, Assert.Single(oldReader.Acknowledgments[FakeNamespace.DefaultName]).Version); var compact = new DisseminationBroadcastResponse { AllVersionsAcknowledged = true, Acknowledgments = new() { [FakeNamespace.DefaultName] = [] }, }; - var ignoresNewField = serializer.Deserialize(serializer.SerializeToArray(compact)); + var ignoresNewField = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(compact))); Assert.Equal(FakeNamespace.DefaultName, Assert.Single(ignoresNewField.Acknowledgments.Keys)); Assert.Empty(ignoresNewField.Acknowledgments[FakeNamespace.DefaultName]); } @@ -197,7 +202,8 @@ public void CompactAcknowledgmentBodyIsSmallerAndRoundTrips() var explicitBytes = serializer.SerializeToArray(explicitResponse); var compactBytes = serializer.SerializeToArray(compactResponse); - var roundTrip = serializer.Deserialize(compactBytes); + var roundTrip = Assert.IsType( + serializer.Deserialize(compactBytes)); Assert.True(roundTrip.AllVersionsAcknowledged); Assert.Empty(roundTrip.Acknowledgments[FakeNamespace.DefaultName]); From 5c05b945198aae07dd4a4769bb01674b84897a04 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 17:26:28 -0700 Subject: [PATCH 24/50] test(dissemination): include compact acknowledgment in wire contract --- .../Dissemination/DisseminationProtocolTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index 2533513223b..f920787e0f0 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -6052,7 +6052,7 @@ public async Task DisseminationPushBroadcastWireContractPreservesAcknowledgments var method = typeof(IDisseminationSystemTarget).GetMethod(nameof(IDisseminationSystemTarget.PushBroadcast)); Assert.Equal(typeof(Task), method!.ReturnType); Assert.Equal( - [("Acknowledgments", 0), ("UnsupportedNamespaces", 1)], + [("Acknowledgments", 0), ("UnsupportedNamespaces", 1), ("AllVersionsAcknowledged", 2)], typeof(DisseminationBroadcastResponse) .GetProperties() .Select(property => ( @@ -6061,6 +6061,8 @@ public async Task DisseminationPushBroadcastWireContractPreservesAcknowledgments .Single(attribute => attribute.AttributeType == typeof(IdAttribute)) .ConstructorArguments.Single().Value))) .OrderBy(static property => property.Item2)); + Assert.False(batch.SupportsCompactAcknowledgments); + Assert.False(response.AllVersionsAcknowledged); var acknowledgment = Assert.Single(response.Acknowledgments); Assert.Equal(ns.Name, acknowledgment.Key); var digest = Assert.Single(acknowledgment.Value); From ab21609cb687ee96072e171cbf32541576eaf54d Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 18:40:52 -0700 Subject: [PATCH 25/50] test(dissemination): bound pruning scratch allocations --- ...ionProtocolTests.CompactAcknowledgments.cs | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs index e6335c00d97..d6feb4b1728 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs @@ -211,6 +211,45 @@ public void CompactAcknowledgmentBodyIsSmallerAndRoundTrips() $"Compact acknowledgment body={compactBytes.Length} bytes; explicit body={explicitBytes.Length} bytes."); } + [Fact] + public async Task PruningScratchReuseBoundsSteadyStateAllocations() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40751); + var ns = new CompactReviewInventoryNamespace(local); + var keys = Enumerable.Range(0, 1024).Select(index => new DisseminationKey($"key-{index}")).ToArray(); + ns.ReadKeys = () => keys; + var transport = new FakeTransport(local); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + var membership = new DisseminationMembership( + transport.MembershipManager, new FakeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + var snapshots = membership.CurrentSnapshots; + try + { + await queue.Prune(snapshots, cancellationToken); + await queue.Prune(snapshots, cancellationToken); + const int iterations = 128; + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var iteration = 0; iteration < iterations; iteration++) + { + var operation = queue.Prune(snapshots, cancellationToken); + if (!operation.IsCompletedSuccessfully) + { + throw new InvalidOperationException("An empty peer set must prune synchronously."); + } + } + + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.True(allocated < iterations * 512, + $"Pruning a stable 1024-key inventory allocated {allocated} bytes over {iterations} operations."); + } + finally + { + await queue.StopAsync(cancellationToken); + } + } + [Fact] public async Task PruningScratchInventoryIsClearedAfterEnumerationFailure() { From 3e2fcef004a00f68187456af807591de3f14158c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 19:37:36 -0700 Subject: [PATCH 26/50] perf(dissemination): aggregate load updates on a parent-child tree --- .../implementation/runtime-dissemination.md | 12 +- .../Options/DeploymentLoadPublisherOptions.cs | 14 +- ...entLoadStatisticsDisseminationNamespace.cs | 2 + .../DisseminationMembershipSnapshot.cs | 31 +++ .../Dissemination/DisseminationProtocol.cs | 4 +- .../Dissemination/IDisseminationNamespace.cs | 8 + .../Placement/DeploymentLoadPublisher.cs | 13 +- .../Shared/Protocol.cs | 1 + .../Silo/NewRuntime.cs | 5 +- .../Tests/ScalingTests.cs | 14 +- .../methodology.json | 3 +- .../DisseminationProtocolTests.Aggregation.cs | 202 ++++++++++++++++++ .../DisseminationProtocolTests.cs | 8 +- 13 files changed, 293 insertions(+), 24 deletions(-) create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index ed1586bd370..516fc902cdf 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -61,13 +61,17 @@ After the owner accepts a load sample, its serialized payload is reused for forw Every silo derives routing from the same ordered membership projection. Members are ordered by status and silo address. Silo addresses order by generation, port, and IP address, providing stable ordering across membership storage providers with different timestamp precision. Joining, Active, ShuttingDown, and Stopping entries participate. -For fanout `f` and zero-based member index `i`, a forwarding node selects children starting at `f * (i + 1)` and continuing for at most `f` members. An originator sends to the first `f` members, excluding itself, plus its normal forwarding children. Stable ordering makes parent and child selection deterministic for a given membership version and fanout. +Membership uses the broadcast forest: for fanout `f` and zero-based member index `i`, a forwarding node selects children starting at `f * (i + 1)` and continuing for at most `f` members. An originator sends to the first `f` members, excluding itself, plus its normal forwarding children. + +Deployment load uses a rooted aggregation tree. Node `i > 0` has parent `(i - 1) / f`; its children start at `f * i + 1`. A publication enters only the local parent/child queues. Receivers merge updates into authoritative state and notify their other tree neighbors, so updates gather toward common ancestors and distribute into other subtrees. Queues combine distinct keys and retain the latest version of a repeatedly published key. Acknowledged peer versions suppress reflected duplicates. + +The aggregation tree has `N - 1` undirected edges and `2 * (N - 1)` directed queues. With the default 250 ms load batching window, stable membership, normal priority, fitting batches, and no retries or explicit forced flushes, load traffic targets roughly `8 * (N - 1)` requests per second instead of per-update all-to-all fanout. Acknowledgment responses add the corresponding return messages. Batch splitting, repair, and other runtime traffic are accounted separately. Payload delivery still includes each recipient's copy of the statistics; the savings come from aggregating those copies into fewer messages. A longer batching window trades propagation delay for fewer envelopes, while a shorter window favors freshness. Namespaces select a membership scope before topology construction: | Namespace | Scope | Operational effect | |---|---|---| -| Deployment load | Active members | Placement data follows serving capacity. | +| Deployment load | Active members, aggregation tree | Parent/child batches distribute placement statistics with a linear number of directed queues. | | Membership | All dissemination members | Joining and graceful-shutdown transitions can propagate. | Fanout is derived from the target hop count and bounded by the configured minimum and maximum, or selected by the code-configured callback. A membership or fanout change creates a new topology from the next snapshot; acknowledged ledgers and anti-entropy repair convergence across the transition. @@ -95,7 +99,7 @@ The subsystem and both built-in namespaces are temporarily enabled by default fo Each integration has its own . Operators can enable and tune membership and deployment-load dissemination independently while retaining the local concurrency and per-message bounds. -Deployment load uses a 10 ms coalescing window and a 5-second expected update cadence. The shorter window batches nearby samples while limiting the delay added at each forwarding hop. Membership retains high priority and bypasses coalescing. +Deployment load uses a 250 ms coalescing window and a 5-second expected update cadence. The window aggregates updates across producers at each tree edge. Propagation delay includes a batching window at each traversed hop. Membership retains high priority and bypasses coalescing. The anti-entropy loop waits without periodic timer wakeups while the subsystem is disabled. An options-change notification wakes the loop when enablement changes; disabling it returns the loop to the dormant wait. Shutdown removes the options subscription and observes the loop's completion. @@ -145,7 +149,7 @@ Repairs from each sender remain ordered. Competing sender chains are ranked befo Peer namespace support is evidence-driven. Inbound traffic, broadcast acknowledgments, and authoritative anti-entropy responses confirm support for one silo generation. An explicit unsupported response revokes it and retires the publication generations covered by that flush. Publications admitted during the send remain queued with a fresh peer baseline. Membership pruning removes evidence for departed generations. -Deployment-load publication uses dissemination for confirmed peers and the direct system target for unconfirmed peers. If dissemination is disabled, unavailable, rejected, or unable to accept the update within one refresh interval, direct fanout covers all Active peers. Membership direct gossip begins before optional dissemination and keeps the caller's cancellation and shutdown deadline. +Deployment-load publication uses the aggregation tree once all captured Active peers confirm namespace support. During bootstrap or mixed-version operation, direct fanout covers all Active peers so that an unsupported intermediate node cannot separate otherwise capable participants. Direct fanout also applies if dissemination is disabled, unavailable, rejected, or unable to accept the update within one refresh interval. Membership direct gossip begins before optional dissemination and keeps the caller's cancellation and shutdown deadline. ## Concurrency and lifetime diff --git a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs index ec935aa4aa9..5d0f4b2912d 100644 --- a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs @@ -21,17 +21,17 @@ public class DeploymentLoadPublisherOptions /// Gets or sets dissemination options for deployment load statistics. /// /// - /// When enabled, dissemination replaces per-refresh direct fan-out only for active peers which have - /// confirmed support for this namespace. Confirmation remains valid for that silo generation until the peer - /// explicitly rejects the namespace or leaves the eligible membership set. During rolling or mixed-version - /// operation, active peers without confirmation continue to receive direct publications. If dissemination is - /// unavailable, declines the update, throws, or cannot accept it within the refresh interval, direct - /// publication targets all active peers. + /// When all active peers confirm support, dissemination aggregates updates along a parent/child tree + /// using a 250 millisecond batching window by default. Confirmation remains valid for that silo generation + /// until the peer explicitly rejects the namespace or leaves the eligible membership set. During bootstrap + /// or mixed-version operation, direct publication covers all active peers so that an unsupported intermediate + /// node cannot interrupt delivery. Direct publication also covers all active peers when dissemination is + /// unavailable, declines the update, throws, or cannot accept it within the refresh interval. /// public DisseminationNamespaceOptions Dissemination { get; set; } = new() { ExpectedUpdateCadence = TimeSpan.FromSeconds(5), - MaxCoalescingDelay = TimeSpan.FromMilliseconds(10), + MaxCoalescingDelay = TimeSpan.FromMilliseconds(250), }; } } diff --git a/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs index 951c69ea7c6..fec048e4e18 100644 --- a/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs @@ -17,6 +17,8 @@ internal sealed class DeploymentLoadStatisticsDisseminationNamespace( public DisseminationMembershipScope MembershipScope => DisseminationMembershipScope.ActiveMembers; + public DisseminationRoutingMode RoutingMode => DisseminationRoutingMode.AggregationTree; + public DisseminationNamespaceOptions Options => options.CurrentValue.Dissemination; public DisseminationValue CreateValue(SiloAddress origin, SiloRuntimeStatistics statistics) diff --git a/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs b/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs index e6165a37cb0..dd19c9b7372 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs @@ -61,6 +61,7 @@ public DisseminationMembershipSnapshot( } OriginatorTreeTargets = ComputeOriginatorTreeTargets(localSilo, localIndex, fanout); + AggregationTreeTargets = ComputeAggregationTreeTargets(localIndex, fanout); } public MembershipVersion MembershipVersion { get; } @@ -71,6 +72,14 @@ public DisseminationMembershipSnapshot( public ImmutableArray ForwardingTreeTargets { get; } + public ImmutableArray AggregationTreeTargets { get; } + + public ImmutableArray GetOriginatorTargets(DisseminationRoutingMode mode) => + mode == DisseminationRoutingMode.AggregationTree ? AggregationTreeTargets : OriginatorTreeTargets; + + public ImmutableArray GetForwardingTargets(DisseminationRoutingMode mode) => + mode == DisseminationRoutingMode.AggregationTree ? AggregationTreeTargets : ForwardingTreeTargets; + public bool ContainsMember(SiloAddress silo) => _set.Contains(silo); public ImmutableArray SelectAntiEntropyPeers(int peerCount) @@ -137,6 +146,28 @@ private ImmutableArray ComputeForwardingTreeTargets(int index, int return result.ToImmutable(); } + private ImmutableArray ComputeAggregationTreeTargets(int index, int fanout) + { + if (index < 0) + { + return []; + } + + var result = ImmutableArray.CreateBuilder(Math.Min(fanout + 1, Members.Length - 1)); + if (index > 0) + { + result.Add(Members[(index - 1) / fanout]); + } + + var firstChild = (long)fanout * index + 1; + for (var child = firstChild; child < Members.Length && child < firstChild + fanout; child++) + { + result.Add(Members[(int)child]); + } + + return result.ToImmutable(); + } + private sealed class AntiEntropyPeerSelection { public int Cursor; diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index 5ea431ff467..22c53484f10 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -119,7 +119,7 @@ public async ValueTask Publish( // Notifications carry identity only; each peer pump asks the namespace for the latest repair at send time. RecordValueUpdate(disseminationNamespace.Name, key, publishedVersion); var accepted = true; - foreach (var peer in membership.OriginatorTreeTargets) + foreach (var peer in membership.GetOriginatorTargets(disseminationNamespace.RoutingMode)) { accepted &= _broadcastQueue.Notify(peer, disseminationNamespace, key); } @@ -215,7 +215,7 @@ public async Task ReceiveBroadcast( continue; } - foreach (var peer in membership.ForwardingTreeTargets) + foreach (var peer in membership.GetForwardingTargets(disseminationNamespace.RoutingMode)) { if (!Equals(peer, batch.Sender)) { diff --git a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs index eda1a73809d..47c7d06f8ea 100644 --- a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs @@ -10,6 +10,8 @@ internal interface IDisseminationNamespace DisseminationMembershipScope MembershipScope => DisseminationMembershipScope.AllMembers; + DisseminationRoutingMode RoutingMode => DisseminationRoutingMode.BroadcastTree; + DisseminationNamespaceOptions Options { get; } IEnumerable Digests { get; } @@ -41,6 +43,12 @@ internal enum DisseminationMembershipScope AllMembers, } +internal enum DisseminationRoutingMode +{ + BroadcastTree, + AggregationTree, +} + // A null FromVersion means no known peer baseline; a null ToVersion asks for the highest repairable version. internal readonly struct DisseminationRepairRequest( DisseminationKey key, diff --git a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs index b93f0224ae2..f600b474d2a 100644 --- a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs +++ b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs @@ -129,8 +129,17 @@ internal async Task PublishStatistics(CancellationToken cancellationToken) { var dissemination = _serviceProvider.GetRequiredService(); var disseminationNamespace = _serviceProvider.GetRequiredService(); - var unconfirmedPeers = dissemination.GetUnconfirmedPeers(disseminationNamespace).ToHashSet(); - directRecipients = members.Where(unconfirmedPeers.Contains).ToArray(); + var unconfirmedPeers = dissemination.GetUnconfirmedPeers(disseminationNamespace); + if (unconfirmedPeers.Count == 0) + { + directRecipients = []; + } + else + { + // An unsupported intermediate node can separate otherwise capable tree participants. + var unconfirmed = unconfirmedPeers.ToHashSet(); + directRecipients = members.Any(unconfirmed.Contains) ? members : []; + } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { diff --git a/test/Dissemination.IntegrationHarness/Shared/Protocol.cs b/test/Dissemination.IntegrationHarness/Shared/Protocol.cs index be57b5a9aaf..ded44193831 100644 --- a/test/Dissemination.IntegrationHarness/Shared/Protocol.cs +++ b/test/Dissemination.IntegrationHarness/Shared/Protocol.cs @@ -238,6 +238,7 @@ internal sealed record NodeSnapshot public required string[] ForwardingTargets { get; init; } public string[] TopologyMembers { get; init; } = []; public int Fanout { get; init; } + public bool AggregationTree { get; init; } public int AntiEntropyPeerCount { get; init; } public double AntiEntropyIntervalMilliseconds { get; init; } public required Dictionary Metrics { get; init; } diff --git a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs index 5481a9212ec..bc13b294b03 100644 --- a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs +++ b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs @@ -72,10 +72,11 @@ public static NodeSnapshot Decorate(IServiceProvider services, NodeSnapshot snap NamespaceEnabled = ns.Options.Enabled, UnconfirmedPeers = services.GetRequiredService() .GetUnconfirmedPeers(ns).Select(address => address.ToParsableString()).Order(StringComparer.Ordinal).ToArray(), - OriginatorTargets = topology.OriginatorTreeTargets.Select(address => address.ToParsableString()).ToArray(), - ForwardingTargets = topology.ForwardingTreeTargets.Select(address => address.ToParsableString()).ToArray(), + OriginatorTargets = topology.GetOriginatorTargets(ns.RoutingMode).Select(address => address.ToParsableString()).ToArray(), + ForwardingTargets = topology.GetForwardingTargets(ns.RoutingMode).Select(address => address.ToParsableString()).ToArray(), TopologyMembers = topology.Members.Select(address => address.ToParsableString()).ToArray(), Fanout = options.Overlay.GetFanOutFactor(topology.Members.Length), + AggregationTree = ns.RoutingMode == DisseminationRoutingMode.AggregationTree, AntiEntropyPeerCount = options.Overlay.AntiEntropyPeerCount, AntiEntropyIntervalMilliseconds = options.Overlay.AntiEntropyInterval.TotalMilliseconds, TreeGate = services.GetService()?.Snapshot, diff --git a/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs b/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs index ff62a1d41b6..39cec4a49cc 100644 --- a/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs +++ b/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs @@ -185,7 +185,7 @@ async Task PublishRound() snapshot.Identity.ProcessId, snapshot.Address, index >= 0 ? index : null, - index >= 0 && index < topology.Fanout, + index >= 0 && (topology.AggregationTree ? index <= topology.Fanout : index < topology.Fanout), topology.OriginatorTargets, topology.ForwardingTargets, requestMessages.Count + oneWayMessages.Count, @@ -205,7 +205,9 @@ async Task PublishRound() snapshot.PrivateBytes, Delta(snapshot, baseline, "orleans-dissemination-bytes-sent|").Sum, Delta(snapshot, baseline, "orleans-dissemination-broadcast-sent|").Sum, - Delta(snapshot, baseline, "orleans-dissemination-anti-entropy-exchanges|", "direction=out").Sum); + Delta(snapshot, baseline, "orleans-dissemination-anti-entropy-exchanges|", "direction=out").Sum, + Delta(snapshot, baseline, "orleans-dissemination-broadcast-sent|", "namespace=load").Sum, + Delta(snapshot, baseline, "orleans-dissemination-values-sent|", "namespace=load").Sum); }).ToArray(); Assert.All(nodes, node => { @@ -252,6 +254,10 @@ async Task PublishRound() PartitionRecoveryMilliseconds = recoveryLatencies, TotalRpcs = nodes.Sum(node => node.Rpcs), TotalMessages = nodes.Sum(node => node.SentMessages), + LoadBroadcastRequests = nodes.Sum(node => node.LoadBroadcastRequests), + LoadValueTransmissions = nodes.Sum(node => node.LoadValueTransmissions), + LoadBroadcastRequestsPerRound = nodes.Sum(node => node.LoadBroadcastRequests) / iterations, + LoadBroadcastRequestsPerSecond = nodes.Sum(node => node.LoadBroadcastRequests) / (elapsed / 1000), SerializedBytesSent = nodes.Sum(node => node.SerializedBytesSent), SocketBytesSent = nodes.Sum(node => node.SocketBytesSent), TransportBytesSubmitted = nodes.Sum(node => node.TransportBytesSubmitted), @@ -377,5 +383,7 @@ private sealed record NodeCost( long PrivateBytes, double DisseminationPayloadBytes, double DisseminationBroadcasts, - double AntiEntropyExchanges); + double AntiEntropyExchanges, + double LoadBroadcastRequests, + double LoadValueTransmissions); } diff --git a/test/Dissemination.IntegrationHarness/methodology.json b/test/Dissemination.IntegrationHarness/methodology.json index 3750c7dccc3..533a94af65b 100644 --- a/test/Dissemination.IntegrationHarness/methodology.json +++ b/test/Dissemination.IntegrationHarness/methodology.json @@ -124,12 +124,13 @@ "window": "Two warmup publication rounds and complete enabled-peer capability discovery before measuring. Discovery has a minimum 90-second controller budget, expanded to two full peer rotations plus 60 seconds using the actual advertised repair cadence and peer count. Production repair settings stay unchanged. Startup/warmup/preparation duration, CPU and allocations are reported separately. Replacements also finish capability discovery before offered rounds; their bootstrap/fallback/discovery costs remain inside the churn window. Two-second OS-counter sampling margins on all paths. Full-GC retained-memory probes are after the CPU/allocation window.", "metrics": [ "All transmitted request AND one-way RPC messages; responses reported in the separate total-message count", + "Load-namespace tree requests and logical value transmissions separately, including load requests per completed publication round and per measured window second", "Complete serialized Orleans message bytes, including headers/body, from message-size observations", "Actual process-wide System.Net.Sockets sent/received bytes, sampled each second; transport-pipe submitted/read bytes separately", "Process CPU time, GC total allocated bytes, post-full-GC retained managed bytes, working set and private bytes", "Per-round exact state-convergence latency and partition recovery latency", "Separate per-round publication duration and residual convergence wait, retaining the original end-to-end latency and 100 ms polling cadence", - "Per-node counters, actual originator/forwarding targets, tree rank and upper-tree role; raw snapshots preserve topology changes", + "Per-node counters, actual originator/forwarding targets, tree rank and upper-tree role; load uses rooted parent/child aggregation with 250 ms default batching; raw snapshots preserve topology changes", "Raw protocol payload, namespace-batch and anti-entropy counters, never substituted for total RPC or byte counts" ], "limitations": [ diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs new file mode 100644 index 00000000000..ef04d9d171f --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs @@ -0,0 +1,202 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData(1, 4)] + [InlineData(8, 4)] + [InlineData(32, 6)] + [InlineData(100, 10)] + [InlineData(256, 4)] + public void AggregationTopologyHasLinearSymmetricEdges(int count, int fanout) + { + var members = CreateSilos(count).ToImmutableArray(); + var snapshots = members.ToDictionary( + silo => silo, + silo => new DisseminationMembershipSnapshot(new MembershipVersion(1), silo, members, CreateOverlayOptions(fanout))); + Assert.Equal(2 * (count - 1), snapshots.Values.Sum(snapshot => snapshot.AggregationTreeTargets.Length)); + foreach (var (silo, snapshot) in snapshots) + { + Assert.DoesNotContain(silo, snapshot.AggregationTreeTargets); + Assert.Equal(snapshot.AggregationTreeTargets.Length, snapshot.AggregationTreeTargets.Distinct().Count()); + Assert.InRange(snapshot.AggregationTreeTargets.Length, 0, fanout + 1); + foreach (var peer in snapshot.AggregationTreeTargets) + { + Assert.Contains(silo, snapshots[peer].AggregationTreeTargets); + } + } + + var reached = new HashSet { members[0] }; + var pending = new Queue(reached); + while (pending.TryDequeue(out var silo)) + { + foreach (var peer in snapshots[silo].AggregationTreeTargets) + { + if (reached.Add(peer)) + { + pending.Enqueue(peer); + } + } + } + + Assert.Equal(members.Order(), reached.Order()); + } + + [Theory] + [InlineData(8, 4)] + [InlineData(32, 6)] + [InlineData(100, 10)] + [InlineData(256, 4)] + public async Task AggregationTreeDeliversWholeRoundsWithLinearBatchCount(int count, int fanout) + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(count); + var clock = new FakeTimeProvider(); + var nodes = new Dictionary(); + foreach (var member in members) + { + var ns = new FakeNamespace(member) { RoutingMode = DisseminationRoutingMode.AggregationTree }; + ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); + var transport = new FakeTransport(member, members.Where(peer => !peer.Equals(member)).ToArray()); + var protocol = CreateProtocol(transport, ns, options => + { + options.Overlay.FanOutFactor = _ => fanout; + options.MaxConcurrentSends = 1; + }, clock); + nodes.Add(member, (ns, transport, protocol)); + transport.SendBroadcastResponseHandler = (peer, batch, token) => nodes[peer].Protocol.ReceiveBroadcast(batch, token); + } + + try + { + for (var round = 1; round <= 3; round++) + { + var before = nodes.Values.Sum(node => node.Transport.BroadcastBatches.Count); + foreach (var member in members) + { + var node = nodes[member]; + node.Namespace.SetValue(member, round); + Assert.True(await node.Protocol.Publish(node.Namespace, member, round, cancellationToken)); + } + + // Explicit phases isolate aggregation from timer phase/order: gather, then distribute. + foreach (var member in members.Reverse()) + { + await nodes[member].Protocol.FlushPendingBroadcast(cancellationToken); + } + foreach (var member in members) + { + await nodes[member].Protocol.FlushPendingBroadcast(cancellationToken); + } + + foreach (var node in nodes.Values) + { + foreach (var origin in members) + { + Assert.Equal(round, node.Namespace.GetVersion(origin)); + } + } + + var batches = nodes.Values.Sum(node => node.Transport.BroadcastBatches.Count) - before; + Assert.InRange(batches, count - 1, 4 * (count - 1)); + Assert.Empty(nodes.Values.SelectMany(node => node.Transport.AntiEntropyRequests)); + } + } + finally + { + foreach (var node in nodes.Values) + { + node.Namespace.Options.Enabled = false; + } + foreach (var node in nodes.Values) + { + await node.Protocol.StopAsync(cancellationToken); + } + } + } + + [Fact] + public async Task AggregationWindowBatchesStaggeredKeysAndPreservesNextWindow() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40801); + var peer = CreateSilo(40802); + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local) { RoutingMode = DisseminationRoutingMode.AggregationTree }; + ns.Options.MaxCoalescingDelay = new DeploymentLoadPublisherOptions().Dissemination.MaxCoalescingDelay; + var transport = new FakeTransport(local, peer); + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var timestamps = new List(); + transport.SendBroadcastResponseHandler = async (_, batch, _) => + { + timestamps.Add(clock.GetTimestamp()); + if (timestamps.Count == 1) + { + firstStarted.SetResult(); + await releaseFirst.Task; + } + else + { + secondStarted.TrySetResult(); + } + return FakeTransport.CreateAcknowledgment(batch); + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); + try + { + var start = clock.GetTimestamp(); + for (var key = 0; key < 100; key++) + { + ns.SetValue($"key-{key}", 1); + Assert.True(queue.Notify(peer, ns, $"key-{key}")); + clock.Advance(TimeSpan.FromMilliseconds(1)); + } + clock.Advance(TimeSpan.FromMilliseconds(149)); + Assert.Empty(transport.BroadcastBatches); + clock.Advance(TimeSpan.FromMilliseconds(1)); + await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + var firstFlush = queue.FlushPendingBroadcast(cancellationToken); + Assert.Equal(100, GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch).Count()); + Assert.Equal(TimeSpan.FromMilliseconds(250), clock.GetElapsedTime(start, timestamps[0])); + + for (var key = 0; key < 100; key++) + { + ns.SetValue($"key-{key}", 2); + Assert.True(queue.Notify(peer, ns, $"key-{key}")); + clock.Advance(TimeSpan.FromMilliseconds(1)); + } + releaseFirst.SetResult(); + await firstFlush.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + clock.Advance(TimeSpan.FromMilliseconds(149)); + Assert.Single(transport.BroadcastBatches); + clock.Advance(TimeSpan.FromMilliseconds(1)); + await secondStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + + Assert.Equal(TimeSpan.FromMilliseconds(500), clock.GetElapsedTime(start, timestamps[1])); + Assert.Equal(2, transport.BroadcastBatches.Count); + Assert.Equal(100, GetBroadcastValues(transport.BroadcastBatches[1].Batch).Count()); + Assert.All(GetBroadcastValues(transport.BroadcastBatches[1].Batch), value => Assert.Equal(2, value.Value.ToVersion)); + } + finally + { + releaseFirst.TrySetResult(); + ns.Options.Enabled = false; + await queue.StopAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index f920787e0f0..4bcd5c36dd5 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -3198,7 +3198,7 @@ public void OptionsValidatorRejectsInvalidExpectedUpdateCadence() public void NamespaceOptionsUseExpectedUpdateCadenceDefaults() { Assert.Equal(TimeSpan.FromSeconds(5), new DeploymentLoadPublisherOptions().Dissemination.ExpectedUpdateCadence); - Assert.Equal(TimeSpan.FromMilliseconds(10), new DeploymentLoadPublisherOptions().Dissemination.MaxCoalescingDelay); + Assert.Equal(TimeSpan.FromMilliseconds(250), new DeploymentLoadPublisherOptions().Dissemination.MaxCoalescingDelay); Assert.Equal(TimeSpan.FromSeconds(10), new ClusterMembershipOptions().Dissemination.ExpectedUpdateCadence); } @@ -3950,6 +3950,8 @@ public FakeNamespace(SiloAddress localSilo, DisseminationNamespace? name = null) public DisseminationNamespace Name => _name; + public DisseminationRoutingMode RoutingMode { get; set; } + public DisseminationNamespaceOptions Options { get; } = new() { Enabled = true }; public bool ReturnRepairChain { get; set; } @@ -6073,14 +6075,14 @@ public async Task DisseminationPushBroadcastWireContractPreservesAcknowledgments } [Fact] - public async Task DeploymentLoadPublisherDirectSendsToUnconfirmedActivePeersAfterQueueAcceptance() + public async Task DeploymentLoadPublisherDirectSendsToAllActivePeersWhileTreeSupportIsIncomplete() { var harness = CreateDeploymentLoadPublisherHarness(); harness.Dissemination.UnconfirmedPeers = [harness.ActiveTwo]; await harness.PublishStatistics(TestContext.Current.CancellationToken); - AssertDirectRecipients(harness, harness.ActiveTwo); + AssertDirectRecipients(harness, harness.ActiveOne, harness.ActiveTwo); Assert.Single(harness.Dissemination.PublishCalls); Assert.Single(harness.Dissemination.QueryCalls); } From 854b9cc126aaf7f1db1d10604e6dc2d519644053 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 19:52:24 -0700 Subject: [PATCH 27/50] test(dissemination): implement updated membership refresh contract --- .../Dissemination/DisseminationMembershipSnapshotTests.cs | 4 ++-- .../Dissemination/DisseminationProtocolTests.cs | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs index 6db0da8f703..32e993bd8c5 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs @@ -639,7 +639,7 @@ public Task TrySuspectSilo( SiloAddress? indirectProbingSilo, CancellationToken cancellationToken) => Task.FromResult(false); - public Task Refresh(MembershipVersion? targetVersion, CancellationToken cancellationToken) => Task.CompletedTask; + public Task Refresh(MembershipVersion? targetVersion, CancellationToken cancellationToken, bool requireFresh = false) => Task.CompletedTask; public Task ProcessGossipSnapshot( MembershipTableSnapshot value, @@ -723,7 +723,7 @@ public Task TrySuspectSilo( SiloAddress? indirectProbingSilo, CancellationToken cancellationToken) => Task.FromResult(false); - public Task Refresh(MembershipVersion? targetVersion, CancellationToken cancellationToken) => Task.CompletedTask; + public Task Refresh(MembershipVersion? targetVersion, CancellationToken cancellationToken, bool requireFresh = false) => Task.CompletedTask; public Task ProcessGossipSnapshot( MembershipTableSnapshot value, diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index 4bcd5c36dd5..e80d3255b57 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -3280,6 +3280,7 @@ public async Task DisseminationMembershipRefreshDelegatesToMembershipManager() Assert.Equal(1, membershipManager.RefreshCallCount); Assert.Null(Assert.Single(membershipManager.RefreshTargetVersions)); + Assert.False(Assert.Single(membershipManager.RefreshFreshnessRequirements)); } [Fact] @@ -4521,6 +4522,8 @@ public MembershipTableSnapshot CurrentSnapshot public List RefreshTargetVersions { get; } = new(); + public List RefreshFreshnessRequirements { get; } = new(); + public Func? ProcessGossipSnapshotHandler { get; set; } public IAsyncEnumerable MembershipUpdates => EmptyUpdates(TestContext.Current.CancellationToken); @@ -4533,10 +4536,11 @@ public MembershipTableSnapshot CurrentSnapshot public Task TrySuspectSilo(SiloAddress silo, SiloAddress? indirectProbingSilo, CancellationToken cancellationToken) => Task.FromResult(false); - public Task Refresh(MembershipVersion? targetVersion, CancellationToken cancellationToken) + public Task Refresh(MembershipVersion? targetVersion, CancellationToken cancellationToken, bool requireFresh = false) { RefreshCallCount++; RefreshTargetVersions.Add(targetVersion); + RefreshFreshnessRequirements.Add(requireFresh); return _refresh?.Invoke(cancellationToken) ?? Task.CompletedTask; } From 0d7fe8e1eabcb120d4950585e4b580cac13bfcec Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 20:08:26 -0700 Subject: [PATCH 28/50] test(dissemination): iterate aggregation nodes portably --- .../Dissemination/DisseminationProtocolTests.Aggregation.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs index ef04d9d171f..7fac563eff9 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs @@ -93,9 +93,9 @@ public async Task AggregationTreeDeliversWholeRoundsWithLinearBatchCount(int cou } // Explicit phases isolate aggregation from timer phase/order: gather, then distribute. - foreach (var member in members.Reverse()) + for (var index = members.Length - 1; index >= 0; index--) { - await nodes[member].Protocol.FlushPendingBroadcast(cancellationToken); + await nodes[members[index]].Protocol.FlushPendingBroadcast(cancellationToken); } foreach (var member in members) { From 896d415aa5d229f77c2e7fe60d41c7a5d3fda549 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 20:11:56 -0700 Subject: [PATCH 29/50] test(dissemination): record custom aggregation transport sends --- .../DisseminationProtocolTests.Aggregation.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs index 7fac563eff9..2fdc4434b0e 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs @@ -77,7 +77,14 @@ public async Task AggregationTreeDeliversWholeRoundsWithLinearBatchCount(int cou options.MaxConcurrentSends = 1; }, clock); nodes.Add(member, (ns, transport, protocol)); - transport.SendBroadcastResponseHandler = (peer, batch, token) => nodes[peer].Protocol.ReceiveBroadcast(batch, token); + transport.SendBroadcastResponseHandler = (peer, batch, token) => + { + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Add((peer, batch)); + } + return nodes[peer].Protocol.ReceiveBroadcast(batch, token); + }; } try @@ -144,6 +151,10 @@ public async Task AggregationWindowBatchesStaggeredKeysAndPreservesNextWindow() var timestamps = new List(); transport.SendBroadcastResponseHandler = async (_, batch, _) => { + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Add((peer, batch)); + } timestamps.Add(clock.GetTimestamp()); if (timestamps.Count == 1) { From fdb753391dabff50c69c18b2eed7690b5e6f1a29 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 21:09:17 -0700 Subject: [PATCH 30/50] perf(runtime): batch load updates only at the aggregation root --- .../implementation/runtime-dissemination.md | 12 +- .../Options/DeploymentLoadPublisherOptions.cs | 7 +- .../DisseminationBroadcastQueue.cs | 230 +++++++++++----- .../DisseminationMembershipSnapshot.cs | 20 +- .../Dissemination/DisseminationProtocol.cs | 35 ++- .../methodology.json | 2 +- .../DisseminationProtocolTests.Aggregation.cs | 259 +++++++++++++++++- .../DisseminationProtocolTests.cs | 16 +- 8 files changed, 495 insertions(+), 86 deletions(-) diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index 516fc902cdf..89a190e72ee 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -63,15 +63,19 @@ Every silo derives routing from the same ordered membership projection. Members Membership uses the broadcast forest: for fanout `f` and zero-based member index `i`, a forwarding node selects children starting at `f * (i + 1)` and continuing for at most `f` members. An originator sends to the first `f` members, excluding itself, plus its normal forwarding children. -Deployment load uses a rooted aggregation tree. Node `i > 0` has parent `(i - 1) / f`; its children start at `f * i + 1`. A publication enters only the local parent/child queues. Receivers merge updates into authoritative state and notify their other tree neighbors, so updates gather toward common ancestors and distribute into other subtrees. Queues combine distinct keys and retain the latest version of a repeatedly published key. Acknowledged peer versions suppress reflected duplicates. +Deployment load uses root aggregation followed by tree distribution. Node `i > 0` has parent `(i - 1) / f`; its children start at `f * i + 1`. Each non-root producer sends its own update directly to the root immediately. The root merges updates and batches them for 500 ms. Relays atomically enqueue every key from a received batch before waking their send loops, then forward to their children immediately. The batching delay is paid at the root once, rather than at every hop. Queues retain the latest version of a repeatedly published key, and acknowledgments suppress reflected duplicates. -The aggregation tree has `N - 1` undirected edges and `2 * (N - 1)` directed queues. With the default 250 ms load batching window, stable membership, normal priority, fitting batches, and no retries or explicit forced flushes, load traffic targets roughly `8 * (N - 1)` requests per second instead of per-update all-to-all fanout. Acknowledgment responses add the corresponding return messages. Batch splitting, repair, and other runtime traffic are accounted separately. Payload delivery still includes each recipient's copy of the statistics; the savings come from aggregating those copies into fewer messages. A longer batching window trades propagation delay for fewer envelopes, while a shorter window favors freshness. +Aggregation ledgers advance on broadcast acknowledgments, after the receiver processes the batch and attempts downstream queue admission. An ingress producer which is also a child receives its own update in the root's distribution batch and forwards it to its descendants. Passive evidence of a peer's local value serves repair; broadcast acknowledgments establish distribution progress. Anti-entropy repairs gaps left by bounded queue admission or changed topology. + +At one publication per silo per second, root ingress contributes `N - 1` requests per second. Up to two 500 ms root batches traverse `N - 1` distribution edges, targeting roughly `3 * (N - 1)` requests per second under stable, healthy, fitting-batch conditions. One fully collected publication round requires `2 * (N - 1)` requests. Acknowledgment responses add the corresponding return messages. Batch splitting, repair, topology transitions, and other runtime traffic are accounted separately. Payload delivery still includes each recipient's copy of the statistics; the savings come from aggregating those copies into fewer messages. + +Active members are ordered by silo address. Ingress moves toward a smaller root; distribution moves toward larger children. A non-root node receiving an ingress from a higher-address sender redirects it to its current root immediately, accommodating a changed root without adding another batching window. Membership repair reconciles differing views. Namespaces select a membership scope before topology construction: | Namespace | Scope | Operational effect | |---|---|---| -| Deployment load | Active members, aggregation tree | Parent/child batches distribute placement statistics with a linear number of directed queues. | +| Deployment load | Active members, root aggregation | Producers send to one root; already-aggregated batches traverse the distribution tree immediately. | | Membership | All dissemination members | Joining and graceful-shutdown transitions can propagate. | Fanout is derived from the target hop count and bounded by the configured minimum and maximum, or selected by the code-configured callback. A membership or fanout change creates a new topology from the next snapshot; acknowledged ledgers and anti-entropy repair convergence across the transition. @@ -99,7 +103,7 @@ The subsystem and both built-in namespaces are temporarily enabled by default fo Each integration has its own . Operators can enable and tune membership and deployment-load dissemination independently while retaining the local concurrency and per-message bounds. -Deployment load uses a 250 ms coalescing window and a 5-second expected update cadence. The window aggregates updates across producers at each tree edge. Propagation delay includes a batching window at each traversed hop. Membership retains high priority and bypasses coalescing. +Deployment load uses a 500 ms root batching window and a 5-second expected update cadence. Producer ingress and relay forwarding bypass that window; relays preserve whole batches atomically. Membership retains high priority and bypasses coalescing. The anti-entropy loop waits without periodic timer wakeups while the subsystem is disabled. An options-change notification wakes the loop when enablement changes; disabling it returns the loop to the dormant wait. Shutdown removes the options subscription and observes the loop's completion. diff --git a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs index 5d0f4b2912d..13ae59021b8 100644 --- a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs @@ -21,8 +21,9 @@ public class DeploymentLoadPublisherOptions /// Gets or sets dissemination options for deployment load statistics. /// /// - /// When all active peers confirm support, dissemination aggregates updates along a parent/child tree - /// using a 250 millisecond batching window by default. Confirmation remains valid for that silo generation + /// When all active peers confirm support, producers send updates directly to the aggregation root. + /// The root batches for 500 milliseconds by default, then relays forward whole batches immediately + /// through the parent/child tree. Confirmation remains valid for that silo generation /// until the peer explicitly rejects the namespace or leaves the eligible membership set. During bootstrap /// or mixed-version operation, direct publication covers all active peers so that an unsupported intermediate /// node cannot interrupt delivery. Direct publication also covers all active peers when dissemination is @@ -31,7 +32,7 @@ public class DeploymentLoadPublisherOptions public DisseminationNamespaceOptions Dissemination { get; set; } = new() { ExpectedUpdateCadence = TimeSpan.FromSeconds(5), - MaxCoalescingDelay = TimeSpan.FromMilliseconds(250), + MaxCoalescingDelay = TimeSpan.FromMilliseconds(500), }; } } diff --git a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs index e5a7397431b..9e471585104 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs @@ -54,7 +54,8 @@ public bool Notify( SiloAddress peer, IDisseminationNamespace disseminationNamespace, DisseminationKey key, - bool force = true) + bool force = true, + bool immediate = false) { PeerQueuePump pump; lock (_lock) @@ -69,9 +70,36 @@ public bool Notify( // Notification diagnostics run outside the queue lock and can reenter the queue. var version = disseminationNamespace.GetVersion(key); - return pump.Notify(disseminationNamespace, key, version, force); + return pump.Notify(disseminationNamespace, key, version, force, immediate); } + public bool NotifyBatch( + SiloAddress peer, + IDisseminationNamespace disseminationNamespace, + IReadOnlyList notifications, + bool immediate) + { + PeerQueuePump pump; + lock (_lock) + { + if (_stopped) + { + return false; + } + + if (notifications.Count == 0) + { + return true; + } + + pump = GetOrCreatePeerUnsafe(peer); + } + + return pump.NotifyBatch(disseminationNamespace, notifications, immediate); + } + + internal readonly record struct KeyNotification(DisseminationKey Key, long Version, bool Force); + public void ObservePeerVersion( SiloAddress peer, DisseminationNamespace namespaceName, @@ -85,6 +113,13 @@ public void ObservePeerVersion( return; } + // Aggregation acknowledgments also confirm distribution processing. A peer's possession alone + // cannot suppress distribution: its descendants may still need the value. + if (disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree) + { + return; + } + // Passive evidence only sharpens an existing ledger; it must not allocate a timer for a peer with no outbound work. lock (_lock) { @@ -317,87 +352,157 @@ public PeerQueuePump(SiloAddress peer, DisseminationBroadcastQueue owner) private int DirtyCount { get; set; } - public bool Notify(IDisseminationNamespace disseminationNamespace, DisseminationKey key, long version, bool force) + public bool Notify(IDisseminationNamespace disseminationNamespace, DisseminationKey key, long version, bool force, bool immediate) { ScheduledFlush? scheduled = null; - var admissionRejected = false; + bool admissionRejected; + bool accepted; lock (_lock) { - if (_stopping) + var wasEmpty = DirtyCount == 0; + var wasRetrying = _retryAttempt > 0; + accepted = NotifyKeyUnsafe(disseminationNamespace, key, version, force, out var changed, out admissionRejected); + if (changed) { - return false; + scheduled = ScheduleNotificationUnsafe(disseminationNamespace, immediate, wasEmpty, wasRetrying); } + } - if (_pumpFailure is { } pumpFailure) + EmitNotification(disseminationNamespace, admissionRejected ? 1 : 0, scheduled); + return accepted; + } + + public bool NotifyBatch( + IDisseminationNamespace disseminationNamespace, + IReadOnlyList notifications, + bool immediate) + { + ScheduledFlush? scheduled = null; + var rejections = 0; + var accepted = true; + lock (_lock) + { + var wasEmpty = DirtyCount == 0; + var wasRetrying = _retryAttempt > 0; + var anyChanged = false; + for (var index = 0; index < notifications.Count; index++) { - throw new InvalidOperationException($"The dissemination broadcast pump for {Peer} has failed.", pumpFailure); + var notification = notifications[index]; + accepted &= NotifyKeyUnsafe( + disseminationNamespace, notification.Key, notification.Version, notification.Force, + out var changed, out var rejected); + anyChanged |= changed; + rejections += rejected ? 1 : 0; } - var namespaceState = GetOrCreateNamespaceStateUnsafe(disseminationNamespace); - namespaceState.TryGetKey(key, out var keyState); - if (!force) + if (anyChanged) { - // Duplicate deliveries can repair a new child, but cannot perpetuate a cycle between - // skewed trees. Applied same-version state (membership liveness) forces a notification. - if (keyState is { Dirty: true } or { InFlight: true } && keyState.NotificationVersion >= version) - { - return true; - } + // Publish the complete key set before waking a relay's send loop. + scheduled = ScheduleNotificationUnsafe(disseminationNamespace, immediate, wasEmpty, wasRetrying); + } + } - var knownVersion = keyState?.KnownVersion; - if (knownVersion is null && namespaceState.KnownVersions.TryGetValue(key, out var recordedVersion)) - { - knownVersion = recordedVersion; - } + EmitNotification(disseminationNamespace, rejections, scheduled); + return accepted; + } - if (knownVersion >= version) - { - return true; - } - } + private bool NotifyKeyUnsafe( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + bool force, + out bool changed, + out bool admissionRejected) + { + changed = false; + admissionRejected = false; + if (_stopping) + { + return false; + } + + if (_pumpFailure is { } pumpFailure) + { + throw new InvalidOperationException($"The dissemination broadcast pump for {Peer} has failed.", pumpFailure); + } - if (keyState is null && namespaceState.Keys.Count >= disseminationNamespace.Options.MaxPendingItemCount) + var namespaceState = GetOrCreateNamespaceStateUnsafe(disseminationNamespace); + namespaceState.TryGetKey(key, out var keyState); + if (!force) + { + // Duplicate deliveries seed unknown peers without perpetuating cycles between skewed views. + if (keyState is { Dirty: true } or { InFlight: true } && keyState.NotificationVersion >= version) { - admissionRejected = true; + return true; } - else + + var knownVersion = keyState?.KnownVersion; + if (knownVersion is null && namespaceState.KnownVersions.TryGetValue(key, out var recordedVersion)) { - keyState ??= namespaceState.AddKey(key); - // A notification is only a wake-up. The namespace will choose the latest repair when this key is drained. - keyState.NotificationVersion = version; - keyState.NotificationGeneration = ++_notificationEpoch; - var wasRetrying = _retryAttempt > 0; - _retryAttempt = 0; - var wasEmpty = DirtyCount == 0; - MarkDirtyUnsafe(namespaceState, keyState); + knownVersion = recordedVersion; + } - // Filled queues flush immediately; otherwise one timer coalesces all pending namespaces. - var currentOptions = _owner._options.CurrentValue; - if (disseminationNamespace.Options.Priority == DisseminationPriority.High) - { - // High-priority updates skip the coalescing window and pull any pending flush forward. - _flushTimer.Change(TimeSpan.Zero); - _wakeScheduled = true; - scheduled = new(DisseminationBroadcastScheduleReason.Priority, TimeSpan.Zero, _retryAttempt, _notificationEpoch); - } - else if (DirtyCount >= currentOptions.MaxBatchItems - || namespaceState.DirtyCount >= disseminationNamespace.Options.MaxPendingItemCount) - { - _flushTimer.Change(TimeSpan.Zero); - _wakeScheduled = true; - scheduled = new(DisseminationBroadcastScheduleReason.Immediate, TimeSpan.Zero, _retryAttempt, _notificationEpoch); - } - else if (wasEmpty || wasRetrying || !_wakeScheduled) - { - var delay = _owner.GetCoalescingDelay(disseminationNamespace.Options.MaxCoalescingDelay); - _flushTimer.Change(delay); - _wakeScheduled = true; - scheduled = new(DisseminationBroadcastScheduleReason.Coalesce, delay, _retryAttempt, _notificationEpoch); - } + if (knownVersion >= version) + { + return true; } } - if (admissionRejected) + if (keyState is null && namespaceState.Keys.Count >= disseminationNamespace.Options.MaxPendingItemCount) + { + admissionRejected = true; + return false; + } + + keyState ??= namespaceState.AddKey(key); + keyState.NotificationVersion = version; + keyState.NotificationGeneration = ++_notificationEpoch; + _retryAttempt = 0; + MarkDirtyUnsafe(namespaceState, keyState); + changed = true; + return true; + } + + private ScheduledFlush? ScheduleNotificationUnsafe( + IDisseminationNamespace disseminationNamespace, bool immediate, bool wasEmpty, bool wasRetrying) + { + var currentOptions = _owner._options.CurrentValue; + if (immediate) + { + _flushTimer.Wake(); + _wakeScheduled = true; + return new(DisseminationBroadcastScheduleReason.Immediate, TimeSpan.Zero, _retryAttempt, _notificationEpoch); + } + + if (disseminationNamespace.Options.Priority == DisseminationPriority.High) + { + _flushTimer.Change(TimeSpan.Zero); + _wakeScheduled = true; + return new(DisseminationBroadcastScheduleReason.Priority, TimeSpan.Zero, _retryAttempt, _notificationEpoch); + } + + if (DirtyCount >= currentOptions.MaxBatchItems + || _statesByNamespace[disseminationNamespace.Name].DirtyCount >= disseminationNamespace.Options.MaxPendingItemCount) + { + _flushTimer.Change(TimeSpan.Zero); + _wakeScheduled = true; + return new(DisseminationBroadcastScheduleReason.Immediate, TimeSpan.Zero, _retryAttempt, _notificationEpoch); + } + + if (wasEmpty || wasRetrying || !_wakeScheduled) + { + var delay = _owner.GetCoalescingDelay(disseminationNamespace.Options.MaxCoalescingDelay); + _flushTimer.Change(delay); + _wakeScheduled = true; + return new(DisseminationBroadcastScheduleReason.Coalesce, delay, _retryAttempt, _notificationEpoch); + } + + return null; + } + + private void EmitNotification(IDisseminationNamespace disseminationNamespace, int rejections, ScheduledFlush? scheduled) + { + for (var index = 0; index < rejections; index++) { DisseminationInstruments.OnQueueAdmissionRejected(disseminationNamespace.Name); try @@ -415,7 +520,6 @@ public bool Notify(IDisseminationNamespace disseminationNamespace, Dissemination } EmitScheduled(scheduled); - return !admissionRejected; } private void EmitScheduled(ScheduledFlush? scheduled) diff --git a/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs b/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs index dd19c9b7372..7dc313917d5 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs @@ -10,6 +10,8 @@ internal sealed class DisseminationMembershipSnapshot private readonly FrozenSet _set; private readonly SiloAddress[] _antiEntropyPeers; private readonly AntiEntropyPeerSelection _antiEntropySelection; + private readonly SiloAddress _localSilo; + private readonly ImmutableArray _aggregationOriginatorTargets; public DisseminationMembershipSnapshot( MembershipVersion membershipVersion, @@ -19,6 +21,7 @@ public DisseminationMembershipSnapshot( DisseminationMembershipSnapshot? previous = null) { MembershipVersion = membershipVersion; + _localSilo = localSilo; Members = members.IsDefault ? [] : members; // Membership versions also advance without changing the eligible peers. Preserve progress so that // frequent updates cannot keep repair rounds confined to the first few members. @@ -62,6 +65,9 @@ public DisseminationMembershipSnapshot( OriginatorTreeTargets = ComputeOriginatorTreeTargets(localSilo, localIndex, fanout); AggregationTreeTargets = ComputeAggregationTreeTargets(localIndex, fanout); + IsAggregationRoot = localIndex == 0; + AggregationChildren = localIndex > 0 ? AggregationTreeTargets.RemoveAt(0) : AggregationTreeTargets; + _aggregationOriginatorTargets = localIndex < 0 ? [] : IsAggregationRoot ? AggregationChildren : [Members[0]]; } public MembershipVersion MembershipVersion { get; } @@ -74,11 +80,21 @@ public DisseminationMembershipSnapshot( public ImmutableArray AggregationTreeTargets { get; } + public ImmutableArray AggregationChildren { get; } + + public bool IsAggregationRoot { get; } + public ImmutableArray GetOriginatorTargets(DisseminationRoutingMode mode) => - mode == DisseminationRoutingMode.AggregationTree ? AggregationTreeTargets : OriginatorTreeTargets; + mode == DisseminationRoutingMode.AggregationTree ? _aggregationOriginatorTargets : OriginatorTreeTargets; public ImmutableArray GetForwardingTargets(DisseminationRoutingMode mode) => - mode == DisseminationRoutingMode.AggregationTree ? AggregationTreeTargets : ForwardingTreeTargets; + mode == DisseminationRoutingMode.AggregationTree ? AggregationChildren : ForwardingTreeTargets; + + // Active members are address-ordered: ingest moves toward a smaller root, distribution toward larger children. + public ImmutableArray GetForwardingTargets(DisseminationRoutingMode mode, SiloAddress sender) => + mode == DisseminationRoutingMode.AggregationTree && !IsAggregationRoot && sender.CompareTo(_localSilo) > 0 + ? _aggregationOriginatorTargets + : GetForwardingTargets(mode); public bool ContainsMember(SiloAddress silo) => _set.Contains(silo); diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index 22c53484f10..a883ee3c251 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -51,6 +51,15 @@ public DisseminationProtocol( _timeProvider = timeProvider; _logger = logger; _namespaces = disseminationNamespaces.ToFrozenDictionary(static ns => ns.Name); + foreach (var ns in _namespaces.Values) + { + if (ns.RoutingMode == DisseminationRoutingMode.AggregationTree + && ns.MembershipScope != DisseminationMembershipScope.ActiveMembers) + { + throw new ArgumentException("Root aggregation requires an ActiveMembers membership scope.", nameof(disseminationNamespaces)); + } + } + _antiEntropySendGate = new(Math.Max(1, options.CurrentValue.Overlay.AntiEntropyPeerCount)); _broadcastQueue = new DisseminationBroadcastQueue( @@ -121,7 +130,9 @@ public async ValueTask Publish( var accepted = true; foreach (var peer in membership.GetOriginatorTargets(disseminationNamespace.RoutingMode)) { - accepted &= _broadcastQueue.Notify(peer, disseminationNamespace, key); + accepted &= _broadcastQueue.Notify( + peer, disseminationNamespace, key, + immediate: disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree && !membership.IsAggregationRoot); } DisseminationInstruments.OnPublication( @@ -208,6 +219,28 @@ public async Task ReceiveBroadcast( foreach (var (disseminationNamespace, keys) in receivedKeys) { var membership = membershipSnapshots.GetSnapshot(disseminationNamespace.MembershipScope); + if (disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree) + { + var notifications = new List(keys.Count); + foreach (var (key, state) in keys) + { + var version = disseminationNamespace.GetVersion(key); + if (version > 0) + { + notifications.Add(new(key, version, state.Applied)); + } + } + + foreach (var peer in membership.GetForwardingTargets(disseminationNamespace.RoutingMode, batch.Sender)) + { + // A producer which is also a child needs its update returned in the distribution batch + // so that it forwards the update to its own descendants. + _broadcastQueue.NotifyBatch(peer, disseminationNamespace, notifications, immediate: !membership.IsAggregationRoot); + } + + continue; + } + foreach (var (key, state) in keys) { if (disseminationNamespace.GetVersion(key) <= 0) diff --git a/test/Dissemination.IntegrationHarness/methodology.json b/test/Dissemination.IntegrationHarness/methodology.json index 533a94af65b..f1950cabcc2 100644 --- a/test/Dissemination.IntegrationHarness/methodology.json +++ b/test/Dissemination.IntegrationHarness/methodology.json @@ -130,7 +130,7 @@ "Process CPU time, GC total allocated bytes, post-full-GC retained managed bytes, working set and private bytes", "Per-round exact state-convergence latency and partition recovery latency", "Separate per-round publication duration and residual convergence wait, retaining the original end-to-end latency and 100 ms polling cadence", - "Per-node counters, actual originator/forwarding targets, tree rank and upper-tree role; load uses rooted parent/child aggregation with 250 ms default batching; raw snapshots preserve topology changes", + "Per-node counters, actual originator/forwarding targets, tree rank and upper-tree role; load sends directly to one root, batches there for 500 ms, and atomically forwards whole batches down the tree immediately; raw snapshots preserve topology changes", "Raw protocol payload, namespace-batch and anti-entropy counters, never substituted for total RPC or byte counts" ], "limitations": [ diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs index 2fdc4434b0e..1dba83671af 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs @@ -28,6 +28,7 @@ public void AggregationTopologyHasLinearSymmetricEdges(int count, int fanout) silo => silo, silo => new DisseminationMembershipSnapshot(new MembershipVersion(1), silo, members, CreateOverlayOptions(fanout))); Assert.Equal(2 * (count - 1), snapshots.Values.Sum(snapshot => snapshot.AggregationTreeTargets.Length)); + Assert.Equal(count - 1, snapshots.Values.Sum(snapshot => snapshot.AggregationChildren.Length)); foreach (var (silo, snapshot) in snapshots) { Assert.DoesNotContain(silo, snapshot.AggregationTreeTargets); @@ -37,6 +38,15 @@ public void AggregationTopologyHasLinearSymmetricEdges(int count, int fanout) { Assert.Contains(silo, snapshots[peer].AggregationTreeTargets); } + if (snapshot.IsAggregationRoot) + { + Assert.Equal(members[0], silo); + Assert.Equal(snapshot.AggregationChildren, snapshot.GetOriginatorTargets(DisseminationRoutingMode.AggregationTree)); + } + else + { + Assert.Equal(members[0], Assert.Single(snapshot.GetOriginatorTargets(DisseminationRoutingMode.AggregationTree))); + } } var reached = new HashSet { members[0] }; @@ -68,7 +78,11 @@ public async Task AggregationTreeDeliversWholeRoundsWithLinearBatchCount(int cou var nodes = new Dictionary(); foreach (var member in members) { - var ns = new FakeNamespace(member) { RoutingMode = DisseminationRoutingMode.AggregationTree }; + var ns = new FakeNamespace(member) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); var transport = new FakeTransport(member, members.Where(peer => !peer.Equals(member)).ToArray()); var protocol = CreateProtocol(transport, ns, options => @@ -99,7 +113,7 @@ public async Task AggregationTreeDeliversWholeRoundsWithLinearBatchCount(int cou Assert.True(await node.Protocol.Publish(node.Namespace, member, round, cancellationToken)); } - // Explicit phases isolate aggregation from timer phase/order: gather, then distribute. + // Finish producer ingresses before flushing the root, then drain distribution in tree order. for (var index = members.Length - 1; index >= 0; index--) { await nodes[members[index]].Protocol.FlushPendingBroadcast(cancellationToken); @@ -118,7 +132,7 @@ public async Task AggregationTreeDeliversWholeRoundsWithLinearBatchCount(int cou } var batches = nodes.Values.Sum(node => node.Transport.BroadcastBatches.Count) - before; - Assert.InRange(batches, count - 1, 4 * (count - 1)); + Assert.Equal(2 * (count - 1), batches); Assert.Empty(nodes.Values.SelectMany(node => node.Transport.AntiEntropyRequests)); } } @@ -135,6 +149,231 @@ public async Task AggregationTreeDeliversWholeRoundsWithLinearBatchCount(int cou } } + [Theory] + [InlineData(1)] + [InlineData(14)] + public async Task RootAggregationBatchesOnceAndCrossesMultipleLevelsWithoutAnotherDelay(int originIndex) + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(15); + var root = members[0]; + var origin = members[originIndex]; + var clock = new FakeTimeProvider(); + var ingested = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var nodes = new Dictionary(); + foreach (var member in members) + { + var ns = new FakeNamespace(member) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + ApplyObserved = new(TaskCreationOptions.RunContinuationsAsynchronously), + }; + ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(500); + var transport = new FakeTransport(member, members.Where(peer => !peer.Equals(member)).ToArray()); + var protocol = CreateProtocol(transport, ns, options => + { + options.Overlay.FanOutFactor = static _ => 2; + options.MaxConcurrentSends = 1; + }, clock); + nodes.Add(member, (ns, transport, protocol)); + transport.SendBroadcastResponseHandler = async (peer, batch, token) => + { + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Add((peer, batch)); + } + var response = await nodes[peer].Protocol.ReceiveBroadcast(batch, token); + if (member.Equals(origin) && peer.Equals(root)) + { + ingested.TrySetResult(); + } + return response; + }; + } + + try + { + var start = clock.GetTimestamp(); + var source = nodes[origin]; + source.Namespace.SetValue("value", 1); + Assert.True(await source.Protocol.Publish(source.Namespace, "value", 1, cancellationToken)); + await ingested.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(1, nodes[root].Namespace.GetVersion("value")); + Assert.Equal(1, nodes.Values.Sum(node => node.Transport.BroadcastBatches.Count)); + + clock.Advance(TimeSpan.FromMilliseconds(499)); + Assert.All(members.Where(silo => !silo.Equals(root) && !silo.Equals(origin)), + silo => Assert.Equal(0, nodes[silo].Namespace.GetVersion("value"))); + clock.Advance(TimeSpan.FromMilliseconds(1)); + await Task.WhenAll(members.Where(silo => !silo.Equals(origin)) + .Select(silo => nodes[silo].Namespace.ApplyObserved!.Task)) + .WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + + Assert.All(nodes.Values, node => Assert.Equal(1, node.Namespace.GetVersion("value"))); + Assert.Equal(TimeSpan.FromMilliseconds(500), clock.GetElapsedTime(start)); + } + finally + { + foreach (var node in nodes.Values) + { + node.Namespace.Options.Enabled = false; + } + foreach (var node in nodes.Values) + { + await node.Protocol.StopAsync(cancellationToken); + } + } + } + + [Fact] + public async Task AggregationDistributionRequiresBroadcastAcknowledgmentBeforeSuppressingKnownValue() + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(2); + var ns = new FakeNamespace(members[0]) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(500); + var transport = new FakeTransport(members[0], members[1]); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + try + { + ns.SetValue("value", 1); + Assert.True(queue.Notify(members[1], ns, "value", force: false)); + queue.ObservePeerVersion(members[1], ns.Name, "value", 1); + await queue.FlushPendingBroadcast(cancellationToken); + + var batch = Assert.Single(transport.BroadcastBatches).Batch; + Assert.Equal(1, Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); + + Assert.True(queue.Notify(members[1], ns, "value", force: false)); + await queue.FlushPendingBroadcast(cancellationToken); + Assert.Single(transport.BroadcastBatches); + } + finally + { + ns.Options.Enabled = false; + await queue.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task FormerAggregationRootRedirectsIngressToCurrentRootImmediately() + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(7); + var local = members[1]; + var sender = members[^1]; + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(500); + var transport = new FakeTransport(local, members.Where(peer => !peer.Equals(local)).ToArray()); + var forwarded = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = (peer, batch, _) => + { + transport.BroadcastBatches.Add((peer, batch)); + forwarded.TrySetResult(); + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 2, clock); + var start = clock.GetTimestamp(); + try + { + await protocol.ReceiveBroadcast(new() + { + Sender = sender, + Values = CreateValueGroups([ns.CreateItem(sender, sender, 1)]), + }, cancellationToken); + await forwarded.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await protocol.FlushPendingBroadcast(cancellationToken); + + var sent = Assert.Single(transport.BroadcastBatches); + Assert.Equal(members[0], sent.Peer); + Assert.Equal(sender, Assert.Single(GetBroadcastValues(sent.Batch)).Value.Key.Value); + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(start)); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task AggregationRelayForwardsWholeBatchAtomicallyWithoutCoalescing() + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(7); + var local = members[1]; + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(500); + var transport = new FakeTransport(local, members.Where(peer => !peer.Equals(local)).ToArray()); + var forwarded = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var count = 0; + transport.SendBroadcastResponseHandler = (peer, batch, _) => + { + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Add((peer, batch)); + count++; + if (count == 2) + { + forwarded.TrySetResult(); + } + } + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 2, clock); + var values = Enumerable.Range(0, 64).Select(index => ns.CreateItem(members[0], $"key-{index}", 1)).ToArray(); + var start = clock.GetTimestamp(); + try + { + await protocol.ReceiveBroadcast(new() + { + Sender = members[0], + Values = CreateValueGroups(values), + }, cancellationToken); + await forwarded.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await protocol.FlushPendingBroadcast(cancellationToken); + + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(start)); + Assert.Equal(new[] { members[3], members[4] }, transport.BroadcastBatches.Select(batch => batch.Peer).Order()); + Assert.All(transport.BroadcastBatches, batch => + { + Assert.Equal(64, GetBroadcastValues(batch.Batch).Count()); + Assert.All(GetBroadcastValues(batch.Batch), value => Assert.Equal(1, value.Value.ToVersion)); + }); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public void RootAggregationRequiresAddressOrderedActiveMembership() + { + var local = CreateSilo(40811); + var ns = new FakeNamespace(local) { RoutingMode = DisseminationRoutingMode.AggregationTree }; + + var exception = Assert.Throws(() => CreateProtocol(new FakeTransport(local), ns)); + + Assert.Equal("disseminationNamespaces", exception.ParamName); + } + [Fact] public async Task AggregationWindowBatchesStaggeredKeysAndPreservesNextWindow() { @@ -142,7 +381,11 @@ public async Task AggregationWindowBatchesStaggeredKeysAndPreservesNextWindow() var local = CreateSilo(40801); var peer = CreateSilo(40802); var clock = new FakeTimeProvider(); - var ns = new FakeNamespace(local) { RoutingMode = DisseminationRoutingMode.AggregationTree }; + var ns = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; ns.Options.MaxCoalescingDelay = new DeploymentLoadPublisherOptions().Dissemination.MaxCoalescingDelay; var transport = new FakeTransport(local, peer); var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -177,13 +420,13 @@ public async Task AggregationWindowBatchesStaggeredKeysAndPreservesNextWindow() Assert.True(queue.Notify(peer, ns, $"key-{key}")); clock.Advance(TimeSpan.FromMilliseconds(1)); } - clock.Advance(TimeSpan.FromMilliseconds(149)); + clock.Advance(TimeSpan.FromMilliseconds(399)); Assert.Empty(transport.BroadcastBatches); clock.Advance(TimeSpan.FromMilliseconds(1)); await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); var firstFlush = queue.FlushPendingBroadcast(cancellationToken); Assert.Equal(100, GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch).Count()); - Assert.Equal(TimeSpan.FromMilliseconds(250), clock.GetElapsedTime(start, timestamps[0])); + Assert.Equal(TimeSpan.FromMilliseconds(500), clock.GetElapsedTime(start, timestamps[0])); for (var key = 0; key < 100; key++) { @@ -193,12 +436,12 @@ public async Task AggregationWindowBatchesStaggeredKeysAndPreservesNextWindow() } releaseFirst.SetResult(); await firstFlush.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - clock.Advance(TimeSpan.FromMilliseconds(149)); + clock.Advance(TimeSpan.FromMilliseconds(399)); Assert.Single(transport.BroadcastBatches); clock.Advance(TimeSpan.FromMilliseconds(1)); await secondStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - Assert.Equal(TimeSpan.FromMilliseconds(500), clock.GetElapsedTime(start, timestamps[1])); + Assert.Equal(TimeSpan.FromMilliseconds(1000), clock.GetElapsedTime(start, timestamps[1])); Assert.Equal(2, transport.BroadcastBatches.Count); Assert.Equal(100, GetBroadcastValues(transport.BroadcastBatches[1].Batch).Count()); Assert.All(GetBroadcastValues(transport.BroadcastBatches[1].Batch), value => Assert.Equal(2, value.Value.ToVersion)); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index e80d3255b57..365c2fa6212 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -3198,7 +3198,7 @@ public void OptionsValidatorRejectsInvalidExpectedUpdateCadence() public void NamespaceOptionsUseExpectedUpdateCadenceDefaults() { Assert.Equal(TimeSpan.FromSeconds(5), new DeploymentLoadPublisherOptions().Dissemination.ExpectedUpdateCadence); - Assert.Equal(TimeSpan.FromMilliseconds(250), new DeploymentLoadPublisherOptions().Dissemination.MaxCoalescingDelay); + Assert.Equal(TimeSpan.FromMilliseconds(500), new DeploymentLoadPublisherOptions().Dissemination.MaxCoalescingDelay); Assert.Equal(TimeSpan.FromSeconds(10), new ClusterMembershipOptions().Dissemination.ExpectedUpdateCadence); } @@ -3953,6 +3953,8 @@ public FakeNamespace(SiloAddress localSilo, DisseminationNamespace? name = null) public DisseminationRoutingMode RoutingMode { get; set; } + public DisseminationMembershipScope MembershipScope { get; set; } = DisseminationMembershipScope.AllMembers; + public DisseminationNamespaceOptions Options { get; } = new() { Enabled = true }; public bool ReturnRepairChain { get; set; } @@ -7744,8 +7746,10 @@ public async Task PublishReturnsFalseAfterBroadcastQueueStops() Assert.Empty(transport.BroadcastBatches); } - [Fact] - public async Task BroadcastNotificationDiagnosticsRunOutsideQueueLock() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task BroadcastNotificationDiagnosticsRunOutsideQueueLock(bool batch) { var local = CreateSilo(39009); var peer = CreateSilo(39010); @@ -7762,7 +7766,11 @@ public async Task BroadcastNotificationDiagnosticsRunOutsideQueueLock() releaseCallback.Task.GetAwaiter().GetResult(); }), static name => name == DisseminationEvents.BroadcastScheduledEventName); - var notification = Task.Run(() => queue.Notify(peer, ns, "value"), TestContext.Current.CancellationToken); + var notification = Task.Run( + () => batch + ? queue.NotifyBatch(peer, ns, [new("value", 1, true)], immediate: true) + : queue.Notify(peer, ns, "value"), + TestContext.Current.CancellationToken); try { From 258a73079a59db7145d720fa5d85d43038b882c7 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 08:08:17 -0700 Subject: [PATCH 31/50] refactor(dissemination): simplify routing and queue notification state --- .../implementation/runtime-dissemination.md | 4 +- .../DisseminationBroadcastQueue.cs | 73 +++------------- .../DisseminationMembershipSnapshot.cs | 51 ++++------- .../Dissemination/DisseminationProtocol.cs | 48 +++++----- .../DisseminationProtocolTests.Aggregation.cs | 18 ++-- .../DisseminationProtocolTests.QueueReview.cs | 87 +++++++++++++++++++ 6 files changed, 151 insertions(+), 130 deletions(-) diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index 89a190e72ee..e2809038de6 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -35,6 +35,8 @@ flowchart LR Queue entries contain `(namespace, key)` identities. A pump acquires both destination ownership and a global broadcast slot before materializing a repair. Coalesced notifications therefore use the latest namespace state and acknowledged baseline after admission, without retaining serialized payloads for waiting peers. +Single-key publication and received-batch forwarding share one synchronous queue-admission path. A borrowed notification span is consumed under the pump lock before scheduling once, and diagnostic callbacks run after releasing the lock. Peer capability records store confirmed namespace identities for the lifetime of that silo generation; explicit rejection or membership removal retires the confirmation. + Applied state wakes each forwarding child, including same-version membership liveness advances. Duplicate deliveries wake children whose versions are still missing and whose queues contain no equivalent work. This lets a first tree delivery forward state already learned through a direct path while suppressing repeated forwarding cycles between silos with different topology views. ## Version and application invariants @@ -49,7 +51,7 @@ A `DisseminationValue` names one key and a `[FromVersion, ToVersion]` transition The namespace reports whether a repair is current, produced, unavailable, or unable to fit the supplied item and byte budgets. Produced repairs can be complete or a valid prefix. Prefix acknowledgments establish the receiver's new baseline and let the next batch continue the chain. -Broadcast senders advertise support for compact acknowledgments. When the entire batch fits the receiver's limits and each key's current version equals the maximum version transmitted for that key, one response flag certifies those versions. Namespace entries retain capability evidence with empty per-key lists. The sender advances each ledger to its captured transmitted version, including when that batch is only a prefix of a larger repair. Receivers report explicit versions for older callers, truncated batches, and differing receiver versions, preserving precise repair baselines across version skew. +Broadcast senders advertise support for compact acknowledgments. Receive selection explicitly reports whether the whole batch passed validation. When it does and each key's current version equals the maximum version transmitted for that key, one response flag certifies those versions. Namespace entries retain capability evidence with empty per-key lists. The sender advances each ledger to its captured transmitted version, including when that batch is only a prefix of a larger repair. Receivers report explicit versions for older callers, truncated batches, and differing receiver versions, preserving precise repair baselines across version skew. Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. It retains 32 snapshots, prefers a smaller diff when the peer baseline is present, and falls back to a full snapshot when history or capacity makes the diff unsuitable. Current diffs identify their entries as the complete target inventory, so table cleanup is reflected even when a receiver retained a different same-version baseline. Older partial-diff payloads retain their incremental semantics. Same-version snapshots can advance liveness or remove inactive entries while preserving every Active entry and each retained status. This allows cleanup to converge while heartbeats are unchanged, and an advancing heartbeat preserves those same membership guards. The membership manager merges maximum per-entry liveness and retains its local-death handling; dissemination reports whether that manager actually published changed state. diff --git a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs index 9e471585104..e035ba75135 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs @@ -70,13 +70,13 @@ public bool Notify( // Notification diagnostics run outside the queue lock and can reenter the queue. var version = disseminationNamespace.GetVersion(key); - return pump.Notify(disseminationNamespace, key, version, force, immediate); + return pump.Notify(disseminationNamespace, [new(key, version, force)], immediate); } public bool NotifyBatch( SiloAddress peer, IDisseminationNamespace disseminationNamespace, - IReadOnlyList notifications, + ReadOnlySpan notifications, bool immediate) { PeerQueuePump pump; @@ -87,7 +87,7 @@ public bool NotifyBatch( return false; } - if (notifications.Count == 0) + if (notifications.IsEmpty) { return true; } @@ -95,7 +95,7 @@ public bool NotifyBatch( pump = GetOrCreatePeerUnsafe(peer); } - return pump.NotifyBatch(disseminationNamespace, notifications, immediate); + return pump.Notify(disseminationNamespace, notifications, immediate); } internal readonly record struct KeyNotification(DisseminationKey Key, long Version, bool Force); @@ -285,23 +285,7 @@ private TimeSpan GetRetryDelay(int attempt) { // Retries begin at the normal coalescing cadence and back off no further than anti-entropy. // High-priority namespaces do not coalesce, so they do not define the retry floor. - var floor = TimeSpan.MaxValue; - foreach (var disseminationNamespace in _disseminationNamespaces) - { - var namespaceOptions = disseminationNamespace.Options; - if (namespaceOptions.Enabled - && namespaceOptions.Priority != DisseminationPriority.High - && namespaceOptions.MaxCoalescingDelay < floor) - { - floor = namespaceOptions.MaxCoalescingDelay; - } - } - - if (floor == TimeSpan.MaxValue) - { - floor = GetBoundedFallbackDelay(); - } - + var floor = GetCoalescingDelay(TimeSpan.MaxValue); var cap = _options.CurrentValue.Overlay.AntiEntropyInterval; if (floor > cap) { @@ -326,7 +310,7 @@ private sealed class PeerQueuePump private readonly CancellationTokenSource _shutdownCts = new(); private readonly WakeTimer _flushTimer; private readonly Task _flushTask; - private Dictionary _statesByNamespace = []; + private readonly Dictionary _statesByNamespace = []; private TaskCompletionSource _nextFlushCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); private TaskCompletionSource? _activeFlushCompletion; private Exception? _pumpFailure; @@ -352,29 +336,9 @@ public PeerQueuePump(SiloAddress peer, DisseminationBroadcastQueue owner) private int DirtyCount { get; set; } - public bool Notify(IDisseminationNamespace disseminationNamespace, DisseminationKey key, long version, bool force, bool immediate) - { - ScheduledFlush? scheduled = null; - bool admissionRejected; - bool accepted; - lock (_lock) - { - var wasEmpty = DirtyCount == 0; - var wasRetrying = _retryAttempt > 0; - accepted = NotifyKeyUnsafe(disseminationNamespace, key, version, force, out var changed, out admissionRejected); - if (changed) - { - scheduled = ScheduleNotificationUnsafe(disseminationNamespace, immediate, wasEmpty, wasRetrying); - } - } - - EmitNotification(disseminationNamespace, admissionRejected ? 1 : 0, scheduled); - return accepted; - } - - public bool NotifyBatch( + public bool Notify( IDisseminationNamespace disseminationNamespace, - IReadOnlyList notifications, + ReadOnlySpan notifications, bool immediate) { ScheduledFlush? scheduled = null; @@ -385,9 +349,8 @@ public bool NotifyBatch( var wasEmpty = DirtyCount == 0; var wasRetrying = _retryAttempt > 0; var anyChanged = false; - for (var index = 0; index < notifications.Count; index++) + foreach (var notification in notifications) { - var notification = notifications[index]; accepted &= NotifyKeyUnsafe( disseminationNamespace, notification.Key, notification.Version, notification.Force, out var changed, out var rejected); @@ -1622,9 +1585,8 @@ private static bool ValidateRepair( var byteCount = 0; var expectedFromVersion = request.FromVersion; - for (var i = 0; i < repair.Values.Length; i++) + foreach (var value in repair.Values) { - var value = repair.Values[i]; if (value.Key != request.Key || value.FromVersion < 0 || value.ToVersion <= value.FromVersion @@ -1634,17 +1596,10 @@ private static bool ValidateRepair( return false; } - if (i == 0) - { - if (expectedFromVersion is null && value.FromVersion != 0 - || expectedFromVersion is { } fromVersion - && value.FromVersion != 0 - && value.FromVersion != fromVersion) - { - return false; - } - } - else if (value.FromVersion != 0 && value.FromVersion != expectedFromVersion) + if (expectedFromVersion is null && value.FromVersion != 0 + || expectedFromVersion is { } fromVersion + && value.FromVersion != 0 + && value.FromVersion != fromVersion) { return false; } diff --git a/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs b/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs index 7dc313917d5..8cc998045ee 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs @@ -44,7 +44,8 @@ public DisseminationMembershipSnapshot( } _set = memberSet.ToFrozenSet(); - ForwardingTreeTargets = localIndex < 0 ? [] : ComputeForwardingTreeTargets(localIndex, fanout); + ForwardingTreeTargets = localIndex < 0 ? [] : ComputeChildren((long)fanout * (localIndex + 1), fanout); + AggregationChildren = localIndex < 0 ? [] : ComputeChildren((long)fanout * localIndex + 1, fanout); if (localIndex < 0) { @@ -64,10 +65,15 @@ public DisseminationMembershipSnapshot( } OriginatorTreeTargets = ComputeOriginatorTreeTargets(localSilo, localIndex, fanout); - AggregationTreeTargets = ComputeAggregationTreeTargets(localIndex, fanout); IsAggregationRoot = localIndex == 0; - AggregationChildren = localIndex > 0 ? AggregationTreeTargets.RemoveAt(0) : AggregationTreeTargets; - _aggregationOriginatorTargets = localIndex < 0 ? [] : IsAggregationRoot ? AggregationChildren : [Members[0]]; + if (localIndex < 0) + { + _aggregationOriginatorTargets = []; + } + else + { + _aggregationOriginatorTargets = IsAggregationRoot ? AggregationChildren : [Members[0]]; + } } public MembershipVersion MembershipVersion { get; } @@ -78,8 +84,6 @@ public DisseminationMembershipSnapshot( public ImmutableArray ForwardingTreeTargets { get; } - public ImmutableArray AggregationTreeTargets { get; } - public ImmutableArray AggregationChildren { get; } public bool IsAggregationRoot { get; } @@ -144,44 +148,21 @@ private ImmutableArray ComputeOriginatorTreeTargets(SiloAddress loc return result.ToImmutable(); } - private ImmutableArray ComputeForwardingTreeTargets(int index, int fanout) + private ImmutableArray ComputeChildren(long firstChild, int fanout) { - var result = ImmutableArray.CreateBuilder(Math.Min(fanout, Members.Length)); - var firstChild = (long)fanout * (index + 1); - for (var i = 0; i < fanout; i++) - { - var childIndex = firstChild + i; - if (childIndex >= Members.Length) - { - break; - } - - result.Add(Members[(int)childIndex]); - } - - return result.ToImmutable(); - } - - private ImmutableArray ComputeAggregationTreeTargets(int index, int fanout) - { - if (index < 0) + if (firstChild >= Members.Length) { return []; } - var result = ImmutableArray.CreateBuilder(Math.Min(fanout + 1, Members.Length - 1)); - if (index > 0) - { - result.Add(Members[(index - 1) / fanout]); - } - - var firstChild = (long)fanout * index + 1; - for (var child = firstChild; child < Members.Length && child < firstChild + fanout; child++) + var count = (int)Math.Min(fanout, Members.Length - firstChild); + var result = ImmutableArray.CreateBuilder(count); + for (var child = firstChild; child < firstChild + count; child++) { result.Add(Members[(int)child]); } - return result.ToImmutable(); + return result.MoveToImmutable(); } private sealed class AntiEntropyPeerSelection diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index a883ee3c251..24eea719fe7 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -31,7 +31,7 @@ internal sealed partial class DisseminationProtocol private readonly object _valueUpdateLock = new(); private readonly Dictionary _lastValueUpdates = []; private readonly object _peerSupportLock = new(); - private readonly Dictionary> _confirmedPeerNamespaces = []; + private readonly Dictionary> _confirmedPeerNamespaces = []; private readonly FrozenDictionary _namespaces; public DisseminationProtocol( @@ -170,7 +170,7 @@ public async Task ReceiveBroadcast( } } - var selectedValues = SelectReceivedValues(batch.Sender, batch.Values, options, antiEntropy: false); + var selectedValues = SelectReceivedValues(batch.Sender, batch.Values, options, antiEntropy: false, out var isUnfiltered); foreach (var (namespaceName, values) in selectedValues) { var disseminationNamespace = _namespaces[namespaceName]; @@ -221,13 +221,14 @@ public async Task ReceiveBroadcast( var membership = membershipSnapshots.GetSnapshot(disseminationNamespace.MembershipScope); if (disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree) { - var notifications = new List(keys.Count); + var notifications = new DisseminationBroadcastQueue.KeyNotification[keys.Count]; + var notificationCount = 0; foreach (var (key, state) in keys) { var version = disseminationNamespace.GetVersion(key); if (version > 0) { - notifications.Add(new(key, version, state.Applied)); + notifications[notificationCount++] = new(key, version, state.Applied); } } @@ -235,7 +236,8 @@ public async Task ReceiveBroadcast( { // A producer which is also a child needs its update returned in the distribution batch // so that it forwards the update to its own descendants. - _broadcastQueue.NotifyBatch(peer, disseminationNamespace, notifications, immediate: !membership.IsAggregationRoot); + _broadcastQueue.NotifyBatch( + peer, disseminationNamespace, notifications.AsSpan(0, notificationCount), immediate: !membership.IsAggregationRoot); } continue; @@ -260,7 +262,7 @@ public async Task ReceiveBroadcast( // Once downstream work is queued, report the versions this receiver actually holds. var compact = batch.SupportsCompactAcknowledgments - && ReferenceEquals(selectedValues, batch.Values) + && isUnfiltered && CanAcknowledgeTransmittedVersions(receivedKeys); var acknowledgments = new Dictionary>(receivedKeys.Count); foreach (var (disseminationNamespace, keys) in receivedKeys) @@ -540,18 +542,9 @@ private Dictionary> CreateAntiEntropyR lock (_valueUpdateLock) { - List? removedKeys = null; foreach (var key in _lastValueUpdates.Keys) { if (!currentValueStreams.Contains(key)) - { - (removedKeys ??= []).Add(key); - } - } - - if (removedKeys is not null) - { - foreach (var key in removedKeys) { _lastValueUpdates.Remove(key); } @@ -642,7 +635,7 @@ private async Task ApplyAntiEntropyResponses( // A response only includes namespaces which produced repairs. Absence is not evidence that an // up-to-date or unrelated namespace is unsupported, so confirmations are additive here. ConfirmPeerNamespaces(response.Sender, response.Values.Keys); - foreach (var (namespaceName, values) in SelectReceivedValues(response.Sender, response.Values, options, antiEntropy: true)) + foreach (var (namespaceName, values) in SelectReceivedValues(response.Sender, response.Values, options, antiEntropy: true, out _)) { cancellationToken.ThrowIfCancellationRequested(); var disseminationNamespace = _namespaces[namespaceName]; @@ -699,8 +692,10 @@ private Dictionary> Se SiloAddress peer, Dictionary> values, DisseminationOptions options, - bool antiEntropy) + bool antiEntropy, + out bool isUnfiltered) { + isUnfiltered = false; long totalCount = 0; foreach (var entries in values.Values) { @@ -725,6 +720,8 @@ private Dictionary> Se PruneReceivedBatchCursors(fastPathMembers); } + // Compact acknowledgments require this whole-batch validation, independently of collection reuse. + isUnfiltered = true; return values; } @@ -1298,7 +1295,7 @@ internal IReadOnlyList GetUnconfirmedPeers( } if (!_confirmedPeerNamespaces.TryGetValue(peer, out var namespaces) - || !namespaces.ContainsKey(disseminationNamespace.Name)) + || !namespaces.Contains(disseminationNamespace.Name)) { (result ??= []).Add(peer); } @@ -1326,10 +1323,9 @@ private void ConfirmPeerNamespaces( return; } - var now = _timeProvider.GetTimestamp(); lock (_peerSupportLock) { - Dictionary? confirmedNamespaces = null; + HashSet? confirmedNamespaces = null; foreach (var namespaceName in namespaceNames) { if (!_namespaces.TryGetValue(namespaceName, out var disseminationNamespace) @@ -1338,14 +1334,16 @@ private void ConfirmPeerNamespaces( continue; } - if (confirmedNamespaces is null - && !_confirmedPeerNamespaces.TryGetValue(peer, out confirmedNamespaces)) + if (confirmedNamespaces is null) { - confirmedNamespaces = []; - _confirmedPeerNamespaces.Add(peer, confirmedNamespaces); + if (!_confirmedPeerNamespaces.TryGetValue(peer, out confirmedNamespaces)) + { + confirmedNamespaces = []; + _confirmedPeerNamespaces.Add(peer, confirmedNamespaces); + } } - confirmedNamespaces![namespaceName] = now; + confirmedNamespaces.Add(namespaceName); } } } diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs index 1dba83671af..ff5cfa80c87 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs @@ -21,23 +21,21 @@ public partial class DisseminationProtocolTests [InlineData(32, 6)] [InlineData(100, 10)] [InlineData(256, 4)] - public void AggregationTopologyHasLinearSymmetricEdges(int count, int fanout) + public void AggregationTopologyHasOneRootAndLinearDistributionEdges(int count, int fanout) { var members = CreateSilos(count).ToImmutableArray(); var snapshots = members.ToDictionary( silo => silo, silo => new DisseminationMembershipSnapshot(new MembershipVersion(1), silo, members, CreateOverlayOptions(fanout))); - Assert.Equal(2 * (count - 1), snapshots.Values.Sum(snapshot => snapshot.AggregationTreeTargets.Length)); Assert.Equal(count - 1, snapshots.Values.Sum(snapshot => snapshot.AggregationChildren.Length)); + Assert.Equal(members.Skip(1), snapshots.Values.SelectMany(snapshot => snapshot.AggregationChildren).Order()); foreach (var (silo, snapshot) in snapshots) { - Assert.DoesNotContain(silo, snapshot.AggregationTreeTargets); - Assert.Equal(snapshot.AggregationTreeTargets.Length, snapshot.AggregationTreeTargets.Distinct().Count()); - Assert.InRange(snapshot.AggregationTreeTargets.Length, 0, fanout + 1); - foreach (var peer in snapshot.AggregationTreeTargets) - { - Assert.Contains(silo, snapshots[peer].AggregationTreeTargets); - } + Assert.DoesNotContain(silo, snapshot.AggregationChildren); + Assert.Equal(snapshot.AggregationChildren.Length, snapshot.AggregationChildren.Distinct().Count()); + Assert.InRange(snapshot.AggregationChildren.Length, 0, fanout); + Assert.All(snapshot.AggregationChildren, child => Assert.True(child.CompareTo(silo) > 0)); + Assert.Equal(snapshot.AggregationChildren, snapshot.GetForwardingTargets(DisseminationRoutingMode.AggregationTree)); if (snapshot.IsAggregationRoot) { Assert.Equal(members[0], silo); @@ -53,7 +51,7 @@ public void AggregationTopologyHasLinearSymmetricEdges(int count, int fanout) var pending = new Queue(reached); while (pending.TryDequeue(out var silo)) { - foreach (var peer in snapshots[silo].AggregationTreeTargets) + foreach (var peer in snapshots[silo].AggregationChildren) { if (reached.Add(peer)) { diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs index f3d8a7ab969..edff81df901 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs @@ -21,6 +21,93 @@ namespace UnitTests.Dissemination; public partial class DisseminationProtocolTests { + [Fact] + public async Task SingleKeyNotificationAvoidsTemporaryAllocations() + { + var local = CreateSilo(40901); + var peer = CreateSilo(40902); + var ns = new FakeNamespace(local); + ns.SetValue("value", 1); + var transport = new FakeTransport(local, peer); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + try + { + Assert.True(queue.Notify(peer, ns, "value")); + for (var iteration = 0; iteration < 128; iteration++) + { + Assert.True(queue.Notify(peer, ns, "value", force: false)); + } + + const int iterations = 1024; + var accepted = true; + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var iteration = 0; iteration < iterations; iteration++) + { + accepted &= queue.Notify(peer, ns, "value", force: false); + } + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.True(accepted); + Assert.True(allocated < 512, $"Repeated single-key notifications allocated {allocated} bytes."); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(1, Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)).Value.ToVersion); + } + finally + { + ns.Options.Enabled = false; + await queue.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task BatchNotificationPreservesLaterUpdatesWhenAdmissionRejectsAKey() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40903); + var peer = CreateSilo(40904); + var ns = new FakeNamespace(local); + ns.Options.MaxPendingItemCount = 1; + ns.SetValue("value", 1); + ns.SetValue("rejected", 1); + var transport = new FakeTransport(local, peer); + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = async (_, batch, _) => + { + transport.BroadcastBatches.Add((peer, batch)); + if (transport.BroadcastBatches.Count == 1) + { + firstStarted.TrySetResult(); + await releaseFirst.Task; + } + return FakeTransport.CreateAcknowledgment(batch); + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + try + { + Assert.True(queue.Notify(peer, ns, "value")); + var firstFlush = queue.FlushPendingBroadcast(cancellationToken); + await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + ns.SetValue("value", 2); + Assert.False(queue.NotifyBatch(peer, ns, + [new("rejected", 1, true), new("value", 2, true)], immediate: true)); + releaseFirst.TrySetResult(); + await firstFlush.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await queue.FlushPendingBroadcast(cancellationToken); + + Assert.Equal(new long[] { 1, 2 }, transport.BroadcastBatches.Select( + entry => Assert.Single(GetBroadcastValues(entry.Batch)).Value.ToVersion)); + Assert.All(transport.BroadcastBatches, entry => + Assert.Equal(new DisseminationKey("value"), Assert.Single(GetBroadcastValues(entry.Batch)).Value.Key)); + } + finally + { + releaseFirst.TrySetResult(); + ns.Options.Enabled = false; + await queue.StopAsync(cancellationToken); + } + } + [Fact] public void DisseminationKeysSortAcrossSupportedKinds() { From c04daf30d0653453d3307f038175592deb54dfe6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 08:56:18 -0700 Subject: [PATCH 32/50] test(dissemination): retain focused compatibility CI and extract measurements --- ...ce.ps1 => dissemination-compatibility.ps1} | 100 +---- .../workflows/dissemination-compatibility.yml | 78 ++++ .github/workflows/dissemination-evidence.yml | 166 -------- .../implementation/runtime-dissemination.md | 8 +- .../README.md | 46 +++ .../Shared/Protocol.cs | 28 +- .../Silo/NewRuntime.cs | 8 - .../Silo/Observation.cs | 172 +------- .../Silo/Program.cs | 39 +- ...tTests.cs => CompatibilityHarnessTests.cs} | 166 +------- .../Tests/ProcessCluster.cs | 51 +-- .../Tests/ScalingComparison.cs | 106 ----- .../Tests/ScalingTests.cs | 389 ------------------ .../Tests/VersionSkewTests.cs | 46 ++- .../methodology.json | 170 -------- 15 files changed, 217 insertions(+), 1356 deletions(-) rename .github/scripts/{dissemination-evidence.ps1 => dissemination-compatibility.ps1} (58%) create mode 100644 .github/workflows/dissemination-compatibility.yml delete mode 100644 .github/workflows/dissemination-evidence.yml create mode 100644 test/Dissemination.IntegrationHarness/README.md rename test/Dissemination.IntegrationHarness/Tests/{MeasurementTests.cs => CompatibilityHarnessTests.cs} (60%) delete mode 100644 test/Dissemination.IntegrationHarness/Tests/ScalingComparison.cs delete mode 100644 test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs delete mode 100644 test/Dissemination.IntegrationHarness/methodology.json diff --git a/.github/scripts/dissemination-evidence.ps1 b/.github/scripts/dissemination-compatibility.ps1 similarity index 58% rename from .github/scripts/dissemination-evidence.ps1 rename to .github/scripts/dissemination-compatibility.ps1 index e92fc6f7fe2..b7121b98ced 100644 --- a/.github/scripts/dissemination-evidence.ps1 +++ b/.github/scripts/dissemination-compatibility.ps1 @@ -1,76 +1,18 @@ [CmdletBinding()] param( - [ValidateSet('Gate', 'Cost', 'Scale')] - [string] $Mode = 'Gate', [string] $BaselineDirectory = '.dissemination-baseline', - [string] $ArtifactsDirectory = 'Artifacts/DisseminationEvidence', - [ValidatePattern('^\d+(,\d+){0,5}$')] - [string] $Sizes = '4,8,16', - [ValidateRange(3, 200)] - [int] $Iterations = 10, - [ValidateRange(1, 5)] - [int] $Repetitions = 1, - [ValidateSet('All', 'Current')] - [string] $RuntimePaths = 'All', - [ValidatePattern('^(stable|churn|partition)(,(stable|churn|partition)){0,2}$')] - [string] $Scenarios = 'stable,churn,partition', - [ValidateRange(0, 64)] - [int] $SiloProcessorCount = 0, - [ValidateRange(0, 9)] - [int] $GCConserveMemory = 0, + [string] $ArtifactsDirectory = 'Artifacts/DisseminationCompatibility', [switch] $SkipBuild ) $ErrorActionPreference = 'Stop' Set-StrictMode -Version Latest -if ($Mode -eq 'Cost') { - # This automatic PR profile cannot be expanded by dispatch inputs or caller overrides. - # Three rounds ensure that both the churn and partition scenarios actually inject a failure. - $Sizes = '4,8' - $Iterations = 3 - $Repetitions = 1 - $RuntimePaths = 'All' - $Scenarios = 'stable,churn,partition' - $SiloProcessorCount = 0 - $GCConserveMemory = 0 -} - $baselineSha = '6739589254b746a8790cf53524e6abe372bb53d4' $root = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..')) $artifacts = [System.IO.Path]::GetFullPath((Join-Path $root $ArtifactsDirectory)) $binaries = Join-Path $artifacts 'bin' -$results = Join-Path $artifacts "results/$Mode" +$results = Join-Path $artifacts 'results' New-Item -ItemType Directory -Force $binaries, $results | Out-Null -Copy-Item -LiteralPath (Join-Path $root 'test/Dissemination.IntegrationHarness/methodology.json') -Destination $results -foreach ($size in $Sizes.Split(',')) { - if ([int]$size -lt 3 -or [int]$size -gt 128) { - throw 'Silo counts must be between 3 and 128.' - } -} - -if ($Mode -eq 'Scale' -and $IsLinux) { - $memory = Get-Content -LiteralPath '/proc/meminfo' - $availableLine = $memory | Where-Object { $_ -match '^MemAvailable:\s+(\d+)\s+kB$' } - if (!$availableLine -or $availableLine -notmatch '^MemAvailable:\s+(\d+)\s+kB$') { - throw 'Cannot establish available host memory for the scale run.' - } - $availableBytes = [long]$Matches[1] * 1024 - $largestSize = ($Sizes.Split(',') | ForEach-Object { [int]$_ } | Measure-Object -Maximum).Maximum - $requiredBytes = [long]$largestSize * 128MB + 1GB - @{ - AvailableBytes = $availableBytes - RequiredPlanningBytes = $requiredBytes - PlanningBytesPerSilo = 128MB - ControllerHeadroomBytes = 1GB - HostProcessorCount = [Environment]::ProcessorCount - SiloProcessorCount = $SiloProcessorCount - GCConserveMemory = $GCConserveMemory - MemoryInformation = $memory - } | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $results 'host-resources.json') - if ($availableBytes -lt $requiredBytes) { - throw "Insufficient host memory for $largestSize silo processes: available=$availableBytes, planning requirement=$requiredBytes. Use a larger runner." - } -} function Invoke-DotNet([string[]] $Arguments) { & dotnet @Arguments @@ -168,50 +110,28 @@ if (!$SkipBuild) { $env:ORLEANS_DISSEMINATION_OLD = Join-Path $binaries 'Old' $env:ORLEANS_DISSEMINATION_NEW = Join-Path $binaries 'New' $env:ORLEANS_DISSEMINATION_RESULTS = $results -$env:ORLEANS_DISSEMINATION_SIZES = $Sizes -$env:ORLEANS_DISSEMINATION_ITERATIONS = $Iterations.ToString([System.Globalization.CultureInfo]::InvariantCulture) -$env:ORLEANS_DISSEMINATION_REPETITIONS = $Repetitions.ToString([System.Globalization.CultureInfo]::InvariantCulture) -$env:ORLEANS_DISSEMINATION_PROFILE = $Mode -$env:ORLEANS_DISSEMINATION_RUNTIME_PATHS = $RuntimePaths -$env:ORLEANS_DISSEMINATION_SCENARIOS = $Scenarios -$env:ORLEANS_DISSEMINATION_SILO_PROCESSOR_COUNT = $SiloProcessorCount.ToString([System.Globalization.CultureInfo]::InvariantCulture) -$env:ORLEANS_DISSEMINATION_GC_CONSERVE_MEMORY = $GCConserveMemory.ToString([System.Globalization.CultureInfo]::InvariantCulture) $runner = Join-Path $binaries 'Runner/Dissemination.IntegrationHarness.Tests.dll' if (!(Test-Path -LiteralPath $runner)) { throw "Published runner not found: $runner" } -$testClass = if ($Mode -eq 'Gate') { - 'Orleans.Dissemination.IntegrationHarness.VersionSkewTests' -} else { - 'Orleans.Dissemination.IntegrationHarness.ScalingTests' -} -$minimum = if ($Mode -eq 'Gate') { '7' } else { '1' } +$testClass = 'Orleans.Dissemination.IntegrationHarness.VersionSkewTests' +$minimum = '10' @{ - Mode = $Mode PinnedBaseline = $baselineSha CurrentCommit = Get-Revision $root - Sizes = $Sizes - Iterations = $Iterations - Repetitions = $Repetitions - RuntimePaths = $RuntimePaths - Scenarios = $Scenarios - SiloProcessorCount = $SiloProcessorCount - GCConserveMemory = $GCConserveMemory StartUtc = [DateTimeOffset]::UtcNow Command = "dotnet $runner --filter-class $testClass --minimum-expected-tests $minimum --report-trx" - Limitation = 'Loopback OS processes; controlled test membership provider; not cross-machine production throughput.' + Environment = 'Separate loopback OS processes with a controlled test membership provider.' } | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $results 'invocation.json') Push-Location $root try { - if ($Mode -eq 'Gate') { - Invoke-DotNet @( - $runner, '--filter-class', 'Orleans.Dissemination.IntegrationHarness.MeasurementTests', - '--minimum-expected-tests', '16', '--report-trx', - '--results-directory', (Join-Path $results 'instrument-checks') - ) - } + Invoke-DotNet @( + $runner, '--filter-class', 'Orleans.Dissemination.IntegrationHarness.CompatibilityHarnessTests', + '--minimum-expected-tests', '8', '--report-trx', + '--results-directory', (Join-Path $results 'harness-checks') + ) Invoke-DotNet @( $runner, '--filter-class', $testClass, '--minimum-expected-tests', $minimum, diff --git a/.github/workflows/dissemination-compatibility.yml b/.github/workflows/dissemination-compatibility.yml new file mode 100644 index 00000000000..bfa2fc77038 --- /dev/null +++ b/.github/workflows/dissemination-compatibility.yml @@ -0,0 +1,78 @@ +name: Dissemination compatibility + +on: + pull_request: + branches: [main] + paths: + - '.github/workflows/dissemination-compatibility.yml' + - '.github/scripts/dissemination-compatibility.ps1' + - 'test/Dissemination.IntegrationHarness/**' + - 'src/Orleans.Core.Abstractions/IDs/SiloAddress.cs' + - 'src/Orleans.Runtime/Dissemination/**' + - 'src/Orleans.Runtime/MembershipService/**' + - 'src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs' + - 'src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs' + - 'src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs' + - 'src/Orleans.Runtime/Hosting/DefaultSiloServices.cs' + - 'src/Orleans.Runtime/OrleansContracts.txt' + - 'src/Orleans.Core/OrleansContracts.txt' + - 'src/Orleans.Core/Runtime/Constants.cs' + - 'src/Orleans.Core/Runtime/MembershipTableSnapshot.cs' + - 'src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs' + - 'src/Orleans.Core/SystemTargetInterfaces/IDeploymentLoadPublisher.cs' + - 'src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs' + - 'src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs' + - 'src/Orleans.Core/Configuration/Options/DisseminationOptions.cs' + merge_group: + types: [checks_requested] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: dissemination-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + DOTNET_NOLOGO: true + +jobs: + version-skew: + name: Pinned-binary upgrade, rollback and recovery + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - name: Isolated pre-dissemination checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + repository: dotnet/orleans + ref: 6739589254b746a8790cf53524e6abe372bb53d4 + path: .dissemination-baseline + persist-credentials: false + - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 + with: + global-json-file: global.json + - name: Build isolated binaries and run bounded merge-gate checks + shell: pwsh + run: ./.github/scripts/dissemination-compatibility.ps1 + - name: Preserve test results and process diagnostics + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-compatibility-results + retention-days: 14 + path: | + Artifacts/DisseminationCompatibility/results + Artifacts/DisseminationCompatibility/*.binlog + - name: Preserve reproducible binary pair + if: success() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-compatibility-binaries + retention-days: 7 + path: Artifacts/DisseminationCompatibility/bin diff --git a/.github/workflows/dissemination-evidence.yml b/.github/workflows/dissemination-evidence.yml deleted file mode 100644 index af50619d295..00000000000 --- a/.github/workflows/dissemination-evidence.yml +++ /dev/null @@ -1,166 +0,0 @@ -name: Dissemination evidence - -on: - pull_request: - branches: [main] - types: [opened, synchronize, reopened, labeled] - paths: - - '.github/workflows/dissemination-evidence.yml' - - '.github/scripts/dissemination-evidence.ps1' - - 'test/Dissemination.IntegrationHarness/**' - - 'src/Orleans.Core.Abstractions/IDs/SiloAddress.cs' - - 'src/Orleans.Runtime/Dissemination/**' - - 'src/Orleans.Runtime/MembershipService/**' - - 'src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs' - - 'src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs' - - 'src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs' - - 'src/Orleans.Runtime/Hosting/DefaultSiloServices.cs' - - 'src/Orleans.Runtime/OrleansContracts.txt' - - 'src/Orleans.Core/OrleansContracts.txt' - - 'src/Orleans.Core/Runtime/Constants.cs' - - 'src/Orleans.Core/Runtime/MembershipTableSnapshot.cs' - - 'src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs' - - 'src/Orleans.Core/SystemTargetInterfaces/IDeploymentLoadPublisher.cs' - - 'src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs' - - 'src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs' - - 'src/Orleans.Core/Configuration/Options/DisseminationOptions.cs' - merge_group: - types: [checks_requested] - workflow_dispatch: - inputs: - scale: - description: 'Also run the larger controlled off/on scaling comparison' - type: boolean - default: false - sizes: - description: 'Comma-separated process counts (3..128, maximum six sizes)' - type: string - default: '8,16,32,64,100' - iterations: - description: 'Publication rounds per scenario (3..200)' - type: number - default: 10 - repetitions: - description: 'Paired off/on repetitions (1..5)' - type: number - default: 2 - -permissions: - contents: read - -concurrency: - group: dissemination-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 - DOTNET_NOLOGO: true - -jobs: - version-skew: - name: Pinned-binary upgrade, rollback and recovery - if: github.event_name != 'pull_request' || github.event.action != 'labeled' || github.event.label.name == 'dissemination-scale' - runs-on: ubuntu-latest - timeout-minutes: 35 - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - persist-credentials: false - - name: Isolated pre-dissemination checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - repository: dotnet/orleans - ref: 6739589254b746a8790cf53524e6abe372bb53d4 - path: .dissemination-baseline - persist-credentials: false - - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 - with: - global-json-file: global.json - - name: Build isolated binaries and run bounded merge-gate checks - shell: pwsh - run: ./.github/scripts/dissemination-evidence.ps1 -Mode Gate - - name: Preserve gate results and complete process evidence - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: dissemination-gate-evidence - retention-days: 14 - path: | - Artifacts/DisseminationEvidence/results - Artifacts/DisseminationEvidence/*.binlog - - name: Preserve reproducible binary pair - if: success() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: dissemination-binaries - retention-days: 7 - path: Artifacts/DisseminationEvidence/bin - - bounded-cost: - name: Bounded original, disabled and enabled costs - needs: version-skew - runs-on: ubuntu-latest - timeout-minutes: 25 - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - persist-credentials: false - - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 - with: - global-json-file: global.json - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: dissemination-binaries - path: Artifacts/DisseminationEvidence/bin - - name: Measure bounded original, explicitly disabled and supported-enabled paths - shell: pwsh - run: ./.github/scripts/dissemination-evidence.ps1 -Mode Cost -SkipBuild - - name: Preserve the automatic PR cost evidence - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: dissemination-bounded-cost-evidence - retention-days: 14 - path: Artifacts/DisseminationEvidence/results/Cost - - scaling: - name: Dissemination scaling (${{ matrix.size }} silos) - if: (github.event_name == 'workflow_dispatch' && inputs.scale) || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'dissemination-scale')) - needs: [version-skew, bounded-cost] - runs-on: ubuntu-latest - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - size: ${{ fromJSON(format('[{0}]', inputs.sizes || '8,16,32,64,100')) }} - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - persist-credentials: false - - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 - with: - global-json-file: global.json - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: dissemination-binaries - path: Artifacts/DisseminationEvidence/bin - - name: Measure paired current-binary stable scaling - shell: pwsh - env: - EVIDENCE_SIZE: ${{ matrix.size }} - EVIDENCE_ITERATIONS: ${{ inputs.iterations || 10 }} - EVIDENCE_REPETITIONS: ${{ inputs.repetitions || 2 }} - run: | - ./.github/scripts/dissemination-evidence.ps1 -Mode Scale -SkipBuild ` - -Sizes $env:EVIDENCE_SIZE ` - -Iterations ([int]$env:EVIDENCE_ITERATIONS) ` - -Repetitions ([int]$env:EVIDENCE_REPETITIONS) ` - -RuntimePaths Current -Scenarios stable ` - -SiloProcessorCount 1 -GCConserveMemory 9 - - name: Preserve raw cost windows, per-node load and failure evidence - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: dissemination-scale-${{ matrix.size }}-evidence - retention-days: 30 - path: Artifacts/DisseminationEvidence/results diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index e2809038de6..3ed5f8935c1 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -111,13 +111,13 @@ The anti-entropy loop waits without periodic timer wakeups while the subsystem i Enablement requires representative measurements of convergence, RPC volume, serialized bytes, allocation, CPU, and tail latency across stable membership, churn, and partition recovery. Tree delivery can reduce RPC counts while increasing serialized bytes through forwarding and repair. Compare the original runtime, the updated explicitly disabled path, and the enabled path at the intended cluster sizes and publication rates, including skewed peer speeds and rolling upgrades. -### Measuring cluster-size scaling +### Compatibility and recovery validation -The evidence workflow pairs the same current binary with dissemination disabled and enabled. Reports retain raw totals and normalize RPCs, serialized bytes, allocations, and CPU by offered publication count. Actual live silo count, binary identity, workload, and resource settings must match within each pair. End-to-end convergence is reported alongside publication duration and the remaining convergence wait, making the controller's sequential publication work and 100 ms polling cadence visible. +The `Dissemination compatibility` workflow builds the pre-feature runtime and current candidate in isolated checkouts, then starts separate silo processes. Assembly identities, hashes, and paths establish which binary each process actually loaded. Rolling-upgrade and rollback tests exercise old binaries alongside enabled and explicitly disabled current binaries. -The opt-in `dissemination-scale` PR label or scale dispatch runs separate jobs at 8, 16, 32, 64, and 100 silo processes. Each size uses ten publication rounds and two order-reversed repetitions of the current off/on stable workload. The regular three-path 4/8-silo stable, churn, and partition checks remain in place. Remove the label after collecting a curve to return subsequent pushes to the small profile. +Four- and eight-silo partition tests cover all three runtime modes. Healing drains affected connections at both endpoints, including unfinished handshakes, then confirms acknowledged bidirectional control calls before offering one publication round. Convergence requires exact load inventory and values. Recovery diagnostics include connection draining and readiness in elapsed time. -The large curve uses a runtime processor-count override of one and `GCConserveMemory=9` uniformly across sizes and variants. All silo processes share the runner's physical CPUs. Artifacts record the effective processor count, GC mode, host memory, startup costs, per-node measurements, and paired results. A memory preflight reserves a planning allowance of 128 MiB per silo plus 1 GiB for the controller. These single-host results expose protocol scaling under shared CPU and memory; cross-machine capacity and production tail latency require a separately provisioned deployment. +Additional tests isolate anti-entropy from tree delivery, direct gossip, and membership-table reads to establish history-miss and same-version heartbeat repair. Cancellation and partitioned shutdown have explicit completion bounds. Artifacts retain test results, binary manifests, process logs, and failure snapshots for diagnosis. Performance comparisons use separately maintained, explicitly invoked tooling. ## Broadcast pumps and backpressure diff --git a/test/Dissemination.IntegrationHarness/README.md b/test/Dissemination.IntegrationHarness/README.md new file mode 100644 index 00000000000..06dba28a4df --- /dev/null +++ b/test/Dissemination.IntegrationHarness/README.md @@ -0,0 +1,46 @@ +# Dissemination compatibility tests + +This harness tests actual old/new runtime binaries in separate OS processes. The +controller references shared control types and loads neither Orleans runtime. +The `Dissemination compatibility` workflow builds each silo executable against +its own isolated runtime checkout and verifies the loaded assembly hashes, +versions, and paths. + +The original runtime is pinned to +`6739589254b746a8790cf53524e6abe372bb53d4`. Only harness sources are copied into +that checkout; Orleans runtime sources remain unchanged. + +## Coverage + +- Rolling upgrades, mixed enablement, legacy fallback, and rollback. +- Exact load-state convergence after three partition/heal cycles, at four and + eight silos, using original, current-disabled, and current-enabled runtimes. +- History-miss and same-version heartbeat repair with tree delivery, direct + gossip, and membership-table reads independently isolated. +- Cancellation of an entered remote RPC and bounded partitioned shutdown. +- Harness guards for connection draining, exact inventory/value comparison, + public readonly statistics fields, and outgoing-activity observation. + +Commands own the load-publication cadence. Partition healing drains middleware +and transport closure at both endpoints, then establishes acknowledged +bidirectional readiness before one publication round. Recovery elapsed time +includes those probes. The disposed publication timer reference is retained for +the original runtime's shutdown lifecycle. + +## Execution and artifacts + +CI runs `.github\scripts\dissemination-compatibility.ps1`. Runtime builds are +CI-only. To reproduce locally, download the matching +`dissemination-compatibility-binaries` artifact into +`Artifacts\DisseminationCompatibility\bin`, then run: + +```powershell +.\.github\scripts\dissemination-compatibility.ps1 -SkipBuild +``` + +The runner checks eight harness cases and ten process cases. Results are written +under `Artifacts\DisseminationCompatibility\results`: TRX files, invocation +metadata, scenario snapshots, process logs, and final shutdown records. + +Scaling experiments and performance reports are maintained separately from this +correctness suite. diff --git a/test/Dissemination.IntegrationHarness/Shared/Protocol.cs b/test/Dissemination.IntegrationHarness/Shared/Protocol.cs index ded44193831..24c92a5f314 100644 --- a/test/Dissemination.IntegrationHarness/Shared/Protocol.cs +++ b/test/Dissemination.IntegrationHarness/Shared/Protocol.cs @@ -9,9 +9,7 @@ internal sealed record NodeConfiguration( int SiloPort, int ParentProcessId, bool Enabled, - bool FastRecovery = true, - int SiloProcessorCount = 0, - int GCConserveMemory = 0); + bool FastRecovery = true); internal sealed record Command( int Id, @@ -45,8 +43,6 @@ internal sealed record AssemblyProof( string Sha256, string Path); -internal sealed record MetricValue(long Count, double Sum); - internal sealed record TreeGateSnapshot(bool Blocked, int InFlight, long Admitted, long Rejected); internal sealed record StateComparisonProbe(string Before, string After, string[] PublicFields); @@ -234,27 +230,9 @@ internal sealed record NodeSnapshot public required SortedDictionary LoadVersions { get; init; } public required string[] ActiveMembers { get; init; } public required string[] UnconfirmedPeers { get; init; } - public required string[] OriginatorTargets { get; init; } - public required string[] ForwardingTargets { get; init; } - public string[] TopologyMembers { get; init; } = []; - public int Fanout { get; init; } - public bool AggregationTree { get; init; } - public int AntiEntropyPeerCount { get; init; } - public double AntiEntropyIntervalMilliseconds { get; init; } - public required Dictionary Metrics { get; init; } + public long BroadcastsSent { get; init; } + public long OutgoingRepairs { get; init; } public required ApplyEvidence[] Applies { get; init; } - public long TransportBytesWritten { get; init; } - public long TransportBytesRead { get; init; } - public double SocketBytesSent { get; init; } - public double SocketBytesReceived { get; init; } - public long SocketCounterSamples { get; init; } - public double CpuMilliseconds { get; init; } - public long AllocatedBytes { get; init; } - public long ManagedHeapBytes { get; init; } - public long WorkingSetBytes { get; init; } - public long PrivateBytes { get; init; } - public int ProcessorCount { get; init; } - public bool ServerGC { get; init; } public int PendingControlCalls { get; init; } public int StartedControlCalls { get; init; } public int CancelledControlCalls { get; init; } diff --git a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs index bc13b294b03..e6e8de40487 100644 --- a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs +++ b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs @@ -65,20 +65,12 @@ public static NodeSnapshot Decorate(IServiceProvider services, NodeSnapshot snap } var ns = services.GetRequiredService(); - var topology = services.GetRequiredService().CurrentSnapshots.ActiveMembers; return snapshot with { Enabled = options.Enabled, NamespaceEnabled = ns.Options.Enabled, UnconfirmedPeers = services.GetRequiredService() .GetUnconfirmedPeers(ns).Select(address => address.ToParsableString()).Order(StringComparer.Ordinal).ToArray(), - OriginatorTargets = topology.GetOriginatorTargets(ns.RoutingMode).Select(address => address.ToParsableString()).ToArray(), - ForwardingTargets = topology.GetForwardingTargets(ns.RoutingMode).Select(address => address.ToParsableString()).ToArray(), - TopologyMembers = topology.Members.Select(address => address.ToParsableString()).ToArray(), - Fanout = options.Overlay.GetFanOutFactor(topology.Members.Length), - AggregationTree = ns.RoutingMode == DisseminationRoutingMode.AggregationTree, - AntiEntropyPeerCount = options.Overlay.AntiEntropyPeerCount, - AntiEntropyIntervalMilliseconds = options.Overlay.AntiEntropyInterval.TotalMilliseconds, TreeGate = services.GetService()?.Snapshot, }; } diff --git a/test/Dissemination.IntegrationHarness/Silo/Observation.cs b/test/Dissemination.IntegrationHarness/Silo/Observation.cs index 95b4e10c6c2..65f91c22d8f 100644 --- a/test/Dissemination.IntegrationHarness/Silo/Observation.cs +++ b/test/Dissemination.IntegrationHarness/Silo/Observation.cs @@ -1,8 +1,5 @@ -using System.Buffers; using System.Diagnostics; using System.Diagnostics.Metrics; -using System.Diagnostics.Tracing; -using System.IO.Pipelines; using Microsoft.AspNetCore.Connections; namespace Orleans.Dissemination.IntegrationHarness; @@ -12,42 +9,33 @@ internal sealed class Observation : IDisposable, IObserver, private readonly object _lock = new(); private readonly MeterListener _meters = new(); private readonly List _subscriptions = []; - private readonly Dictionary _metrics = []; private readonly Queue _applies = []; - private readonly SocketCounters _sockets = new(); private readonly object _connectionLock = new(); private readonly Dictionary _connections = []; private readonly HashSet _drainingConnections = []; private Func? _blockedConnections; private int _partitioned; - private long _bytesWritten; - private long _bytesRead; + private long _broadcastsSent; + private long _outgoingRepairs; - private readonly bool _captureProvenance; - - public Observation(bool captureProvenance) + public Observation() { - _captureProvenance = captureProvenance; _meters.InstrumentPublished = (instrument, listener) => { if (instrument.Meter.Name.StartsWith("Microsoft.Orleans", StringComparison.Ordinal) - && (instrument.Name is "orleans-messaging-sent-messages-size" or "orleans-messaging-received-messages-size" - || instrument.Name.StartsWith("orleans-dissemination-", StringComparison.Ordinal))) + && instrument.Name is "orleans-dissemination-broadcast-sent" or "orleans-dissemination-anti-entropy-exchanges") { listener.EnableMeasurementEvents(instrument); } }; _meters.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); - _meters.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); - _meters.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); _meters.Start(); _subscriptions.Add(DiagnosticListener.AllListeners.Subscribe(this)); } public bool Partitioned => Volatile.Read(ref _partitioned) != 0; - public long BytesWritten => Interlocked.Read(ref _bytesWritten); - public long BytesRead => Interlocked.Read(ref _bytesRead); - public (double Sent, double Received, long Samples) SocketSnapshot => _sockets.Snapshot(); + public long BroadcastsSent => Interlocked.Read(ref _broadcastsSent); + public long OutgoingRepairs => Interlocked.Read(ref _outgoingRepairs); public void Partition() { @@ -151,15 +139,12 @@ public async Task Connection(ConnectionContext context, ConnectionDelegate next) return; } - var original = context.Transport; - context.Transport = new CountingDuplexPipe(original, this); try { await next(context); } finally { - context.Transport = original; lock (_connectionLock) { _connections.Remove(context); @@ -175,15 +160,6 @@ private sealed class ConnectionLifetime(ConnectionContext context) public Task? CloseTask { get; set; } } - public Dictionary Metrics() - { - _meters.RecordObservableInstruments(); - lock (_lock) - { - return new(_metrics); - } - } - public ApplyEvidence[] Applies() { lock (_lock) @@ -192,28 +168,27 @@ public ApplyEvidence[] Applies() } } - private void Record(Instrument instrument, double value, ReadOnlySpan> tags) + private void Record(Instrument instrument, long value, ReadOnlySpan> tags) { - var tagValues = new string[tags.Length]; - for (var index = 0; index < tags.Length; index++) + if (instrument.Name == "orleans-dissemination-broadcast-sent") { - tagValues[index] = $"{tags[index].Key}={tags[index].Value}"; + Interlocked.Add(ref _broadcastsSent, value); + return; } - Array.Sort(tagValues, StringComparer.Ordinal); - var key = $"{instrument.Name}|{string.Join(';', tagValues)}"; - lock (_lock) + foreach (var tag in tags) { - var previous = _metrics.GetValueOrDefault(key, new(0, 0)); - _metrics[key] = instrument.IsObservable - ? new(previous.Count + 1, value) - : new(previous.Count + 1, previous.Sum + value); + if (tag.Key == "direction" && Equals(tag.Value, "out")) + { + Interlocked.Add(ref _outgoingRepairs, value); + return; + } } } public void OnNext(DiagnosticListener value) { - if (_captureProvenance && value.Name == "Microsoft.Orleans.Dissemination") + if (value.Name == "Microsoft.Orleans.Dissemination") { lock (_lock) { @@ -254,7 +229,6 @@ public void OnError(Exception error) { } public void Dispose() { _meters.Dispose(); - _sockets.Dispose(); lock (_lock) { foreach (var subscription in _subscriptions) @@ -263,116 +237,4 @@ public void Dispose() } } } - - private sealed class CountingDuplexPipe(IDuplexPipe inner, Observation owner) : IDuplexPipe - { - public PipeReader Input { get; } = new CountingReader(inner.Input, owner); - public PipeWriter Output { get; } = new CountingWriter(inner.Output, owner); - } - - private sealed class CountingWriter(PipeWriter inner, Observation owner) : PipeWriter - { - public override void Advance(int bytes) - { - inner.Advance(bytes); - Interlocked.Add(ref owner._bytesWritten, bytes); - } - - public override Memory GetMemory(int sizeHint = 0) => inner.GetMemory(sizeHint); - public override Span GetSpan(int sizeHint = 0) => inner.GetSpan(sizeHint); - public override void CancelPendingFlush() => inner.CancelPendingFlush(); - public override void Complete(Exception? exception = null) => inner.Complete(exception); - public override ValueTask FlushAsync(CancellationToken cancellationToken = default) => inner.FlushAsync(cancellationToken); - } - - private sealed class CountingReader(PipeReader inner, Observation owner) : PipeReader - { - private ReadOnlySequence _buffer; - - public override void AdvanceTo(SequencePosition consumed) => AdvanceTo(consumed, consumed); - - public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) - { - Interlocked.Add(ref owner._bytesRead, _buffer.Slice(0, consumed).Length); - inner.AdvanceTo(consumed, examined); - _buffer = default; - } - - public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) - { - var result = await inner.ReadAsync(cancellationToken); - _buffer = result.Buffer; - return result; - } - - public override bool TryRead(out ReadResult result) - { - if (!inner.TryRead(out result)) - { - return false; - } - - _buffer = result.Buffer; - return true; - } - - public override void CancelPendingRead() => inner.CancelPendingRead(); - public override void Complete(Exception? exception = null) => inner.Complete(exception); - } - - private sealed class SocketCounters : EventListener - { - private readonly object _lock = new(); - private double _sent; - private double _received; - private long _samples; - - protected override void OnEventSourceCreated(EventSource eventSource) - { - if (eventSource.Name == "System.Net.Sockets") - { - EnableEvents(eventSource, EventLevel.LogAlways, EventKeywords.All, - new Dictionary { ["EventCounterIntervalSec"] = "1" }); - } - } - - protected override void OnEventWritten(EventWrittenEventArgs eventData) - { - if (eventData.EventName != "EventCounters" - || eventData.Payload?.Count != 1 - || eventData.Payload[0] is not IDictionary counter - || !counter.TryGetValue("Name", out var name)) - { - return; - } - - var incremental = counter.TryGetValue("Increment", out var measurement); - if (!incremental && !counter.TryGetValue("Mean", out measurement)) - { - return; - } - - lock (_lock) - { - var value = Convert.ToDouble(measurement, System.Globalization.CultureInfo.InvariantCulture); - if ((string)name == "bytes-sent") - { - _sent = incremental ? _sent + value : value; - _samples++; - } - else if ((string)name == "bytes-received") - { - _received = incremental ? _received + value : value; - } - } - } - - public (double Sent, double Received, long Samples) Snapshot() - { - lock (_lock) - { - return (_sent, _received, _samples); - } - } - } } diff --git a/test/Dissemination.IntegrationHarness/Silo/Program.cs b/test/Dissemination.IntegrationHarness/Silo/Program.cs index 9873c6fba0e..7b9fdcaa5ff 100644 --- a/test/Dissemination.IntegrationHarness/Silo/Program.cs +++ b/test/Dissemination.IntegrationHarness/Silo/Program.cs @@ -39,7 +39,7 @@ public static async Task Main(string[] args) } #endif using var lifetime = new CancellationTokenSource(TimeSpan.FromMinutes(45)); - using var observation = new Observation(configuration.FastRecovery); + using var observation = new Observation(); var membership = new FileMembershipTable(configuration); using var host = TestClusterHostFactory.CreateSiloHost( configuration.Name, @@ -58,7 +58,7 @@ public static async Task Main(string[] args) services.Configure(options => { options.ClusterId = configuration.ClusterId; - options.ServiceId = "dissemination-evidence"; + options.ServiceId = "dissemination-compatibility"; }); services.Configure(options => options.SiloName = configuration.Name); services.Configure(options => @@ -137,9 +137,7 @@ await publisher.RunOrQueueTask(() => using var shutdown = new CancellationTokenSource(TimeSpan.FromSeconds(20)); await host.StopAsync(shutdown.Token).WaitAsync(shutdown.Token); - // Drain the last one-second OS socket-counter bucket before recording shutdown cost. - await Task.Delay(TimeSpan.FromMilliseconds(1100)); - await Write(new Response(-1, Snapshot(host.Services, detailed: true), null)); + await Write(new Response(-1, Snapshot(host.Services), null)); return 0; } finally @@ -159,8 +157,6 @@ private static async Task Execute(IServiceProvider services, Comma case "snapshot": case "stop": break; - case "measure": - return Snapshot(services, detailed: true); case "verify-load-comparison": { var original = new EnvironmentStatistics(10, 20, 100, 100, 900, 900, 1000); @@ -218,11 +214,6 @@ await publisher.RunOrQueueTask(() => return Task.CompletedTask; }).WaitAsync(cancellationToken); break; - case "retained-memory": - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); - break; case "echo": { var peer = services.GetRequiredService().GetSystemTarget( @@ -299,15 +290,12 @@ private static Task DrainConnections(IServiceProvider services, string? peer, Ca cancellationToken); } - internal static NodeSnapshot Snapshot(IServiceProvider services, bool? detailed = null) + internal static NodeSnapshot Snapshot(IServiceProvider services) { - var includeDetails = detailed ?? services.GetRequiredService().FastRecovery; var manager = services.GetRequiredService(); var publisher = services.GetRequiredService(); var observation = services.GetRequiredService(); var membership = manager.CurrentSnapshot; - using var process = Process.GetCurrentProcess(); - var socket = observation.SocketSnapshot; var result = new NodeSnapshot { CapturedAtUtc = DateTimeOffset.UtcNow, @@ -324,22 +312,9 @@ internal static NodeSnapshot Snapshot(IServiceProvider services, bool? detailed LoadVersions = new(publisher.PeriodicStatistics.ToDictionary( pair => pair.Key.ToParsableString(), pair => pair.Value.DateTime.Ticks), StringComparer.Ordinal), UnconfirmedPeers = [], - OriginatorTargets = [], - ForwardingTargets = [], - Metrics = includeDetails ? observation.Metrics() : [], - Applies = includeDetails ? observation.Applies() : [], - TransportBytesWritten = observation.BytesWritten, - TransportBytesRead = observation.BytesRead, - SocketBytesSent = socket.Sent, - SocketBytesReceived = socket.Received, - SocketCounterSamples = socket.Samples, - CpuMilliseconds = process.TotalProcessorTime.TotalMilliseconds, - AllocatedBytes = GC.GetTotalAllocatedBytes(precise: true), - ManagedHeapBytes = GC.GetTotalMemory(forceFullCollection: false), - WorkingSetBytes = process.WorkingSet64, - PrivateBytes = process.PrivateMemorySize64, - ProcessorCount = Environment.ProcessorCount, - ServerGC = System.Runtime.GCSettings.IsServerGC, + BroadcastsSent = observation.BroadcastsSent, + OutgoingRepairs = observation.OutgoingRepairs, + Applies = observation.Applies(), PendingControlCalls = services.GetRequiredService().Pending, StartedControlCalls = services.GetRequiredService().Started, CancelledControlCalls = services.GetRequiredService().Cancelled, diff --git a/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs b/test/Dissemination.IntegrationHarness/Tests/CompatibilityHarnessTests.cs similarity index 60% rename from test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs rename to test/Dissemination.IntegrationHarness/Tests/CompatibilityHarnessTests.cs index 2df171f06bd..85a21e35a16 100644 --- a/test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs +++ b/test/Dissemination.IntegrationHarness/Tests/CompatibilityHarnessTests.cs @@ -1,93 +1,13 @@ -using System.Diagnostics; using System.Diagnostics.Metrics; using System.IO.Pipelines; -using System.Net; -using System.Net.Sockets; using System.Text.Json; using Microsoft.AspNetCore.Connections; using Xunit; namespace Orleans.Dissemination.IntegrationHarness; -public sealed class MeasurementTests +public sealed class CompatibilityHarnessTests { - [Fact] - public void ScalingComparisonNormalizesEqualOfferedWorkAndReportsLatencyDistribution() - { - var environment = JsonSerializer.SerializeToElement(new { ProcessorCount = 4, SiloProcessorCount = 1, GCConserveMemory = 9 }); - var disabled = new ScalingSample( - "CurrentDisabled", 100, 100, "stable", 0, 4, 400, - new("candidate", "same-binary"), environment, - 39600, 800000, 4000000, 2000, 1000000, [40, 10, 30, 20]); - var enabled = disabled with - { - RuntimePath = "CurrentEnabledSupported", - TotalRpcs = 4000, - SerializedBytesSent = 400000, - AllocatedBytes = 2000000, - CpuMilliseconds = 1000, - ConvergenceMilliseconds = [30, 10, 20, 10], - }; - - var result = ScalingComparison.Create(JsonSerializer.Serialize(new[] { enabled, disabled })); - - Assert.Equal(0, result.IncompletePairs); - var pair = Assert.Single(result.Pairs); - Assert.Equal(100, pair.Size); - Assert.Equal(400, pair.OfferedPublications); - Assert.Equal(99, pair.Disabled.RpcsPerPublication); - Assert.Equal(10, pair.Enabled.RpcsPerPublication); - Assert.Equal(2000, pair.Disabled.SerializedBytesPerPublication); - Assert.Equal(1000, pair.Enabled.SerializedBytesPerPublication); - Assert.Equal(10000, pair.Disabled.AllocatedBytesPerPublication); - Assert.Equal(5000, pair.Enabled.AllocatedBytesPerPublication); - Assert.Equal(5, pair.Disabled.CpuMillisecondsPerPublication); - Assert.Equal(2.5, pair.Enabled.CpuMillisecondsPerPublication); - Assert.Equal(25, pair.Disabled.MedianConvergenceMilliseconds); - Assert.Equal(15, pair.Enabled.MedianConvergenceMilliseconds); - Assert.Equal(40, pair.Disabled.P95ConvergenceMilliseconds); - Assert.Equal(30, pair.Enabled.P95ConvergenceMilliseconds); - Assert.Contains("99.00 -> 10.00", result.ToMarkdown(), StringComparison.Ordinal); - } - - [Theory] - [InlineData("binary")] - [InlineData("workload")] - [InlineData("environment")] - [InlineData("silo-count")] - public void ScalingComparisonRejectsIncomparableMeasurements(string mismatch) - { - var sample = new ScalingSample( - "CurrentDisabled", 100, 100, "stable", 0, 3, 300, - new("candidate", "same-binary"), JsonSerializer.SerializeToElement(new { ProcessorCount = 4 }), - 100, 1000, 10000, 100, 1000, [1, 2, 3]); - var enabled = sample with { RuntimePath = "CurrentEnabledSupported" }; - enabled = mismatch switch - { - "binary" => enabled with { Runtime = new("different", "other-binary") }, - "workload" => enabled with { OfferedPublications = 301 }, - "environment" => enabled with { Environment = JsonSerializer.SerializeToElement(new { ProcessorCount = 1 }) }, - "silo-count" => enabled with { LiveSilos = 99 }, - _ => throw new InvalidOperationException(mismatch), - }; - - Assert.Throws(() => ScalingComparison.Create(JsonSerializer.Serialize(new[] { sample, enabled }))); - } - - [Fact] - public void ScalingComparisonReportsPartialPairsExplicitly() - { - var sample = new ScalingSample( - "CurrentDisabled", 8, 8, "stable", 0, 3, 24, - new("candidate", "same-binary"), JsonSerializer.SerializeToElement(new { ProcessorCount = 4 }), - 100, 1000, 10000, 100, 1000, [1, 2, 3]); - - var result = ScalingComparison.Create(JsonSerializer.Serialize(new[] { sample })); - - Assert.Empty(result.Pairs); - Assert.Equal(1, result.IncompletePairs); - } - [Fact] public void LoadStateComparisonRequiresExactInventoryAndValues() { @@ -113,7 +33,7 @@ public void LoadStateComparisonRequiresExactInventoryAndValues() [Fact] public async Task ConnectionDrain_WaitsForInitializationAndCloseBeforeReopening() { - using var observation = new Observation(captureProvenance: false); + using var observation = new Observation(); await using var context = CreateConnectionContext("initializing"); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -172,7 +92,7 @@ await observation.Connection(reopened, _ => [Fact] public async Task ConnectionDrain_CapturesUnidentifiedPeersAndPreservesHealthyConnections() { - using var observation = new Observation(captureProvenance: false); + using var observation = new Observation(); await using var unknown = CreateConnectionContext("unknown"); await using var healthy = CreateConnectionContext("healthy"); healthy.Items["peer"] = "healthy"; @@ -315,78 +235,18 @@ await Assert.ThrowsAsync(() => gate.Invoke( } [Fact] - public void MessageObservation_KeepsRequestOneWayAndBytesDistinct() + public void DisseminationActivityCountsOnlyOutgoingWork() { - using var observation = new Observation(captureProvenance: false); - using var meter = new Meter("Microsoft.Orleans.HarnessMeasurementTest"); - var histogram = meter.CreateHistogram("orleans-messaging-sent-messages-size"); - histogram.Record(97, new KeyValuePair("MessageDirection", "Request")); - histogram.Record(31, new KeyValuePair("MessageDirection", "OneWay")); - histogram.Record(17, new KeyValuePair("MessageDirection", "Response")); - var metrics = observation.Metrics(); - Assert.Equal(new MetricValue(1, 97), metrics["orleans-messaging-sent-messages-size|MessageDirection=Request"]); - Assert.Equal(new MetricValue(1, 31), metrics["orleans-messaging-sent-messages-size|MessageDirection=OneWay"]); - Assert.Equal(new MetricValue(1, 17), metrics["orleans-messaging-sent-messages-size|MessageDirection=Response"]); - } - - [Fact] - public async Task TransportObservation_CountsConsumedAndSubmittedBytes() - { - var cancellationToken = TestContext.Current.CancellationToken; - using var observation = new Observation(captureProvenance: false); - var input = new Pipe(); - var output = new Pipe(); - await using var context = new DefaultConnectionContext("instrumentation-test") - { - Transport = new TestPipe(input.Reader, output.Writer), - }; - var payload = new byte[] { 1, 2, 3, 4, 5 }; - await observation.Connection(context, async connection => - { - await input.Writer.WriteAsync(payload, cancellationToken); - var received = await connection.Transport.Input.ReadAsync(cancellationToken); - connection.Transport.Input.AdvanceTo(received.Buffer.End); - await connection.Transport.Output.WriteAsync(payload, cancellationToken); - }); - Assert.Equal(payload.Length, observation.BytesRead); - Assert.Equal(payload.Length, observation.BytesWritten); - await input.Reader.CompleteAsync(); - await input.Writer.CompleteAsync(); - await output.Reader.CompleteAsync(); - await output.Writer.CompleteAsync(); - } + using var observation = new Observation(); + using var meter = new Meter("Microsoft.Orleans.CompatibilityHarnessTest"); + var broadcasts = meter.CreateCounter("orleans-dissemination-broadcast-sent"); + var repairs = meter.CreateCounter("orleans-dissemination-anti-entropy-exchanges"); + broadcasts.Add(2); + repairs.Add(3, new KeyValuePair("direction", "out")); + repairs.Add(5, new KeyValuePair("direction", "in")); - [Fact] - public async Task SocketObservation_ReportsActualLoopbackTransfer() - { - var cancellationToken = TestContext.Current.CancellationToken; - using var listener = new TcpListener(IPAddress.Loopback, 0); - listener.Start(); - using var observation = new Observation(captureProvenance: false); - using var client = new TcpClient(); - var accepted = listener.AcceptTcpClientAsync(cancellationToken); - await client.ConnectAsync(IPAddress.Loopback, ((IPEndPoint)listener.LocalEndpoint).Port, cancellationToken); - using var server = await accepted; - await WaitFor(() => observation.SocketSnapshot.Samples > 0, cancellationToken); - var before = observation.SocketSnapshot; - var payload = new byte[4096]; - await client.GetStream().WriteAsync(payload, cancellationToken); - await server.GetStream().ReadExactlyAsync(payload, cancellationToken); - await WaitFor(() => - { - var snapshot = observation.SocketSnapshot; - return snapshot.Sent - before.Sent >= payload.Length && snapshot.Received - before.Received >= payload.Length; - }, cancellationToken); - } - - private static async Task WaitFor(Func predicate, CancellationToken cancellationToken) - { - var clock = Stopwatch.StartNew(); - while (!predicate()) - { - Assert.True(clock.Elapsed < TimeSpan.FromSeconds(8), "The OS socket EventCounters did not report the measured transfer."); - await Task.Delay(50, cancellationToken); - } + Assert.Equal(2, observation.BroadcastsSent); + Assert.Equal(3, observation.OutgoingRepairs); } private sealed record TestPipe(PipeReader Input, PipeWriter Output) : IDuplexPipe; diff --git a/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs b/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs index 0ca8f4d622b..aa7a08e4aff 100644 --- a/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs +++ b/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs @@ -23,29 +23,20 @@ internal sealed class ProcessCluster : IAsyncDisposable private readonly string _membershipDirectory; private int _sequence; - public ProcessCluster(string scenario, bool fastRecovery = true, int siloProcessorCount = 0, int gcConserveMemory = 0) + public ProcessCluster(string scenario, bool fastRecovery = true) { var artifactRoot = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RESULTS") - ?? throw new InvalidOperationException("Run .github/scripts/dissemination-evidence.ps1 first, or set ORLEANS_DISSEMINATION_{OLD,NEW,RESULTS}."); + ?? throw new InvalidOperationException("Run .github/scripts/dissemination-compatibility.ps1 first, or set ORLEANS_DISSEMINATION_{OLD,NEW,RESULTS}."); Directory = Path.Combine(Path.GetFullPath(artifactRoot), $"{scenario}-{Guid.NewGuid():N}"); System.IO.Directory.CreateDirectory(Directory); _membershipDirectory = Path.Combine(Directory, "membership"); System.IO.Directory.CreateDirectory(_membershipDirectory); FastRecovery = fastRecovery; - Assert.InRange(siloProcessorCount, 0, 64); - Assert.InRange(gcConserveMemory, 0, 9); - SiloProcessorCount = siloProcessorCount; - GCConserveMemory = gcConserveMemory; } public string Directory { get; } public bool FastRecovery { get; } - public int SiloProcessorCount { get; } - public int GCConserveMemory { get; } - public IReadOnlyList All => _all; public SiloProcess[] Active => _all.Where(node => !node.Stopped).ToArray(); - public double LastPublicationMilliseconds { get; private set; } - public double LastConvergenceWaitMilliseconds { get; private set; } public async Task Start(string runtime, bool enabled, int? port = null) { @@ -66,7 +57,7 @@ public async Task Start(string runtime, bool enabled, int? port = n var node = new SiloProcess( binaryDirectory, new(name, Path.GetFileName(Directory), _membershipDirectory, port ?? AvailablePort(), Environment.ProcessId, - enabled, FastRecovery, SiloProcessorCount, GCConserveMemory), + enabled, FastRecovery), Directory, manifest); _all.Add(node); @@ -86,24 +77,22 @@ await Eventually("every process has the exact active membership", async () => }); } - public async Task AssertControlRpcs(bool allPairs = true) + public async Task AssertControlRpcs() { var nodes = Active; - for (var index = 0; index < nodes.Length; index++) + foreach (var node in nodes) { - var peers = allPairs ? nodes.Where(peer => !ReferenceEquals(peer, nodes[index])) : [nodes[(index + 1) % nodes.Length]]; - foreach (var peer in peers) + foreach (var peer in nodes.Where(peer => !ReferenceEquals(peer, node))) { - var result = await nodes[index].Send("echo", peer: peer.Last.Address); + var result = await node.Send("echo", peer: peer.Last.Address); Assert.Equal(peer.Last.Identity.ProcessId, result.RemoteProcessId); - Assert.NotEqual(nodes[index].Last.Identity.ProcessId, result.RemoteProcessId); + Assert.NotEqual(node.Last.Identity.ProcessId, result.RemoteProcessId); } } } - public async Task PublishAndConverge() + public async Task PublishAndConverge() { - var clock = Stopwatch.StartNew(); var expected = new Dictionary(StringComparer.Ordinal); foreach (var node in Active) { @@ -113,16 +102,11 @@ public async Task PublishAndConverge() expected.Add(snapshot.Address, snapshot.Load[snapshot.Address]); } - var publicationMilliseconds = clock.Elapsed.TotalMilliseconds; await Eventually("exact production load state at every active process", async () => { var snapshots = await Task.WhenAll(Active.Select(node => node.Send("snapshot"))); return snapshots.All(snapshot => MatchesExpectedLoad(snapshot.Load, expected)); }, expected: expected); - var elapsed = clock.Elapsed.TotalMilliseconds; - LastPublicationMilliseconds = publicationMilliseconds; - LastConvergenceWaitMilliseconds = elapsed - publicationMilliseconds; - return elapsed; } internal static bool MatchesExpectedLoad( @@ -273,22 +257,10 @@ public SiloProcess(string binaryDirectory, NodeConfiguration configuration, stri }, }; _pending[0] = _ready; - if (configuration.SiloProcessorCount > 0) - { - _process.StartInfo.Environment["DOTNET_PROCESSOR_COUNT"] = - configuration.SiloProcessorCount.ToString(System.Globalization.CultureInfo.InvariantCulture); - } - - if (configuration.GCConserveMemory > 0) - { - _process.StartInfo.Environment["DOTNET_GCConserveMemory"] = - configuration.GCConserveMemory.ToString(System.Globalization.CultureInfo.InvariantCulture); - } } public NodeConfiguration Configuration { get; } public NodeSnapshot Last { get; private set; } = null!; - public NodeSnapshot Initial { get; private set; } = null!; public bool Stopped { get; private set; } public bool ForcedKill { get; private set; } public double ShutdownMilliseconds { get; private set; } @@ -303,7 +275,6 @@ public async Task Start() var ready = await _ready.Task.WaitAsync(TimeSpan.FromSeconds(100), TestContext.Current.CancellationToken); Assert.Null(ready.Error); Last = ready.Snapshot!; - Initial = Last; Assert.Equal(_process.Id, Last.Identity.ProcessId); Assert.Equal(_manifest.Runtime, Last.Identity.Runtime); Assert.Equal(_manifest.SourceRevision, Last.Identity.SourceRevision); @@ -314,10 +285,6 @@ public async Task Start() Assert.Equal(_manifest.Runtime == "New", Last.Identity.HasDissemination); Assert.Equal(Configuration.Enabled, Last.Enabled); Assert.Equal(Configuration.Enabled, Last.NamespaceEnabled); - if (Configuration.SiloProcessorCount > 0) - { - Assert.Equal(Configuration.SiloProcessorCount, Last.ProcessorCount); - } var actualDirectory = Path.GetDirectoryName(Path.GetFullPath(Last.Identity.AssemblyPath)); Assert.Equal(Path.GetFullPath(_process.StartInfo.WorkingDirectory), actualDirectory); foreach (var name in new[] { "Orleans.Runtime", "Orleans.Core", "Orleans.Core.Abstractions", "Orleans.Serialization" }) diff --git a/test/Dissemination.IntegrationHarness/Tests/ScalingComparison.cs b/test/Dissemination.IntegrationHarness/Tests/ScalingComparison.cs deleted file mode 100644 index 2e45e1e63a2..00000000000 --- a/test/Dissemination.IntegrationHarness/Tests/ScalingComparison.cs +++ /dev/null @@ -1,106 +0,0 @@ -using System.Globalization; -using System.Text; -using System.Text.Json; - -namespace Orleans.Dissemination.IntegrationHarness; - -internal sealed record ScalingComparison(ScalingPair[] Pairs, int IncompletePairs) -{ - public static ScalingComparison Create(string json) - { - var samples = JsonSerializer.Deserialize(json) - ?? throw new InvalidOperationException("Missing scaling samples."); - var pairs = new List(); - var incomplete = 0; - foreach (var group in samples - .Where(sample => sample.RuntimePath is "CurrentDisabled" or "CurrentEnabledSupported") - .GroupBy(sample => (sample.Size, sample.Scenario, sample.Repetition))) - { - if (group.Count() == 1) - { - incomplete++; - continue; - } - - var disabled = group.Single(sample => sample.RuntimePath == "CurrentDisabled"); - var enabled = group.Single(sample => sample.RuntimePath == "CurrentEnabledSupported"); - Validate(disabled); - Validate(enabled); - if (disabled.Iterations != enabled.Iterations - || disabled.OfferedPublications != enabled.OfferedPublications - || disabled.Runtime.SourceRevision != enabled.Runtime.SourceRevision - || disabled.Runtime.Sha256 != enabled.Runtime.Sha256 - || disabled.Environment.GetRawText() != enabled.Environment.GetRawText()) - { - throw new InvalidOperationException($"Mismatched binary, workload, or environment for {group.Key}."); - } - - pairs.Add(new( - disabled.Size, disabled.Scenario, disabled.Repetition, disabled.Iterations, disabled.OfferedPublications, - disabled.Runtime.SourceRevision, disabled.Runtime.Sha256, disabled.Environment, - ScalingCost.From(disabled), ScalingCost.From(enabled))); - } - - return new(pairs.ToArray(), incomplete); - } - - private static void Validate(ScalingSample sample) - { - if (sample.Size < 3 || sample.LiveSilos != sample.Size || sample.OfferedPublications <= 0 - || sample.Runtime is null || string.IsNullOrEmpty(sample.Runtime.SourceRevision) || string.IsNullOrEmpty(sample.Runtime.Sha256) - || sample.Environment.ValueKind != JsonValueKind.Object - || sample.Iterations < 3 || sample.ConvergenceMilliseconds is null || sample.ConvergenceMilliseconds.Length != sample.Iterations - || sample.TotalRpcs <= 0 || sample.SerializedBytesSent <= 0 || sample.AllocatedBytes <= 0 - || sample.CpuMilliseconds < 0 || sample.ConvergenceMilliseconds.Any(value => !double.IsFinite(value) || value < 0)) - { - throw new InvalidOperationException($"Incomplete scaling measurement for {sample.RuntimePath}, {sample.Size} silos."); - } - } - - public string ToMarkdown() - { - var result = new StringBuilder(); - result.AppendLine("### Dissemination disabled vs enabled"); - result.AppendLine("Each cell is disabled -> enabled. Costs per offered publication include background and control traffic. Single-host, resource-shared measurements; raw totals, environment and per-node data are in the artifacts."); - result.AppendLine(); - result.AppendLine("| Silos / scenario / repetition | RPCs / publication | KiB / publication | Allocated KiB / publication | CPU ms / publication | Median ms | P95 ms |"); - result.AppendLine("|---|---:|---:|---:|---:|---:|---:|"); - foreach (var pair in Pairs) - { - result.AppendLine(FormattableString.Invariant( - $"| {pair.Size} / {pair.Scenario} / {pair.Repetition} | {pair.Disabled.RpcsPerPublication:F2} -> {pair.Enabled.RpcsPerPublication:F2} | {pair.Disabled.SerializedBytesPerPublication / 1024:F2} -> {pair.Enabled.SerializedBytesPerPublication / 1024:F2} | {pair.Disabled.AllocatedBytesPerPublication / 1024:F2} -> {pair.Enabled.AllocatedBytesPerPublication / 1024:F2} | {pair.Disabled.CpuMillisecondsPerPublication:F2} -> {pair.Enabled.CpuMillisecondsPerPublication:F2} | {pair.Disabled.MedianConvergenceMilliseconds:F2} -> {pair.Enabled.MedianConvergenceMilliseconds:F2} | {pair.Disabled.P95ConvergenceMilliseconds:F2} -> {pair.Enabled.P95ConvergenceMilliseconds:F2} |")); - } - - result.AppendLine(); - result.AppendLine($"Incomplete pairs: {IncompletePairs.ToString(CultureInfo.InvariantCulture)}."); - return result.ToString(); - } -} - -internal sealed record ScalingSample( - string RuntimePath, int Size, int LiveSilos, string Scenario, int Repetition, int Iterations, int OfferedPublications, - ScalingBinary Runtime, JsonElement Environment, long TotalRpcs, double SerializedBytesSent, - long AllocatedBytes, double CpuMilliseconds, long RetainedManagedBytes, double[] ConvergenceMilliseconds); - -internal sealed record ScalingBinary(string SourceRevision, string Sha256); - -internal sealed record ScalingPair( - int Size, string Scenario, int Repetition, int Iterations, int OfferedPublications, - string SourceRevision, string Sha256, JsonElement Environment, ScalingCost Disabled, ScalingCost Enabled); - -internal sealed record ScalingCost( - long TotalRpcs, double SerializedBytesSent, long AllocatedBytes, double CpuMilliseconds, long RetainedManagedBytes, - double RpcsPerPublication, double SerializedBytesPerPublication, double AllocatedBytesPerPublication, - double CpuMillisecondsPerPublication, double MedianConvergenceMilliseconds, double P95ConvergenceMilliseconds) -{ - public static ScalingCost From(ScalingSample sample) - { - var ordered = sample.ConvergenceMilliseconds.Order().ToArray(); - var median = (ordered[(ordered.Length - 1) / 2] + ordered[ordered.Length / 2]) / 2; - return new( - sample.TotalRpcs, sample.SerializedBytesSent, sample.AllocatedBytes, sample.CpuMilliseconds, sample.RetainedManagedBytes, - (double)sample.TotalRpcs / sample.OfferedPublications, sample.SerializedBytesSent / sample.OfferedPublications, - (double)sample.AllocatedBytes / sample.OfferedPublications, sample.CpuMilliseconds / sample.OfferedPublications, - median, ordered[(int)Math.Ceiling(0.95 * ordered.Length) - 1]); - } -} diff --git a/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs b/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs deleted file mode 100644 index 39cec4a49cc..00000000000 --- a/test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs +++ /dev/null @@ -1,389 +0,0 @@ -using System.Diagnostics; -using System.Runtime.InteropServices; -using System.Text.Json; -using Xunit; - -namespace Orleans.Dissemination.IntegrationHarness; - -public sealed class ScalingTests -{ - [Fact] - [Trait("Category", "DisseminationScale")] - public async Task Scaling_SelectedRuntimePathsConverge() - { - var sizes = (Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_SIZES") ?? "4,8,16") - .Split(',').Select(value => int.Parse(value, System.Globalization.CultureInfo.InvariantCulture)).ToArray(); - var iterations = Setting("ITERATIONS", 10, 3, 200); - var repetitions = Setting("REPETITIONS", 1, 1, 5); - Assert.All(sizes, size => Assert.InRange(size, 3, 128)); - Assert.InRange(sizes.Length, 1, 6); - Assert.Equal(sizes.Length, sizes.Distinct().Count()); - var scenarios = (Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_SCENARIOS") ?? "stable,churn,partition").Split(','); - Assert.InRange(scenarios.Length, 1, 3); - Assert.Equal(scenarios.Length, scenarios.Distinct().Count()); - Assert.All(scenarios, scenario => Assert.Contains(scenario, new[] { "stable", "churn", "partition" })); - var paths = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RUNTIME_PATHS") ?? "All"; - Assert.Contains(paths, new[] { "All", "Current" }); - - var records = new List(); - var output = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RESULTS")!; - RuntimeVariant[] variants = - [ - new("OriginalLegacy", "Old", Enabled: false), - new("CurrentDisabled", "New", Enabled: false), - new("CurrentEnabledSupported", "New", Enabled: true), - ]; - if (paths == "Current") - { - variants = variants[1..]; - } - - foreach (var size in sizes) - { - foreach (var scenario in scenarios) - { - for (var repetition = 0; repetition < repetitions; repetition++) - { - // Rotate paths to expose order effects across repetitions. - for (var order = 0; order < variants.Length; order++) - { - var variant = variants[(repetition + order) % variants.Length]; - var record = await Measure(size, scenario, variant, iterations, repetition, order); - records.Add(record); - var json = JsonSerializer.Serialize(records, new JsonSerializerOptions { WriteIndented = true }); - await File.WriteAllTextAsync(Path.Combine(output, "cost-results.json"), json, TestContext.Current.CancellationToken); - await File.WriteAllTextAsync(Path.Combine(output, "comparisons.json"), - JsonSerializer.Serialize(ScalingComparison.Create(json), new JsonSerializerOptions { WriteIndented = true }), - TestContext.Current.CancellationToken); - } - } - } - } - - var report = ScalingComparison.Create(JsonSerializer.Serialize(records)); - Assert.Equal(0, report.IncompletePairs); - Assert.Equal(sizes.Length * scenarios.Length * repetitions, report.Pairs.Length); - if (Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY") is { Length: > 0 } summary) - { - await File.AppendAllTextAsync(summary, report.ToMarkdown(), TestContext.Current.CancellationToken); - } - } - - private static async Task Measure(int size, string scenario, RuntimeVariant variant, int iterations, int repetition, int order) - { - var enabled = variant.Enabled; - var siloProcessorCount = Setting("SILO_PROCESSOR_COUNT", 0, 0, 64); - var gcConserveMemory = Setting("GC_CONSERVE_MEMORY", 0, 0, 9); - var bootstrap = Stopwatch.StartNew(); - await using var cluster = new ProcessCluster( - $"scale-{size}-{scenario}-{variant.Name}-{repetition}", fastRecovery: false, siloProcessorCount, gcConserveMemory); - for (var index = 0; index < size; index++) - { - await cluster.Start(variant.Runtime, enabled); - } - - await cluster.Stabilize(); - await cluster.AssertControlRpcs(allPairs: false); - await cluster.PublishAndConverge(); - await cluster.PublishAndConverge(); - if (enabled) - { - await ConfirmSupport(cluster); - } - - await cluster.Save("warmup.json", await Task.WhenAll(cluster.Active.Select(node => node.Send("measure")))); - // Socket EventCounters arrive in one-second buckets. Every path has the same two-second margins. - await Task.Delay(TimeSpan.FromSeconds(2), TestContext.Current.CancellationToken); - var start = (await Task.WhenAll(cluster.Active.Select(node => node.Send("measure")))) - .ToDictionary(snapshot => snapshot.Identity.ProcessId); - Assert.Equal(size, start.Count); - Assert.All(start.Values, snapshot => Assert.Equal(size, snapshot.ActiveMembers.Length)); - var bootstrapMilliseconds = bootstrap.Elapsed.TotalMilliseconds; - var startedAt = DateTimeOffset.UtcNow; - var clock = Stopwatch.StartNew(); - var latencies = new List(); - var publicationLatencies = new List(); - var convergenceWaits = new List(); - var recoveryLatencies = new List(); - var offeredPublications = 0; - var restarts = 0; - var partitions = 0; - for (var iteration = 0; iteration < iterations; iteration++) - { - if (scenario == "churn" && iteration > 0 && iteration % 2 == 0) - { - var victim = cluster.Active[iteration % size]; - await victim.Stop(); - // Reuse the endpoint to exercise a real new silo incarnation and stale-generation cleanup. - await cluster.Start(variant.Runtime, enabled, victim.Configuration.SiloPort); - await cluster.Stabilize(); - if (enabled) - { - // Bootstrap/fallback work stays inside the churn window, but offered rounds use supported peers. - await ConfirmSupport(cluster); - } - - restarts++; - } - else if (scenario == "partition" && iteration % 3 == 1) - { - var victim = cluster.Active[iteration % size]; - await victim.Send("partition", value: true); - partitions++; - foreach (var source in cluster.Active.Where(node => !ReferenceEquals(node, victim))) - { - await source.Send("publish"); - offeredPublications++; - } - - var recovery = Stopwatch.StartNew(); - // Establish delivery paths before offering the next sample, including on the one-way legacy path. - await cluster.HealPartition(victim); - await PublishRound(); - recoveryLatencies.Add(recovery.Elapsed.TotalMilliseconds); - continue; - } - - await PublishRound(); - } - - async Task PublishRound() - { - latencies.Add(await cluster.PublishAndConverge()); - publicationLatencies.Add(cluster.LastPublicationMilliseconds); - convergenceWaits.Add(cluster.LastConvergenceWaitMilliseconds); - offeredPublications += size; - } - - await Task.Delay(TimeSpan.FromSeconds(2), TestContext.Current.CancellationToken); - await Task.WhenAll(cluster.Active.Select(node => node.Send("measure"))); - var elapsed = clock.Elapsed.TotalMilliseconds; - var finishedAt = DateTimeOffset.UtcNow; - var finish = cluster.All.Select(node => node.Last).ToArray(); - var retained = await Task.WhenAll(cluster.Active.Select(node => node.Send("retained-memory"))); - var retainedByPid = retained.ToDictionary(snapshot => snapshot.Identity.ProcessId, snapshot => snapshot.ManagedHeapBytes); - Assert.Equal(scenario == "churn" ? (iterations - 1) / 2 : 0, restarts); - Assert.Equal(scenario == "partition" ? (iterations + 1) / 3 : 0, partitions); - if (enabled) - { - Assert.All(start.Values, snapshot => Assert.Empty(snapshot.UnconfirmedPeers)); - Assert.All(cluster.Active, node => Assert.Empty(node.Last.UnconfirmedPeers)); - } - - var nodes = finish.Select(snapshot => - { - start.TryGetValue(snapshot.Identity.ProcessId, out var baseline); - var allMessages = Delta(snapshot, baseline, "orleans-messaging-sent-messages-size|"); - var requestMessages = Delta(snapshot, baseline, "orleans-messaging-sent-messages-size|", "MessageDirection=Request"); - var oneWayMessages = Delta(snapshot, baseline, "orleans-messaging-sent-messages-size|", "MessageDirection=OneWay"); - var received = Delta(snapshot, baseline, "orleans-messaging-received-messages-size|"); - var topology = snapshot.TopologyMembers.Contains(snapshot.Address) - ? snapshot - : baseline ?? cluster.All.Single(node => node.Last.Identity.ProcessId == snapshot.Identity.ProcessId).Initial; - var index = Array.IndexOf(topology.TopologyMembers, snapshot.Address); - return new NodeCost( - snapshot.Identity.ProcessId, - snapshot.Address, - index >= 0 ? index : null, - index >= 0 && (topology.AggregationTree ? index <= topology.Fanout : index < topology.Fanout), - topology.OriginatorTargets, - topology.ForwardingTargets, - requestMessages.Count + oneWayMessages.Count, - allMessages.Count, - allMessages.Sum, - received.Count, - received.Sum, - snapshot.TransportBytesWritten - (baseline?.TransportBytesWritten ?? 0), - snapshot.TransportBytesRead - (baseline?.TransportBytesRead ?? 0), - snapshot.SocketBytesSent - (baseline?.SocketBytesSent ?? 0), - snapshot.SocketBytesReceived - (baseline?.SocketBytesReceived ?? 0), - snapshot.SocketCounterSamples - (baseline?.SocketCounterSamples ?? 0), - snapshot.CpuMilliseconds - (baseline?.CpuMilliseconds ?? 0), - snapshot.AllocatedBytes - (baseline?.AllocatedBytes ?? 0), - retainedByPid.GetValueOrDefault(snapshot.Identity.ProcessId), - snapshot.WorkingSetBytes, - snapshot.PrivateBytes, - Delta(snapshot, baseline, "orleans-dissemination-bytes-sent|").Sum, - Delta(snapshot, baseline, "orleans-dissemination-broadcast-sent|").Sum, - Delta(snapshot, baseline, "orleans-dissemination-anti-entropy-exchanges|", "direction=out").Sum, - Delta(snapshot, baseline, "orleans-dissemination-broadcast-sent|", "namespace=load").Sum, - Delta(snapshot, baseline, "orleans-dissemination-values-sent|", "namespace=load").Sum); - }).ToArray(); - Assert.All(nodes, node => - { - Assert.True(node.SentMessages > 0, "Orleans message histogram evidence is missing."); - Assert.True(node.SerializedBytesSent > 0, "Serialized byte evidence is missing."); - Assert.True(node.SocketSamples > 0 && node.SocketBytesSent > 0, "OS socket byte evidence is missing."); - }); - if (!enabled) - { - Assert.All(nodes, node => - { - Assert.Equal(0, node.DisseminationBroadcasts); - Assert.Equal(0, node.AntiEntropyExchanges); - }); - } - else - { - Assert.True(nodes.Sum(node => node.DisseminationBroadcasts) > 0, - "The supported-enabled cost window did not exercise any dissemination broadcasts."); - } - - var record = new - { - Schema = 1, - Profile = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_PROFILE") ?? "Standalone", - RuntimePath = variant.Name, - Runtime = finish[0].Identity, - Size = size, - LiveSilos = start.Count, - Scenario = scenario, - Enabled = enabled, - Iterations = iterations, - Repetition = repetition, - ExecutionOrder = order, - OfferedPublications = offeredPublications, - RestartedProcesses = restarts, - InjectedPartitions = partitions, - WindowMilliseconds = elapsed, - WindowStartedAtUtc = startedAt, - WindowFinishedAtUtc = finishedAt, - ConvergenceMilliseconds = latencies, - PublicationMilliseconds = publicationLatencies, - ConvergenceAfterPublicationMilliseconds = convergenceWaits, - PartitionRecoveryMilliseconds = recoveryLatencies, - TotalRpcs = nodes.Sum(node => node.Rpcs), - TotalMessages = nodes.Sum(node => node.SentMessages), - LoadBroadcastRequests = nodes.Sum(node => node.LoadBroadcastRequests), - LoadValueTransmissions = nodes.Sum(node => node.LoadValueTransmissions), - LoadBroadcastRequestsPerRound = nodes.Sum(node => node.LoadBroadcastRequests) / iterations, - LoadBroadcastRequestsPerSecond = nodes.Sum(node => node.LoadBroadcastRequests) / (elapsed / 1000), - SerializedBytesSent = nodes.Sum(node => node.SerializedBytesSent), - SocketBytesSent = nodes.Sum(node => node.SocketBytesSent), - TransportBytesSubmitted = nodes.Sum(node => node.TransportBytesSubmitted), - CpuMilliseconds = nodes.Sum(node => node.CpuMilliseconds), - AllocatedBytes = nodes.Sum(node => node.AllocatedBytes), - RetainedManagedBytes = nodes.Sum(node => node.RetainedManagedBytes), - BootstrapMilliseconds = bootstrapMilliseconds, - BootstrapCpuMilliseconds = start.Values.Sum(snapshot => snapshot.CpuMilliseconds), - BootstrapAllocatedBytes = start.Values.Sum(snapshot => snapshot.AllocatedBytes), - Nodes = nodes, - Environment = new - { - RuntimeInformation.OSDescription, - RuntimeInformation.FrameworkDescription, - System.Environment.ProcessorCount, - SiloProcessorCount = siloProcessorCount, - GCConserveMemory = gcConserveMemory, - WorkerProcessorCounts = start.Values.Select(snapshot => snapshot.ProcessorCount).Distinct().Order().ToArray(), - WorkerServerGC = start.Values.Select(snapshot => snapshot.ServerGC).Distinct().Order().ToArray(), - }, - Methodology = new - { - WarmupRounds = 2, - SettlingMarginSeconds = 2, - AutomaticLoadPublisherPaused = true, - ProductionDisseminationDefaults = variant.Runtime == "New" && enabled, - ExplicitDisseminationOptOut = variant.Runtime == "New" && !enabled, - LoadNamespaceSupportConfirmedBeforeOfferedRounds = enabled, - IncludesUnconfirmedBootstrapTraffic = enabled && scenario == "churn", - LivenessFailureDetectionEnabled = false, - StorageProvider = "locked local JSON file; no external-service network", - Window = "after warmup/capability discovery, through offered load, churn/restarts and final 2-second drain; GC retained-size probe is outside CPU/allocation window", - RpcDefinition = "sent Orleans message-size observations tagged MessageDirection=Request OR MessageDirection=OneWay; not response messages", - SerializedBytes = "complete serialized Orleans messages (headers and bodies), not dissemination payload counters", - SocketBytes = "System.Net.Sockets bytes-sent/bytes-received EventCounters, 1-second sampling; cumulative values differenced across the window (incremental counters also supported); excludes TCP/IP headers and retransmits", - TransportBytes = "bytes advanced at the real connection transport pipe, including handshake; may include buffered bytes lost when a partition aborts a connection", - RetainedMemory = "full GC live managed bytes on surviving nodes only; not a peak; RSS/private bytes and retired-node snapshots also retained", - Includes = "all silo work during the window, including control/snapshot/instrumentation overhead and legacy fallback; restart startup/shutdown costs included for churn", - Observation = "identical state polling on selected paths; detailed counters only at window boundaries; per-value DiagnosticListener capture disabled in scale runs", - Topology = "per-node actual tree at end, or start for a retired node; raw command snapshots preserve intervening topology changes; role is not time-weighted", - Excludes = "controller process CPU/allocation, storage disk byte accounting, cross-machine latency, NIC/TCP wire overhead, production storage/placement/grain work", - PartitionComparison = "retire partitioned connections at both endpoints, then confirm healed connections with bidirectional acknowledged control RPCs before the same single publication round on all paths; teardown and readiness remain inside partition recovery time and total cost; silent anti-entropy recovery is asserted separately in VersionSkewTests", - OriginalBaselineAttribution = "original-to-current disabled deltas include all intervening source changes, including lifecycle prerequisites and load notification/tombstone/dedup changes; only current off/on isolates enabling the subsystem on the same binary", - Interpretation = "separate original-legacy, current-explicitly-disabled and current-enabled-supported measurements; enabled defaults are temporary for pre-merge testing", - }, - }; - await cluster.Save("window-start.json", start); - await cluster.Save("window-finish.json", finish); - await cluster.Save("cost.json", record); - return record; - } - - private static Task ConfirmSupport(ProcessCluster cluster) - { - var timeout = TimeSpan.FromSeconds(90); - foreach (var node in cluster.Active) - { - Assert.True(node.Last.AntiEntropyPeerCount > 0); - Assert.True(node.Last.AntiEntropyIntervalMilliseconds > 0); - var rounds = Math.Ceiling((cluster.Active.Length - 1.0) / node.Last.AntiEntropyPeerCount); - var rotationBudget = TimeSpan.FromMilliseconds(2 * rounds * node.Last.AntiEntropyIntervalMilliseconds) - + TimeSpan.FromSeconds(60); - if (rotationBudget > timeout) - { - timeout = rotationBudget; - } - } - - return cluster.Eventually("all enabled peers confirm support before offered publication rounds", async () => - { - var snapshots = await Task.WhenAll(cluster.Active.Select(node => node.Send("snapshot"))); - return snapshots.All(snapshot => snapshot.UnconfirmedPeers.Length == 0); - }, timeout); - } - - private sealed record RuntimeVariant(string Name, string Runtime, bool Enabled); - - private static MetricValue Delta(NodeSnapshot value, NodeSnapshot? baseline, string prefix, string? tag = null) - { - var entries = value.Metrics.Where(pair => pair.Key.StartsWith(prefix, StringComparison.Ordinal) - && (tag is null || pair.Key.Contains(tag, StringComparison.Ordinal))); - long count = 0; - double sum = 0; - foreach (var (key, metric) in entries) - { - var previous = baseline?.Metrics.GetValueOrDefault(key); - count += metric.Count - (previous?.Count ?? 0); - sum += metric.Sum - (previous?.Sum ?? 0); - } - - return new(count, sum); - } - - private static int Setting(string name, int defaultValue, int minimum, int maximum) - { - var value = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_" + name); - var result = value is null ? defaultValue : int.Parse(value, System.Globalization.CultureInfo.InvariantCulture); - Assert.InRange(result, minimum, maximum); - return result; - } - - private sealed record NodeCost( - int ProcessId, - string Address, - int? TreeRank, - bool UpperTree, - string[] OriginatorTargets, - string[] ForwardingTargets, - long Rpcs, - long SentMessages, - double SerializedBytesSent, - long ReceivedMessages, - double SerializedBytesReceived, - long TransportBytesSubmitted, - long TransportBytesRead, - double SocketBytesSent, - double SocketBytesReceived, - long SocketSamples, - double CpuMilliseconds, - long AllocatedBytes, - long RetainedManagedBytes, - long WorkingSetBytes, - long PrivateBytes, - double DisseminationPayloadBytes, - double DisseminationBroadcasts, - double AntiEntropyExchanges, - double LoadBroadcastRequests, - double LoadValueTransmissions); -} diff --git a/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs index be2f90fec39..45da2189724 100644 --- a/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs +++ b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Text.Json; using Xunit; @@ -8,34 +9,51 @@ namespace Orleans.Dissemination.IntegrationHarness; public sealed class VersionSkewTests { [Theory] - [InlineData("Old", false)] - [InlineData("New", false)] - [InlineData("New", true)] + [InlineData("Old", false, 4)] + [InlineData("Old", false, 8)] + [InlineData("New", false, 4)] + [InlineData("New", false, 8)] + [InlineData("New", true, 4)] + [InlineData("New", true, 8)] [Trait("Category", "DisseminationProcess")] - public async Task PartitionHealing_ConfirmsTransportBeforeSinglePublicationRound(string runtime, bool enabled) + public async Task PartitionHealing_ConfirmsTransportBeforeSinglePublicationRound(string runtime, bool enabled, int size) { - await using var cluster = new ProcessCluster($"partition-publication-{runtime}-{enabled}", fastRecovery: false); - var sender = await cluster.Start(runtime, enabled); - var receiver = await cluster.Start(runtime, enabled); + await using var cluster = new ProcessCluster($"partition-publication-{runtime}-{enabled}-{size}", fastRecovery: false); + for (var index = 0; index < size; index++) + { + await cluster.Start(runtime, enabled); + } + + var nodes = cluster.Active; await cluster.Stabilize(); await cluster.AssertControlRpcs(); await cluster.PublishAndConverge(); for (var iteration = 0; iteration < 3; iteration++) { + var receiver = nodes[iteration % size]; await receiver.Send("partition", value: true); if (iteration == 0) { - var probe = await sender.Send("probe-echo", peer: receiver.Last.Address); + var probe = await nodes[1].Send("probe-echo", peer: receiver.Last.Address); Assert.Null(probe.RemoteProcessId); Assert.NotNull(probe.ProbeError); } - await sender.Send("publish"); + foreach (var source in nodes.Where(node => !ReferenceEquals(node, receiver))) + { + await source.Send("publish"); + } + + var recovery = Stopwatch.StartNew(); await cluster.HealPartition(receiver); await cluster.PublishAndConverge(); Assert.All(cluster.Active, node => Assert.False(node.Last.Partitioned)); - await cluster.Save($"healed-publication-{iteration}.json", cluster.Active.Select(node => node.Last)); + await cluster.Save($"healed-publication-{iteration}.json", new + { + RecoveryMilliseconds = recovery.Elapsed.TotalMilliseconds, + Nodes = cluster.Active.Select(node => node.Last), + }); } } @@ -113,10 +131,8 @@ await cluster.Eventually("enabled peers confirm support, explicitly disabled pee await disabled.Send("snapshot"); Assert.False(disabled.Last.Enabled); Assert.False(disabled.Last.NamespaceEnabled); - Assert.Equal(0, Sum(disabled.Last, "orleans-dissemination-broadcast-sent|")); - Assert.Equal(0, disabled.Last.Metrics.Where(pair => - pair.Key.StartsWith("orleans-dissemination-anti-entropy-exchanges|", StringComparison.Ordinal) - && pair.Key.Contains("direction=out", StringComparison.Ordinal)).Sum(pair => pair.Value.Sum)); + Assert.Equal(0, disabled.Last.BroadcastsSent); + Assert.Equal(0, disabled.Last.OutgoingRepairs); Assert.Empty(disabled.Last.Applies); // The same original binary is reintroduced, not a flag on the new binary. @@ -359,6 +375,4 @@ private static async Task AssertNoTreeAdmissions(ProcessCluster cluster, Diction } } - internal static double Sum(NodeSnapshot snapshot, string prefix) => - snapshot.Metrics.Where(pair => pair.Key.StartsWith(prefix, StringComparison.Ordinal)).Sum(pair => pair.Value.Sum); } diff --git a/test/Dissemination.IntegrationHarness/methodology.json b/test/Dissemination.IntegrationHarness/methodology.json deleted file mode 100644 index f1950cabcc2..00000000000 --- a/test/Dissemination.IntegrationHarness/methodology.json +++ /dev/null @@ -1,170 +0,0 @@ -{ - "schema": 1, - "purpose": "Executable real-binary compatibility/recovery gates and controlled load evidence, not a microbenchmark or a performance claim.", - "entrypoint": ".github/scripts/dissemination-evidence.ps1", - "workflow": ".github/workflows/dissemination-evidence.yml", - "automaticPrTrigger": "pull_request and merge_group run both the pinned-binary version-skew gate and measured bounded original-legacy, current-explicitly-disabled and current-enabled-supported costs. The bounded job reuses published binaries, needs no label or workflow_dispatch registration, and has a 25-minute hard limit.", - "controlledScaleTrigger": "The dissemination-scale PR label or workflow_dispatch with scale=true runs an independent job per size, defaulting to 8,16,32,64,100 real silo processes. Remove the label after collecting evidence to keep subsequent pushes on the small profile. Jobs use read-only permissions.", - "sliceInventory": [ - ".github/scripts/dissemination-evidence.ps1", - ".github/workflows/dissemination-evidence.yml", - "test/Dissemination.IntegrationHarness/methodology.json", - "test/Dissemination.IntegrationHarness/Shared/Protocol.cs", - "test/Dissemination.IntegrationHarness/Silo/ControlTarget.cs", - "test/Dissemination.IntegrationHarness/Silo/Dissemination.Silo.csproj", - "test/Dissemination.IntegrationHarness/Silo/FileMembershipTable.cs", - "test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs", - "test/Dissemination.IntegrationHarness/Silo/Observation.cs", - "test/Dissemination.IntegrationHarness/Silo/Program.cs", - "test/Dissemination.IntegrationHarness/Tests/Dissemination.IntegrationHarness.Tests.csproj", - "test/Dissemination.IntegrationHarness/Tests/MeasurementTests.cs", - "test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs", - "test/Dissemination.IntegrationHarness/Tests/ScalingTests.cs", - "test/Dissemination.IntegrationHarness/Tests/ScalingComparison.cs", - "test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs" - ], - "runtimeDependencies": { - "build": "Existing Orleans.TestingHost project, repository code generation, net10.0 SDK, existing MTP/xUnit packages; controller uses the installed ASP.NET Core shared framework for instrumentation tests.", - "bothBinaries": "TestClusterHostFactory; IMembershipTable; IMembershipManager.CurrentSnapshot/Refresh; DeploymentLoadPublisher.PeriodicStatistics; IInternalGrainFactory/SystemTarget/SystemTargetShared; stable test-only aliased control RPCs.", - "reflectionSeams": "DeploymentLoadPublisher.PublishStatistics(CancellationToken) and its IDisposable _publishTimer field, verified against the pinned source and current source. Publication, timer pause and deliberate cache-loss injection run on the publisher scheduler. Pausing disposes the timer and retains its reference for normal lifecycle cleanup.", - "lifecycleSignature": "GatedMembershipGossiper compiles against the original no-token IMembershipGossiper method for Old, and the required CancellationToken method on the current dissemination top for New.", - "newOnly": "DisseminationOptions and namespace options; IDisseminationService/IDisseminationSystemTarget; MembershipDisseminationNamespace and DeploymentLoadStatisticsDisseminationNamespace; DisseminationMembership topology and bounded repair contracts. Enabled recovery hosts register a test-only incoming-call filter which gates only PushBroadcast, never ExchangeAntiEntropy.", - "diagnostics": "Existing Microsoft.Orleans.Dissemination ValueApply event, Microsoft.Orleans messaging/dissemination meters, and System.Net.Sockets counters. No additional production diagnostic API is required.", - "notRequired": "No cluster-manifest hash RPC, and no direct access to notification locks, tombstone bookkeeping or dedup internals.", - "pendingProviderApi": "The candidate FileMembershipTable still targets the confirmed pinned/current interface. Token-bearing provider implementations and the conditional legacy bridge will be updated after the provider owner confirms the new API shape; no prospective overload signatures are assumed." - }, - "baseline": { - "repository": "dotnet/orleans", - "commit": "6739589254b746a8790cf53524e6abe372bb53d4", - "build": "Separate CI checkout. Only Silo and Shared test sources are copied into it. Its runtime source tree must be pristine and must lack dissemination.", - "identity": "Independent publish directory and dotnet OS process per node. Assert PID, runtime SHA256, assembly name/version, informational commit, MVID difference, and presence/absence of the dissemination target. Every loaded Orleans assembly must match its published SHA/version and live in that process's binary directory; Runtime, Core, Core.Abstractions and Serialization must all be present.", - "assemblyVersion": "Numeric assembly versions can legitimately be equal. They are checked against build manifests, not falsely required to differ.", - "controller": "MTP/xUnit controller references no Orleans assembly. The separate silo executable reuses the existing Orleans.Runtime.Tests friend identity." - }, - "featureBoundary": { - "initialLifecycleBaseCommit": "789e8cc27394b78e9a04887409a4fcec965b1b4f", - "new": "Build the current dissemination top, including its main-based lifecycle-prerequisite commits.", - "old": "Always build the exact pinned pre-feature main commit; do not apply the lifecycle prerequisite or dissemination patches to it.", - "manifestHashRpc": "No test uses a cluster-manifest hash RPC. Binary manifest.json files and local DLL SHA256 checks only attest which assemblies each OS process loaded; they are unrelated to the excluded manifest-hash feature.", - "compatibility": "Probe the actual IDisseminationSystemTarget RPC on an old process, then require a fresh exact production load sample through legacy delivery while that process remains unconfirmed.", - "sliceOwnership": "All harness and CI files belong to the dissemination/evidence slice. The current dissemination top inherits the lifecycle prerequisites and applies load updates on the publisher scheduler while preserving explicitly disabled notification behavior. The pinned old source retains its original runtime implementation." - }, - "infrastructure": { - "hosting": "Existing TestClusterHostFactory, SystemTarget and code generation; OS process orchestration follows StandaloneSiloHandle without its single-binary configuration assumption.", - "membership": "Small JSON provider with inter-process exclusive file locking, atomic replacement, version/row ETags, heartbeat-only updates and per-node frozen read snapshots.", - "control": "Stable aliased Orleans system-target Echo and Hold RPCs with required CancellationToken parameters. Local test commands use redirected stdin/stdout, not an exposed control server.", - "partition": "Abort existing real silo connections and reject new inbound/outbound connections at connection middleware. Cost recovery drains captured middleware and connection-close tasks at both endpoints, including unidentified inbound handshakes, before reopening and confirming acknowledged control RPCs.", - "scope": "One machine, loopback TCP, real serializer/transport, real membership/load namespaces. Liveness failure detection is disabled to keep partition repair distinct from membership eviction." - }, - "gates": [ - { - "test": "PartitionHealing_ConfirmsTransportBeforeSinglePublicationRound", - "assertions": "Repeat three partition/heal cycles on pinned old, current explicitly disabled and current enabled binaries. The first cycle proves an acknowledged control probe fails during the partition; remaining cycles publish and heal immediately. Healing retires old connections at both endpoints while the isolated endpoint rejects new handshakes, then bidirectional control RPCs confirm each exact remote PID. Full load snapshots must converge after one production publication per node with automatic publishers paused." - }, - { - "test": "PinnedBinaries_RollForward_MixedEnablement_AndRollback", - "assertions": "Three pinned old processes are replaced one at a time by one explicitly disabled and two enabled new processes, then rolled back to pinned old processes. Exact load state equality and stable cross-process RPC PID identity are checked at each stage. Unsupported old IDisseminationSystemTarget RPC failures cannot be inferred from timeouts and must be followed by fresh exact load-state delivery to the still-unconfirmed old process. Disabled new nodes return explicit namespace-unsupported and retain legacy delivery. Enabled peers confirm support. Disabled nodes have no dissemination sends, applies or outgoing anti-entropy." - }, - { - "test": "PartitionHistoryAndHeartbeat_RequireDisseminationProvenance", - "assertions": "Freeze central-table reads and suppress legacy membership gossip on all nodes. Close the test-only native PushBroadcast incoming-call gate on every node and await all previously admitted PushBroadcast invocations before creating the history gap. Probe the actual RPC to verify rejection. Keep ExchangeAntiEntropy enabled. Isolate both downstream processes while the origin sees 40 actual table versions, evicting the receiver baseline from its 32-entry history. Heal the first receiver while the relay stays isolated, so its repair must come from the history-missing origin and produce FromVersion=0. After observing exact state and full-repair provenance, heal the relay and require every node to converge. Keep tree gates closed with unchanged admitted-call counts for the history, heartbeat and missing-load assertions. Restore table reads, gossip and network/tree gates in bounded cleanup even when an assertion fails.", - "nonClaims": "TTL expiry is not queue expiry and is not used as a tree-drain proof. FromVersion=0 alone does not prove anti-entropy. Legacy gossip, central-table reads and all newly admitted PushBroadcast calls are excluded throughout these assertions." - }, - { - "test": "RuntimeStatisticsComparison_ObservesPublicReadonlyFieldMutation", - "assertions": "Execute a mutation probe inside both the pinned old and current OS processes using the real EnvironmentStatistics type. Changing only RawCpuUsagePercentage must change the shared comparison representation; every reflected public instance field must be present." - }, - { - "test": "RequiredCancellationAndPartitionedShutdown_AreBounded", - "assertions": "Start a required-token RPC from the sender's system-target scheduler, retain its CTS across commands, and wait for an explicit receiver snapshot proving one pending started call with a live uncanceled token before issuing cancellation. Invoke CancelAsync, await the actual raw RPC task using an independent command deadline, then require exactly one receiver-observed cancellation and no pending server call. Save start/cancel timestamps and raw-task status; do not accept only cancellation of a local WaitAsync. A partitioned dissemination sender must shut down without controller kill inside 35 seconds." - } - ], - "focusedRegressions": [ - { - "test": "ConnectionDrain_WaitsForInitializationAndCloseBeforeReopening", - "scope": "The partition captures middleware before initialization completes, rejects new admissions, and waits for both middleware exit and connection close before reopening." - }, - { - "test": "ConnectionDrain_CapturesUnidentifiedPeersAndPreservesHealthyConnections", - "scope": "Peer-specific drain includes unidentified handshakes, preserves healthy connections, and retains admission blocking when the drain caller is canceled." - }, - { - "test": "ControlledCall_WaitsForRawCancellationAfterExplicitSignal", - "scope": "The durable controller call has no automatic timer, retains a live token until explicit cancellation, and cannot complete merely because cancellation was requested while its raw task is still pending." - }, - { - "test": "ControlledCall_DoesNotTreatRawTimeoutAsCancellation", - "scope": "A raw RPC timeout after cancellation is requested remains a failure, rather than being mistaken for successful cooperative cancellation." - }, - { - "test": "StateComparison_ObservesNestedPublicReadonlyFieldChanges", - "scope": "Lightweight controller regression for the exact cached IncludeFields serializer used by the host load-state comparison. Demonstrates that the previous properties-only serializer misses the mutation." - }, - { - "test": "InFlightCallGate_DrainsPriorCallsAndRejectsLaterCalls", - "scope": "Lightweight regression for the admission/drain primitive used by the native PushBroadcast filter. A prior active call delays the barrier; subsequent calls cannot enter before or after drain until explicitly reopened." - } - ], - "load": { - "test": "Scaling_SelectedRuntimePathsConverge", - "comparison": "Three separately labeled paths: pinned original legacy binary; actual current binary with dissemination and both namespaces explicitly disabled; the same current binary using temporarily enabled defaults after peers have confirmed support. Enabled defaults provide broader pre-merge test exposure.", - "runtimePaths": [ - { "name": "OriginalLegacy", "runtime": "Old", "enabled": false }, - { "name": "CurrentDisabled", "runtime": "New", "enabled": false }, - { "name": "CurrentEnabledSupported", "runtime": "New", "enabled": true } - ], - "automaticPrProfile": { "mode": "Cost", "sizes": [4, 8], "iterations": 3, "pairedRepetitions": 1, "overrideable": false }, - "largeScaleDefaults": { "mode": "Scale", "sizes": [8, 16, 32, 64, 100], "iterations": 10, "pairedRepetitions": 2, "runtimePaths": "Current", "scenarios": ["stable"], "siloProcessorCount": 1, "gcConserveMemory": 9 }, - "controls": { "sizes": "1..6 distinct comma-separated sizes, each 3..128", "iterations": "3..200", "pairedRepetitions": "1..5", "runtimePaths": "All or Current", "scenarios": "distinct stable, churn, partition selections" }, - "largeScaleResources": "All sizes in the large curve use the same one-logical-processor and GCConserveMemory=9 child-process policy on standard shared-host CI capacity. Effective worker processor count and server-GC mode are recorded and processor count is asserted. Linux preflight requires available memory of 128 MiB per requested silo plus 1 GiB controller headroom, and saves host-resources.json. This is a planning guard, not a memory cap or proof against resource exhaustion. An inadequate runner fails before starting the measurement.", - "largeScaleScope": "Large jobs compare the identical current binary with dissemination explicitly off versus enabled defaults, with order reversed on the second repetition. The ordinary 4/8-silo three-path stable/churn/partition profile remains unchanged. Script options support additional scenarios on a suitably provisioned host.", - "comparisonReports": "comparisons.json and GitHub job summaries pair equal-size/scenario/repetition current-off/current-on samples. Pair validation checks actual live silo count, identical binary hash/revision, environment, iterations and offered publications. Reports include raw totals, costs per publication, median/P95 convergence, retained managed memory, and explicit incomplete-pair counts.", - "scenarios": "Stable; graceful stop/restart at the same endpoint every other round; real connection partition and heal with old pooled connections retired at both endpoints before reopening and confirming transport readiness via bidirectional acknowledged control RPCs. Teardown, readiness probes and retries share a 60-second deadline and remain inside recovery latency and total cost. Each path then offers the same single publication round. Gate artifacts separately cover silent repair, retained-history misses and same-version heartbeat recovery.", - "cadence": "Pause automatic deployment-load timers after startup. Invoke the actual production PublishStatistics method once per active node per offered round, then require exact dictionary inventory and values using cached JsonSerializerOptions with IncludeFields=true, including every public readonly EnvironmentStatistics field. Extra departed or restarted generations fail the same comparison as missing or mismatched values.", - "window": "Two warmup publication rounds and complete enabled-peer capability discovery before measuring. Discovery has a minimum 90-second controller budget, expanded to two full peer rotations plus 60 seconds using the actual advertised repair cadence and peer count. Production repair settings stay unchanged. Startup/warmup/preparation duration, CPU and allocations are reported separately. Replacements also finish capability discovery before offered rounds; their bootstrap/fallback/discovery costs remain inside the churn window. Two-second OS-counter sampling margins on all paths. Full-GC retained-memory probes are after the CPU/allocation window.", - "metrics": [ - "All transmitted request AND one-way RPC messages; responses reported in the separate total-message count", - "Load-namespace tree requests and logical value transmissions separately, including load requests per completed publication round and per measured window second", - "Complete serialized Orleans message bytes, including headers/body, from message-size observations", - "Actual process-wide System.Net.Sockets sent/received bytes, sampled each second; transport-pipe submitted/read bytes separately", - "Process CPU time, GC total allocated bytes, post-full-GC retained managed bytes, working set and private bytes", - "Per-round exact state-convergence latency and partition recovery latency", - "Separate per-round publication duration and residual convergence wait, retaining the original end-to-end latency and 100 ms polling cadence", - "Per-node counters, actual originator/forwarding targets, tree rank and upper-tree role; load sends directly to one root, batches there for 500 ms, and atomically forwards whole batches down the tree immediately; raw snapshots preserve topology changes", - "Raw protocol payload, namespace-batch and anti-entropy counters, never substituted for total RPC or byte counts" - ], - "limitations": [ - "Socket counters exclude TCP/IP headers and retransmissions; transport submissions can include bytes buffered then lost on an aborted connection.", - "One-second socket-counter sampling and non-atomic cross-process snapshots impose boundary uncertainty. Raw captured UTC timestamps are included.", - "Instrumentation and stdin/state-snapshot overhead are included. Detailed counters are serialized only at cost boundaries and per-value diagnostic events are disabled for scale runs.", - "Controller CPU/allocation, disk byte costs, production providers, application grains, cross-machine latency and NIC-level wire costs are not measured. This is a closed-loop control-plane workload with a convergence barrier each round, not a saturation-throughput benchmark.", - "Retained managed memory is a post-GC live-size sample on surviving nodes, not peak memory. Retired process snapshots and all-node RSS/private-memory samples remain in artifacts.", - "Normal partition cost comparisons include a transport-readiness barrier followed by the same single publication round. The separate anti-entropy-only gate measures silent repair.", - "Original-to-current disabled deltas include the lifecycle prerequisites and feature registration costs. The current off/on pair uses the identical current binary and isolates enabling the subsystem.", - "Raw samples and repetitions are evidence inputs, not predeclared improvement thresholds or broad-enable approval." - ] - }, - "bounds": { - "startupSeconds": 90, - "controllerStartupSeconds": 100, - "siloCommandSeconds": 60, - "controllerCommandSeconds": 70, - "convergenceSeconds": 60, - "scaleCapabilityDiscovery": "max(90 seconds, two complete peer rotations at actual repair cadence plus 60 seconds)", - "membershipLockSeconds": 10, - "shutdownSeconds": 35, - "individualProcessLifetimeMinutes": 45, - "gateJobMinutes": 35, - "automaticCostJobMinutes": 25, - "manualScaleJobMinutes": 90, - "rpcLifetimeScope": "ResponseTimeout bounds locally observable raw RPC completion, not remote execution. The harness does not require the fourth-slice per-call acknowledgment-scope API. Cooperative Hold cancellation is checked separately through the remote Started/Cancelled/Pending counters, not inferred from caller cancellation or RPC completion.", - "failure": "Fail, preserve exact state/raw stdout/stderr/binlogs/TRX, and kill only the specific owned child process after its shutdown bound. A forced kill is not a successful bounded shutdown.", - "largeDispatch": "The job lifetime is a hard budget, including extreme user-selected size/iteration combinations." - }, - "artifacts": { - "gate": "dissemination-gate-evidence: TRX, identity/probe snapshots, isolated repair checkpoints, bounded shutdown outcomes, raw logs, methodology and binlogs", - "binaries": "dissemination-binaries: pinned old/current/runner publish directories plus SHA manifests for explicit replay using -SkipBuild", - "automaticCost": "dissemination-bounded-cost-evidence: cost-results.json and per-window/raw artifacts for the fixed automatic PR profile, including actual restart and partition counts", - "scale": "dissemination-scale--evidence: cost-results.json, comparisons.json, host-resources.json, per-window cost.json, raw start/finish/warmup/node snapshots and partial results on failure" - } -} From b6ae361819cdd6d80819fd9566ba6f90cdf545f6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 10:39:52 -0700 Subject: [PATCH 33/50] fix(dissemination): fence membership reset state and preserve recovery --- .../implementation/runtime-dissemination.md | 8 +- .../Options/DisseminationOptions.cs | 8 +- .../IMembershipTable.cs | 6 +- .../DisseminationBroadcastQueue.cs | 38 ++- .../Dissemination/DisseminationMembership.cs | 21 +- .../MembershipDisseminationNamespace.cs | 72 ++-- .../Silo/NewRuntime.cs | 7 +- .../DisseminationClusterTests.cs | 56 ++- .../DisseminationMembershipSnapshotTests.cs | 38 ++- ...seminationProtocolTests.MembershipReset.cs | 75 +++++ ...seminationProtocolTests.PumpDiagnostics.cs | 318 ++++++++++++++++++ .../DisseminationProtocolTests.cs | 8 +- 12 files changed, 573 insertions(+), 82 deletions(-) create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index 3ed5f8935c1..861da670476 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -53,7 +53,9 @@ The namespace reports whether a repair is current, produced, unavailable, or una Broadcast senders advertise support for compact acknowledgments. Receive selection explicitly reports whether the whole batch passed validation. When it does and each key's current version equals the maximum version transmitted for that key, one response flag certifies those versions. Namespace entries retain capability evidence with empty per-key lists. The sender advances each ledger to its captured transmitted version, including when that batch is only a prefix of a larger repair. Receivers report explicit versions for older callers, truncated batches, and differing receiver versions, preserving precise repair baselines across version skew. -Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. It retains 32 snapshots, prefers a smaller diff when the peer baseline is present, and falls back to a full snapshot when history or capacity makes the diff unsuitable. Current diffs identify their entries as the complete target inventory, so table cleanup is reflected even when a receiver retained a different same-version baseline. Older partial-diff payloads retain their incremental semantics. Same-version snapshots can advance liveness or remove inactive entries while preserving every Active entry and each retained status. This allows cleanup to converge while heartbeats are unchanged, and an advancing heartbeat preserves those same membership guards. The membership manager merges maximum per-entry liveness and retains its local-death handling; dissemination reports whether that manager actually published changed state. +Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. It retains 32 snapshots and prefers a smaller diff when a lower peer baseline is present. Other baselines, including numerically greater peer versions, receive full snapshots. Current diffs identify their entries as the complete target inventory, so table cleanup is reflected even when a receiver retained a different same-version baseline. Older partial-diff payloads retain their incremental semantics. Same-version peer snapshots can advance liveness or remove inactive entries while preserving every Active entry and every retained versioned field, including status, suspect votes, endpoint metadata, and start time. Authoritative same-version table reads preserve previously accepted versioned fields while merging heartbeat progress, accommodating provider timestamp rounding. Dissemination reports whether the membership manager actually published changed state. + +Development clustering recreates its volatile membership table when the primary restarts. Its manager treats received membership snapshots as notifications to perform a fresh authoritative read, so retained peer state from the prior table cannot replace the recreated table's view. Durable membership providers retain snapshot-based application. Dissemination records current owner state rather than delayed publication arguments, and clears retained history and serialized payloads when the owner replaces its view with a lower version. Deployment-load versions are sample timestamps. Each repair is a full latest value. Application runs on the deployment-load publisher's scheduler and accepts samples only for Active silo generations in the membership oracle. Terminal membership transitions remove placement statistics, and the oracle's monotonic membership state rejects later samples for departed generations. @@ -82,11 +84,11 @@ Namespaces select a membership scope before topology construction: Fanout is derived from the target hop count and bounded by the configured minimum and maximum, or selected by the code-configured callback. A membership or fanout change creates a new topology from the next snapshot; acknowledged ledgers and anti-entropy repair convergence across the transition. -The projection cache tracks its source snapshot as well as its table version. Same-version cleanup of Joining, ShuttingDown, or Stopping participants refreshes routing and peer selection. Heartbeat-only updates and cleanup of non-participants reuse the existing topology, while repaired projections retain rotation progress. The successor guard preserves the newer inventory when concurrent reads arrive out of order. +The projection cache tracks its authoritative source snapshot. On a cache miss it re-reads the owner under the cache lock, so a delayed reader follows the current view, including an owner-authorized version replacement. Eligible membership and status changes refresh routing and peer selection. Heartbeat-only updates and cleanup of non-participants reuse the existing topology, while rebuilt projections retain rotation progress. ## Configuration and defaults -The subsystem and both built-in namespaces are temporarily enabled by default for pre-merge testing, so existing cluster tests exercise broadcast and repair. Set to `false` to select the direct-delivery control path, or set to `false` on an individual integration. Restore the opt-in defaults before merging this testing change. Defaults bound concurrency, memory retention, payload size, and repair work: +Dissemination is opt-in. Enable and on each participating integration. The subsystem and namespace flags default to `false`; ordinary runtime delivery remains active. Explicitly enabled cluster and compatibility tests exercise broadcast and repair. Defaults bound concurrency, memory retention, payload size, and repair work: | Option | Default | Effect | |---|---:|---| diff --git a/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs index df8d9cfb7e3..c6110a78d99 100644 --- a/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs +++ b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs @@ -10,8 +10,8 @@ public sealed class DisseminationOptions /// /// Gets or sets a value indicating whether the dissemination subsystem is enabled. /// - /// during pre-merge testing. - public bool Enabled { get; set; } = true; + /// . Enable dissemination explicitly after configuring the participating namespaces. + public bool Enabled { get; set; } /// /// Gets or sets the maximum number of concurrent local dissemination broadcast attempts. @@ -122,8 +122,8 @@ public sealed class DisseminationNamespaceOptions /// /// Gets or sets a value indicating whether this namespace is enabled. /// - /// during pre-merge testing. - public bool Enabled { get; set; } = true; + /// . Enable each participating namespace explicitly. + public bool Enabled { get; set; } /// /// Gets or sets the dissemination priority for this namespace. diff --git a/src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs b/src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs index 3acd979d042..7b2681ae9ae 100644 --- a/src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs +++ b/src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs @@ -68,8 +68,12 @@ async Task DeleteMembershipTableEntriesAsync(string clusterId, CancellationToken } /// - /// Delete all dead silo entries older than + /// Cleans up defunct silo entries older than . /// + /// + /// Providers can remove expired inactive entries without advancing the table version. + /// Active entries are retained. + /// /// The exclusive upper bound for the last known update time of entries to delete. /// A task representing the cleanup operation. [Obsolete("Use CleanupDefunctSiloEntriesAsync instead.")] diff --git a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs index e035ba75135..e87e43711f7 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs @@ -489,7 +489,15 @@ private void EmitScheduled(ScheduledFlush? scheduled) { if (scheduled is { } info) { - DisseminationInstruments.OnBroadcastScheduled(info.Reason); + try + { + DisseminationInstruments.OnBroadcastScheduled(info.Reason); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + try { DisseminationEvents.EmitBroadcastScheduled(_owner._localSilo, Peer, info.Reason, info.DueTime, info.Attempt, info.Epoch); @@ -786,10 +794,11 @@ private async Task RunScheduledFlushIteration(CancellationToken cancellationToke } catch (Exception exception) { - DisseminationInstruments.OnPumpFailure(DisseminationPumpFailureStatus.Recovered); - LogDebugBroadcastFlushFailed(_owner._logger, exception); + // Restore accepted identities before invoking diagnostics, whose callbacks can throw or reenter. Requeue(work); result = new(RequiresBackoff: true, MadeProgress: false); + LogDebugBroadcastFlushFailed(_owner._logger, exception); + EmitPumpFailure(DisseminationPumpFailureStatus.Recovered); } finally { @@ -852,8 +861,8 @@ private async Task RunScheduledFlushIteration(CancellationToken cancellationToke } } - EmitScheduled(scheduled); flushCompletion.TrySetResult(); + EmitScheduled(scheduled); } } @@ -878,10 +887,11 @@ private bool TryRecoverFromUnexpectedIterationFailure(Exception exception) _activeFlushCompletion = null; } - DisseminationInstruments.OnPumpFailure(DisseminationPumpFailureStatus.Recovered); + activeFlushCompletion?.TrySetResult(); + // Diagnostics follow state recovery, but a failing logger must still fault the pump explicitly. LogWarningBroadcastPumpIterationFailed(_owner._logger, exception, Peer, scheduled?.DueTime); + EmitPumpFailure(DisseminationPumpFailureStatus.Recovered); EmitScheduled(scheduled); - activeFlushCompletion?.TrySetResult(); return true; } catch (Exception recoveryException) @@ -891,12 +901,24 @@ private bool TryRecoverFromUnexpectedIterationFailure(Exception exception) exception, recoveryException); FailPump(failure); - DisseminationInstruments.OnPumpFailure(DisseminationPumpFailureStatus.Permanent); LogErrorBroadcastPumpFailed(_owner._logger, failure, Peer); + EmitPumpFailure(DisseminationPumpFailureStatus.Permanent); return false; } } + private void EmitPumpFailure(DisseminationPumpFailureStatus status) + { + try + { + DisseminationInstruments.OnPumpFailure(status); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + } + private void FailPump(Exception exception) { TaskCompletionSource nextFlushCompletion; @@ -1747,7 +1769,7 @@ private static partial void LogErrorBroadcastPumpFailed( [LoggerMessage( Level = LogLevel.Debug, - Message = "Dissemination broadcast scheduling diagnostic for {Peer} failed.")] + Message = "Dissemination broadcast diagnostic for {Peer} failed.")] private static partial void LogDebugBroadcastDiagnosticFailed( ILogger logger, Exception exception, diff --git a/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs b/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs index de77e76d163..0ab930bd2d0 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs @@ -27,9 +27,14 @@ public DisseminationMembershipSnapshots CurrentSnapshots lock (_membershipLock) { current = Volatile.Read(ref _current); - if (current is not null && !membershipSnapshot.IsSuccessorTo(current.Source)) + if (current is not null) { - return current.Snapshots; + // Re-read the owner after acquiring the lock instead of imposing a second version policy. + membershipSnapshot = membershipManager.CurrentSnapshot; + if (ReferenceEquals(current.Source, membershipSnapshot)) + { + return current.Snapshots; + } } // Same-version heartbeats and removal of non-participants retain the existing topology. @@ -73,14 +78,18 @@ private static bool HasSameTopology(CachedMembership current, MembershipTableSna return false; } - if (current.Source.Entries.Count == source.Entries.Count) + foreach (var member in current.Snapshots.AllMembers.Members) { - return true; + if (!source.Entries.TryGetValue(member, out var entry) + || entry.Status != current.Source.Entries[member].Status) + { + return false; + } } - foreach (var member in current.Snapshots.AllMembers.Members) + foreach (var (member, entry) in source.Entries) { - if (!source.Entries.ContainsKey(member)) + if (IsDisseminationMember(entry.Status) && !current.Snapshots.AllMembers.ContainsMember(member)) { return false; } diff --git a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs index bc5073d2e2a..3082664341c 100644 --- a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs @@ -19,6 +19,7 @@ internal sealed class MembershipDisseminationNamespace( private readonly SortedDictionary _snapshotHistory = new(); private readonly Dictionary> _snapshotPayloads = []; private readonly Dictionary<(long FromVersion, long ToVersion), ReadOnlyMemory> _diffPayloads = []; + private MembershipTableSnapshot? _currentSnapshot; public DisseminationNamespace Name => DisseminationNamespaceNames.Membership; @@ -34,8 +35,8 @@ public async ValueTask PublishAsync( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - // Remember the exact snapshot before waking peer pumps so it is immediately repairable. - RememberSnapshot(snapshot); + // Capture authoritative state: a delayed publication can outlive the snapshot which caused it. + RememberCurrentSnapshot(); return await disseminationService.Publish( this, DisseminationKey.Default, @@ -47,8 +48,7 @@ public IEnumerable Digests { get { - var snapshot = membershipManager.CurrentSnapshot; - RememberSnapshot(snapshot); + var snapshot = RememberCurrentSnapshot(); // Version alone misses same-version liveness advances, so the digest fingerprints heartbeat state too. yield return new DigestEntry( DisseminationKey.Default, @@ -72,8 +72,7 @@ public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest requ // Select history and cached bytes atomically because membership can change without advancing its version. lock (_historyLock) { - var currentSnapshot = membershipManager.CurrentSnapshot; - RememberSnapshotUnsafe(currentSnapshot); + var currentSnapshot = RememberCurrentSnapshotUnsafe(); var targetVersion = request.ToVersion ?? currentSnapshot.Version.Value; if (targetVersion > currentSnapshot.Version.Value || !_snapshotHistory.TryGetValue(targetVersion, out var targetSnapshot)) @@ -81,7 +80,8 @@ public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest requ return DisseminationRepairResult.Unavailable(currentSnapshot.Version.Value); } - // Prefer a retained peer baseline when available; otherwise the full snapshot remains the fallback. + // A numerically greater peer version can belong to an older table incarnation. + // Only a retained lower baseline is useful for a diff; all other peers receive a full snapshot. MembershipTableSnapshot? baseSnapshot = null; if (request.FromVersion is { } fromVersion && fromVersion > 0 @@ -91,11 +91,6 @@ public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest requ } var resolvedVersion = targetSnapshot.Version.Value; - if (request.FromVersion is { } peerVersion && peerVersion > resolvedVersion) - { - return DisseminationRepairResult.Current(resolvedVersion); - } - if (request.MaxItemCount <= 0) { return DisseminationRepairResult.InsufficientCapacity(resolvedVersion); @@ -244,23 +239,31 @@ private async ValueTask ApplySnapshot( CancellationToken cancellationToken) { await membershipManager.ProcessGossipSnapshot(snapshot, cancellationToken); - var current = membershipManager.CurrentSnapshot; - RememberSnapshot(current); + var current = RememberCurrentSnapshot(); return MembershipSnapshotsEqual(previous, current) ? DisseminationApplyResult.Duplicate : DisseminationApplyResult.Applied; } - private void RememberSnapshot(MembershipTableSnapshot snapshot) + private MembershipTableSnapshot RememberCurrentSnapshot() { lock (_historyLock) { - RememberSnapshotUnsafe(snapshot); + return RememberCurrentSnapshotUnsafe(); } } - private void RememberSnapshotUnsafe(MembershipTableSnapshot snapshot) + private MembershipTableSnapshot RememberCurrentSnapshotUnsafe() { + var snapshot = membershipManager.CurrentSnapshot; + if (_currentSnapshot is { } previousCurrent && snapshot.Version < previousCurrent.Version) + { + _snapshotHistory.Clear(); + _snapshotPayloads.Clear(); + _diffPayloads.Clear(); + } + + _currentSnapshot = snapshot; if (_snapshotHistory.TryGetValue(snapshot.Version.Value, out var previous) && !MembershipSnapshotsEqual(previous, snapshot)) { @@ -279,6 +282,8 @@ private void RememberSnapshotUnsafe(MembershipTableSnapshot snapshot) InvalidatePayloads(removedVersion); } } + + return snapshot; } private void InvalidatePayloads(long version) @@ -395,16 +400,7 @@ private static MembershipEntry PreserveIAmAliveTime(MembershipTableSnapshot prev }; private static bool MembershipEntriesEqual(MembershipEntry left, MembershipEntry right) => - left.SiloAddress.Equals(right.SiloAddress) - && left.Status == right.Status - && EqualSuspectTimes(left.SuspectTimes, right.SuspectTimes) - && left.ProxyPort == right.ProxyPort - && string.Equals(left.HostName, right.HostName, StringComparison.Ordinal) - && string.Equals(left.SiloName, right.SiloName, StringComparison.Ordinal) - && string.Equals(left.RoleName, right.RoleName, StringComparison.Ordinal) - && left.UpdateZone == right.UpdateZone - && left.FaultZone == right.FaultZone - && left.StartTime == right.StartTime + MembershipTableSnapshot.AreVersionedFieldsEqual(left, right) && left.IAmAliveTime == right.IAmAliveTime; private static bool MembershipSnapshotsEqual( @@ -433,28 +429,6 @@ private static bool MembershipSnapshotsEqual( return true; } - private static bool EqualSuspectTimes(List>? left, List>? right) - { - if (left is null || right is null) - { - return left is null && right is null; - } - - if (left.Count != right.Count) - { - return false; - } - - for (var i = 0; i < left.Count; i++) - { - if (!Equals(left[i].Item1, right[i].Item1) || left[i].Item2 != right[i].Item2) - { - return false; - } - } - - return true; - } } [GenerateSerializer, Immutable] diff --git a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs index e6e8de40487..d35da8c3206 100644 --- a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs +++ b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs @@ -13,12 +13,11 @@ internal static class NewRuntime { public static void Configure(IServiceCollection services, NodeConfiguration configuration) { - // Preserve the opt-out control while the candidate defaults are enabled for pre-merge testing. + services.Configure(options => options.Enabled = configuration.Enabled); + services.Configure(options => options.Dissemination.Enabled = configuration.Enabled); + services.Configure(options => options.Dissemination.Enabled = configuration.Enabled); if (!configuration.Enabled) { - services.Configure(options => options.Enabled = false); - services.Configure(options => options.Dissemination.Enabled = false); - services.Configure(options => options.Dissemination.Enabled = false); return; } diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs index 65509cbc365..bb039e91245 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs @@ -5,6 +5,8 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; +using Orleans; +using Orleans.Configuration; using Orleans.Runtime; using Orleans.Runtime.Dissemination; using Orleans.Runtime.MembershipService; @@ -33,13 +35,14 @@ public sealed class DisseminationDiagnosticCollection public sealed class DisseminationClusterTests { [Fact] - public async Task MembershipUpdatesAreDisseminatedAcrossRealClusterWithDefaults() + public async Task MembershipUpdatesAreDisseminatedAcrossRealClusterWithExplicitOptIn() { using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); cancellation.CancelAfter(TimeSpan.FromSeconds(120)); var cancellationToken = cancellation.Token; var builder = new InProcessTestClusterBuilder(3); + builder.ConfigureSilo((_, silo) => new EnabledDisseminationConfigurator().Configure(silo)); await using var cluster = builder.Build(); await cluster.DeployAsync(cancellationToken); await cluster.WaitForLivenessToStabilizeAsync().WaitAsync(cancellationToken); @@ -68,6 +71,57 @@ public async Task MembershipUpdatesAreDisseminatedAcrossRealClusterWithDefaults( + string.Join(", ", observer.AppliedSilos.Select(static silo => silo.ToString()))); } + [Fact] + public async Task DevelopmentPrimaryRestartRejectsRetainedPreResetMembership() + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cancellation.CancelAfter(TimeSpan.FromMinutes(3)); + var cancellationToken = cancellation.Token; + var builder = new TestClusterBuilder(3); + builder.Options.InitializeClientOnDeploy = false; + builder.AddSiloBuilderConfigurator(); + await using var cluster = builder.Build(); + await cluster.DeployAsync(cancellationToken); + var primary = Assert.IsType(cluster.Primary); + Assert.IsType(primary.SiloHost.Services.GetRequiredService()); + var retained = primary.SiloHost.Services.GetRequiredService().CurrentSnapshot; + var originalSilos = cluster.GetActiveSilos().ToArray(); + + var restarted = Assert.IsType( + await cluster.RestartSiloAsync(primary).WaitAsync(cancellationToken)); + Assert.NotEqual(primary.SiloAddress, restarted.SiloAddress); + var manager = restarted.SiloHost.Services.GetRequiredService(); + Assert.True(manager.CurrentSnapshot.Version < retained.Version); + await manager.ProcessGossipSnapshot(retained, cancellationToken); + Assert.Equal(SiloStatus.Active, manager.LocalSiloStatus); + Assert.Contains(restarted.SiloAddress, manager.CurrentSnapshot.Entries.Keys); + Assert.DoesNotContain(primary.SiloAddress, manager.CurrentSnapshot.Entries.Keys); + + foreach (var silo in originalSilos.Where(silo => !ReferenceEquals(silo, primary))) + { + await cluster.RestartSiloAsync(silo).WaitAsync(cancellationToken); + } + + foreach (var silo in cluster.GetActiveSilos().Cast()) + { + var current = silo.SiloHost.Services.GetRequiredService(); + await current.Refresh(null, cancellationToken, requireFresh: true); + Assert.Equal(SiloStatus.Active, current.LocalSiloStatus); + Assert.Contains(restarted.SiloAddress, current.CurrentSnapshot.Entries.Keys); + Assert.DoesNotContain(primary.SiloAddress, current.CurrentSnapshot.Entries.Keys); + } + } + + public sealed class EnabledDisseminationConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder builder) => builder.ConfigureServices(services => + { + services.Configure(options => options.Enabled = true); + services.Configure(options => options.Dissemination.Enabled = true); + services.Configure(options => options.Dissemination.Enabled = true); + }); + } + private sealed class ValueApplyObserver( int targetDistinctSilos, IReadOnlySet expectedSilos, diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs index 32e993bd8c5..ab1fa689c38 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs @@ -381,7 +381,7 @@ public async Task ConcurrentRefreshDoesNotReturnOlderProjection() var latest = await secondRefresh; Assert.Equal(new MembershipVersion(42), latest.MembershipVersion); Assert.Equal(new[] { local, firstPeer, secondPeer }, latest.ActiveMembers.Members); - Assert.Equal(3, manager.SnapshotReadCount); + Assert.Equal(5, manager.SnapshotReadCount); } [Fact] @@ -413,7 +413,41 @@ public async Task StaleConcurrentRefreshDoesNotRegressNewerProjection() Assert.Equal(new MembershipVersion(42), newer.MembershipVersion); Assert.Equal(new MembershipVersion(42), stale.MembershipVersion); Assert.Equal(new[] { local, firstPeer, secondPeer }, membership.CurrentSnapshots.ActiveMembers.Members); - Assert.Equal(4, manager.SnapshotReadCount); + Assert.Equal(6, manager.SnapshotReadCount); + } + + [Theory] + [InlineData(1)] + [InlineData(40)] + public async Task AuthoritativeTopologyReplacementWinsOverStaleConcurrentRead(long replacementVersion) + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(3); + var manager = new MutableMembershipManager(CreateSourceSnapshot(40, members[0], members[1]), cancellationToken) + { + BlockSecondSnapshotRead = true, + }; + var membership = new DisseminationMembership(manager, new ScopeLocalSiloDetails(members[0]), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + Assert.Contains(members[1], membership.CurrentSnapshot.Members); + var stale = Task.Run(() => membership.CurrentSnapshots, cancellationToken); + DisseminationMembershipSnapshots replacement; + try + { + await manager.SecondReadCaptured.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + manager.SetSnapshot(CreateSourceSnapshot(replacementVersion, members[0], members[2])); + replacement = membership.CurrentSnapshots; + Assert.Equal(new MembershipVersion(replacementVersion), replacement.MembershipVersion); + Assert.Equal(new[] { members[0], members[2] }, replacement.ActiveMembers.Members); + } + finally + { + manager.ReleaseSecondRead(); + } + + Assert.Same(replacement, await stale.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + Assert.Same(replacement, membership.CurrentSnapshots); + Assert.DoesNotContain(members[1], replacement.AllMembers.Members); } [Theory] diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs new file mode 100644 index 00000000000..09d3656af23 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs @@ -0,0 +1,75 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public void MembershipRepairSendsFullStateWhenPeerVersionIsAhead() + { + var local = CreateSilo(41001); + var snapshot = CreateMembershipSnapshot(2, CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var ns = CreateMembershipNamespace(new FakeMembershipManager(snapshot), serializer); + + var repair = ns.CreateRepair(new( + DisseminationKey.Default, 100, null, 1, 1024 * 1024, 1024 * 1024)); + + Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); + Assert.Equal(2, repair.Version); + var value = Assert.Single(repair.Values); + Assert.Equal(0, value.FromVersion); + Assert.Equal(2, value.ToVersion); + var update = Assert.IsType( + serializer.Deserialize(value.Payload)); + Assert.Null(update.Diff); + var repaired = Assert.IsType(update.Snapshot); + Assert.Equal(snapshot.Version, repaired.Version); + Assert.Equal(local, Assert.Single(repaired.Entries).Key); + } + + [Fact] + public async Task MembershipHistoryResetDiscardsPriorIncarnationAndDelayedPublication() + { + var members = CreateSilos(20); + var old = CreateMembershipSnapshot(1, members.Select( + member => CreateMembershipEntry(member, SiloStatus.Active, DateTime.UnixEpoch)).ToArray()); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(old); + var ns = CreateMembershipNamespace(manager, serializer); + Assert.Equal(1, Assert.Single(ns.Digests).Version); + manager.CurrentSnapshot = CreateMembershipSnapshot(100, old.Entries.Values.ToArray()); + Assert.Equal(100, Assert.Single(ns.Digests).Version); + var reset = CreateMembershipSnapshot(2, old.Entries.Values.Select(entry => + { + var result = entry.Copy(); + result.HostName = "new-table"; + return result; + }).ToArray()); + manager.CurrentSnapshot = reset; + Assert.Equal(2, Assert.Single(ns.Digests).Version); + + Assert.False(await ns.PublishAsync(new FakeDisseminationService(), old, TestContext.Current.CancellationToken)); + var obsoleteBaseline = ns.CreateRepair(new( + DisseminationKey.Default, null, 1, 1, 1024 * 1024, 1024 * 1024)); + Assert.Equal(DisseminationRepairStatus.Unavailable, obsoleteBaseline.Status); + var repair = ns.CreateRepair(new( + DisseminationKey.Default, 1, null, 1, 1024 * 1024, 1024 * 1024)); + Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); + var value = Assert.Single(repair.Values); + Assert.Equal(0, value.FromVersion); + Assert.Equal(2, value.ToVersion); + var payload = Assert.IsType( + serializer.Deserialize(value.Payload)); + Assert.Null(payload.Diff); + var repaired = Assert.IsType(payload.Snapshot); + Assert.Equal(members.Length, repaired.Entries.Count); + Assert.All(repaired.Entries.Values, entry => Assert.Equal("new-table", entry.HostName)); + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs new file mode 100644 index 00000000000..95dd362c8ab --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs @@ -0,0 +1,318 @@ +#nullable enable + +using System; +using System.Collections.Concurrent; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.Configuration; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +// Pump failure metrics have no peer tag, so throwing listeners must not overlap other tests. +[Collection(DisseminationDiagnosticCollection.Name)] +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData(0, false)] + [InlineData(1, false)] + [InlineData(1, true)] + public Task PumpFailureMetricExceptionsPreserveAcceptedWorkAndCapacity(int failedTimerChanges, bool failScheduleMetric) => + VerifyPumpFailureMetricIsolation(failedTimerChanges, failScheduleMetric); + + [Fact] + public Task PumpFailureMetricExceptionsPreservePermanentFailureAndWaiters() => + VerifyPumpFailureMetricIsolation(failedTimerChanges: 2, failScheduleMetric: false); + + [Theory] + [InlineData("warning")] + [InlineData("pump-failure")] + [InlineData("retry-schedule")] + public Task RecoveryDiagnosticLoggerExceptionsFailPumpAndPendingWaiters(string failureStage) => + VerifyPumpFailureMetricIsolation( + failedTimerChanges: 1, failScheduleMetric: failureStage == "retry-schedule", recoveryLogFailureStage: failureStage); + + private static async Task VerifyPumpFailureMetricIsolation(int failedTimerChanges, bool failScheduleMetric, string? recoveryLogFailureStage = null) + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40951); + var peer = CreateSilo(40952); + var clock = new RecordingFakeTimeProvider(); + var logger = new PumpDiagnosticsLogger(); + var ns = new FakeNamespace(local); + ns.Options.MaxPendingItemCount = 1; + ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + ns.SetValue("original", 7); + ns.SetValue("next", 9); + var transport = new FakeTransport(local, peer); + var started = Enumerable.Range(0, 3) + .Select(static _ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)).ToArray(); + var release = Enumerable.Range(0, 3) + .Select(static _ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)).ToArray(); + var sent = new ConcurrentQueue<(string Key, long Version, long Payload)>(); + var sendCount = 0; + transport.SendBroadcastResponseHandler = async (_, batch, token) => + { + var value = Assert.Single(GetBroadcastValues(batch)).Value; + sent.Enqueue((value.Key.ToString(), value.ToVersion, BitConverter.ToInt64(value.Payload.Span))); + var attempt = Interlocked.Increment(ref sendCount); + if (attempt <= started.Length) + { + started[attempt - 1].TrySetResult(); + await release[attempt - 1].Task.WaitAsync(token); + } + + return FakeTransport.CreateAcknowledgment(batch); + }; + var responseFailure = new InvalidOperationException("The first response observer fails before acknowledgment processing."); + var responseCount = 0; + var options = new DisseminationOptions { Enabled = true, MaxConcurrentSends = 1 }; + options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(4); + var queue = new DisseminationBroadcastQueue( + clock, local, transport.GrainFactory, new TestOptionsMonitor(options), [ns], logger, + responseObserver: (_, _) => + { + if (Interlocked.Increment(ref responseCount) == 1) + { + throw responseFailure; + } + }); + using var schedules = new BroadcastScheduleObserver(); + using var listener = new MeterListener(); + var pumpMetricFailure = new InvalidOperationException("Pump failure metric callback fails."); + var scheduleMetricFailure = new InvalidOperationException("Retry schedule metric callback fails."); + var pumpStatuses = new ConcurrentQueue(); + var scheduleMetricFailures = 0; + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == DisseminationInstruments.MeterName + && (instrument.Name == DisseminationInstruments.PumpFailuresName + || failScheduleMetric && instrument.Name == DisseminationInstruments.BroadcastScheduledName)) + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, _, tags, _) => + { + if (instrument.Name == DisseminationInstruments.PumpFailuresName) + { + pumpStatuses.Enqueue((string)Assert.Single(tags.ToArray(), static tag => tag.Key == "status").Value!); + throw pumpMetricFailure; + } + + if (tags.ToArray().Any(static tag => tag.Key == "reason" && Equals(tag.Value, "retry"))) + { + Interlocked.Increment(ref scheduleMetricFailures); + throw scheduleMetricFailure; + } + }); + var recoveryLogFailure = new InvalidOperationException("The recovery diagnostic logger fails."); + var pendingRecoveryFlush = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var recoveryLogFailures = 0; + if (recoveryLogFailureStage is not null) + { + logger.OnLog = (level, exception) => + { + if (level == LogLevel.Warning) + { + var pendingFlush = queue.FlushPendingBroadcast(cancellationToken); + Assert.False(pendingFlush.IsCompleted); + pendingRecoveryFlush.TrySetResult(pendingFlush); + } + + var shouldThrow = recoveryLogFailureStage switch + { + "warning" => level == LogLevel.Warning, + "pump-failure" => ReferenceEquals(exception, pumpMetricFailure), + "retry-schedule" => ReferenceEquals(exception, scheduleMetricFailure), + _ => false, + }; + if (pendingRecoveryFlush.Task.IsCompletedSuccessfully && shouldThrow + && Interlocked.CompareExchange(ref recoveryLogFailures, 1, 0) == 0) + { + throw recoveryLogFailure; + } + }; + } + + Task? stop = null; + try + { + var initialSchedule = schedules.WaitAsync( + value => value.LocalSilo.Equals(local) && value.Peer.Equals(peer) + && value.Reason == DisseminationBroadcastScheduleReason.Immediate, + TimeSpan.FromSeconds(5), cancellationToken); + Assert.True(queue.Notify(peer, ns, "original")); + await WaitForPhase(initialSchedule, "initial timer armed"); + var initial = await initialSchedule; + Assert.Equal(TimeSpan.Zero, initial.DueTime); + Assert.Equal(0, initial.Attempt); + clock.Advance(initial.DueTime); + await WaitForPhase(started[0].Task, "original send in flight"); + Assert.False(queue.Notify(peer, ns, "next")); + + var firstFlush = queue.FlushPendingBroadcast(cancellationToken); + Assert.False(firstFlush.IsCompleted); + var retrySchedule = failedTimerChanges < 2 && recoveryLogFailureStage is null + ? schedules.WaitAsync( + value => value.LocalSilo.Equals(local) && value.Peer.Equals(peer) + && value.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), cancellationToken) + : null; + listener.Start(); + clock.ThrowOnNextTimerChanges(failedTimerChanges); + release[0].TrySetResult(); + + if (failedTimerChanges == 2 || recoveryLogFailureStage is not null) + { + var failedFlush = firstFlush; + if (recoveryLogFailureStage is not null) + { + await WaitForPhase(firstFlush, "active waiter completed before recovery diagnostics"); + await WaitForPhase(pendingRecoveryFlush.Task, "pending waiter attached before recovery logger throws"); + failedFlush = await pendingRecoveryFlush.Task; + } + + var failure = await Assert.ThrowsAsync( + () => WaitForPhase(failedFlush, "permanently failed flush waiter")); + var notificationFailure = Assert.Throws(() => queue.Notify(peer, ns, "next")); + Assert.Same(failure, notificationFailure.InnerException); + var laterFlushFailure = await Assert.ThrowsAsync( + () => WaitForPhase(queue.FlushPendingBroadcast(cancellationToken), "later flush observes terminal failure")); + Assert.Same(failure, laterFlushFailure.InnerException); + stop = queue.StopAsync(cancellationToken); + var stopFailure = await Assert.ThrowsAsync( + () => WaitForPhase(stop, "permanently failed drain waiter")); + Assert.Same(failure, stopFailure); + Assert.Equal(2, failure.InnerExceptions.Count); + Assert.Equal("The test timer fails one scheduled change.", failure.InnerExceptions[0].Message); + if (recoveryLogFailureStage is null) + { + Assert.Equal("The test timer fails one scheduled change.", failure.InnerExceptions[1].Message); + } + else + { + Assert.Same(recoveryLogFailure, failure.InnerExceptions[1]); + Assert.Equal(1, Volatile.Read(ref recoveryLogFailures)); + } + + Assert.Same(failure, Assert.Single(logger.Entries, static entry => entry.Level == LogLevel.Error).Exception); + var recoveredMetrics = recoveryLogFailureStage is "pump-failure" or "retry-schedule" ? 2 : 1; + Assert.Equal(Enumerable.Repeat("recovered", recoveredMetrics).Append("permanent"), pumpStatuses); + Assert.Equal(1, Volatile.Read(ref sendCount)); + } + else + { + await WaitForPhase(firstFlush, "failed iteration flush waiter released"); + await WaitForPhase(retrySchedule!, "recovered retry timer armed"); + var retry = await retrySchedule!; + Assert.Equal(failedTimerChanges + 1, retry.Attempt); + Assert.Equal(TimeSpan.FromSeconds(failedTimerChanges + 1), retry.DueTime); + Assert.Equal(initial.Epoch, retry.Epoch); + Assert.Equal(1, Volatile.Read(ref sendCount)); + + clock.Advance(retry.DueTime); + await WaitForPhase(started[1].Task, "original accepted value retried"); + var retryFlush = queue.FlushPendingBroadcast(cancellationToken); + Assert.False(retryFlush.IsCompleted); + release[1].TrySetResult(); + await WaitForPhase(retryFlush, "original accepted value acknowledged"); + Assert.Equal(new[] { ("original", 7L, 7L), ("original", 7L, 7L) }, sent); + Assert.True(queue.Notify(peer, ns, "original", force: false)); + await WaitForPhase(queue.FlushPendingBroadcast(cancellationToken), "acknowledged duplicate suppressed"); + Assert.Equal(2, ns.RepairRequestCount); + Assert.Equal(2, Volatile.Read(ref sendCount)); + + var nextSchedule = schedules.WaitAsync( + value => value.LocalSilo.Equals(local) && value.Peer.Equals(peer) + && value.Reason == DisseminationBroadcastScheduleReason.Immediate && value.Epoch > retry.Epoch, + TimeSpan.FromSeconds(5), cancellationToken); + Assert.True(queue.Notify(peer, ns, "next")); + await WaitForPhase(nextSchedule, "freed capacity schedules the next key"); + var next = await nextSchedule; + Assert.Equal(0, next.Attempt); + Assert.Equal(retry.Epoch + 1, next.Epoch); + clock.Advance(next.DueTime); + await WaitForPhase(started[2].Task, "next key admitted to transport"); + var nextFlush = queue.FlushPendingBroadcast(cancellationToken); + Assert.False(nextFlush.IsCompleted); + release[2].TrySetResult(); + await WaitForPhase(nextFlush, "next key acknowledged"); + Assert.True(queue.Notify(peer, ns, "next", force: false)); + await WaitForPhase(queue.FlushPendingBroadcast(cancellationToken), "next acknowledged duplicate suppressed"); + Assert.Equal(new[] { ("original", 7L, 7L), ("original", 7L, 7L), ("next", 9L, 9L) }, sent); + Assert.Equal(3, ns.RepairRequestCount); + Assert.Equal(3, Volatile.Read(ref sendCount)); + Assert.Equal(Enumerable.Repeat("recovered", failedTimerChanges + 1), pumpStatuses); + Assert.Equal(failedTimerChanges, logger.Entries.Count(static entry => entry.Level == LogLevel.Warning)); + Assert.DoesNotContain(logger.Entries, static entry => entry.Level == LogLevel.Error); + } + + Assert.Same(responseFailure, Assert.Single(logger.Entries, entry => ReferenceEquals(entry.Exception, responseFailure)).Exception); + Assert.Equal(pumpStatuses.Count, logger.Entries.Count(entry => + entry.Level == LogLevel.Debug && ReferenceEquals(entry.Exception, pumpMetricFailure))); + Assert.Equal(failScheduleMetric ? 1 : 0, Volatile.Read(ref scheduleMetricFailures)); + Assert.Equal(scheduleMetricFailures, logger.Entries.Count(entry => + entry.Level == LogLevel.Debug && ReferenceEquals(entry.Exception, scheduleMetricFailure))); + } + finally + { + listener.Dispose(); + clock.ThrowOnNextTimerChanges(0); + foreach (var completion in release) + { + completion.TrySetResult(); + } + + if (stop is null) + { + ns.Options.Enabled = false; + using var cleanup = new CancellationTokenSource(); + cleanup.Cancel(); + try + { + await WaitForPhase(queue.StopAsync(cleanup.Token), "queue cleanup"); + } + catch (OperationCanceledException exception) when (exception.CancellationToken == cleanup.Token) + { + // An assertion failure can leave accepted work behind a fake-time retry; cancel its drain. + } + } + } + + async Task WaitForPhase(Task task, string phase) + { + try + { + await task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + catch (TimeoutException exception) + { + throw new TimeoutException( + $"Timed out during {phase} for {local} -> {peer}; sends={Volatile.Read(ref sendCount)}, timer failures={failedTimerChanges}.", + exception); + } + } + } + + private sealed class PumpDiagnosticsLogger : ILogger + { + public ConcurrentQueue<(LogLevel Level, Exception? Exception)> Entries { get; } = new(); + + public Action? OnLog { get; set; } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + Entries.Enqueue((logLevel, exception)); + OnLog?.Invoke(logLevel, exception); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index 365c2fa6212..e1a85ed96b8 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -3207,10 +3207,10 @@ public void DisseminationOptionsUseBatchDefaults() { var options = new DisseminationOptions(); - Assert.True(options.Enabled); - Assert.True(new DisseminationNamespaceOptions().Enabled); - Assert.True(new DeploymentLoadPublisherOptions().Dissemination.Enabled); - Assert.True(new ClusterMembershipOptions().Dissemination.Enabled); + Assert.False(options.Enabled); + Assert.False(new DisseminationNamespaceOptions().Enabled); + Assert.False(new DeploymentLoadPublisherOptions().Dissemination.Enabled); + Assert.False(new ClusterMembershipOptions().Dissemination.Enabled); Assert.Equal(1024 * 1024, options.MaxBatchBytes); Assert.Equal(8 * 1024, options.MaxBatchItems); Assert.Equal(options.MaxBatchBytes, new DisseminationNamespaceOptions().MaxPayloadBytes); From 144120b6bb962f660c25da670055e7c6ff2bc442 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 11:22:10 -0700 Subject: [PATCH 34/50] test(dissemination): align fixtures with explicit opt-in and owner state --- .../Membership/MembershipGossiperTests.cs | 10 +++++++--- .../Runtime/DeploymentLoadPublisherTests.cs | 12 +++++++----- .../Dissemination/DisseminationClusterTests.cs | 5 ++++- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/test/Orleans.Core.Tests/Membership/MembershipGossiperTests.cs b/test/Orleans.Core.Tests/Membership/MembershipGossiperTests.cs index 801276a6344..c8733d68380 100644 --- a/test/Orleans.Core.Tests/Membership/MembershipGossiperTests.cs +++ b/test/Orleans.Core.Tests/Membership/MembershipGossiperTests.cs @@ -55,7 +55,8 @@ public async Task GossipToRemoteSilos_EligibleLocalStatus_StartsDirectAndDissemi try { - await Task.WhenAll(directStarted.Task, disseminationStarted.Task).WaitAsync(TestContext.Current.CancellationToken); + await Task.WhenAll(directStarted.Task, disseminationStarted.Task) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); Assert.False(gossipTask.IsCompleted); Assert.False(directCompleted.Task.IsCompleted); Assert.Equal(1, rig.DisseminationServiceResolutionCount); @@ -127,7 +128,8 @@ public async Task GossipToRemoteSilos_CallerCancellation_CancelsDirectGossipAndD try { - await Task.WhenAll(directStarted.Task, disseminationStarted.Task).WaitAsync(TestContext.Current.CancellationToken); + await Task.WhenAll(directStarted.Task, disseminationStarted.Task) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); await rig.RemoteMembershipService.Received(1).MembershipChangeNotification(snapshot, cancellation.Token); } finally @@ -186,7 +188,8 @@ public async Task MembershipGossiperStartsDirectGossipBeforeDissemination() try { - await Task.WhenAll(directStarted.Task, disseminationStarted.Task).WaitAsync(TestContext.Current.CancellationToken); + await Task.WhenAll(directStarted.Task, disseminationStarted.Task) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); Assert.Equal(["MembershipSystemTargetResolved", "DisseminationServiceResolved"], resolutionOrder); } finally @@ -418,6 +421,7 @@ private static DisseminationTestRig CreateDisseminationTestRig( var grainFactory = Substitute.For(); grainFactory.GetSystemTarget(Constants.MembershipServiceType, Arg.Any()).Returns(remote); var membershipManager = Substitute.For(); + membershipManager.CurrentSnapshot.Returns(CreateSnapshot(localSilo, remoteSilo, SiloStatus.Active)); var options = Substitute.For>(); options.CurrentValue.Returns(new ClusterMembershipOptions { diff --git a/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs b/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs index 589dc7217cf..54d106bc486 100644 --- a/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs +++ b/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs @@ -25,10 +25,12 @@ namespace NonSilo.Tests.Runtime; [TestArea("Runtime")] public class DeploymentLoadPublisherTests { - [Fact] - public async Task PublishStatistics_ExplicitlyDisabled_PreservesDirectPublication() + [Theory] + [InlineData(null)] + [InlineData(false)] + public async Task PublishStatistics_DisabledByDefaultOrExplicitly_PreservesDirectPublication(bool? configured) { - using var rig = CreateTestRig(TimeSpan.FromSeconds(5)); + using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: configured); await rig.Publisher.PublishStatistics(TestContext.Current.CancellationToken); @@ -41,9 +43,9 @@ await rig.DirectTarget.Received(1).UpdateRuntimeStatistics( [Theory] [InlineData(false)] [InlineData(true)] - public async Task PublishStatistics_DefaultEnabled_ConfirmedPeersUseDissemination(bool confirmed) + public async Task PublishStatistics_ExplicitlyEnabled_ConfirmedPeersUseDissemination(bool confirmed) { - using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: null); + using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: true); var remoteSilo = SiloAddress.FromParsableString("127.0.0.1:200@100"); rig.Dissemination.GetUnconfirmedPeers(Arg.Any()) .Returns(confirmed ? [] : new[] { remoteSilo }); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs index bb039e91245..4fb1d90785a 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs @@ -84,7 +84,10 @@ public async Task DevelopmentPrimaryRestartRejectsRetainedPreResetMembership() await cluster.DeployAsync(cancellationToken); var primary = Assert.IsType(cluster.Primary); Assert.IsType(primary.SiloHost.Services.GetRequiredService()); - var retained = primary.SiloHost.Services.GetRequiredService().CurrentSnapshot; + var primaryManager = primary.SiloHost.Services.GetRequiredService(); + await primaryManager.Refresh(null, cancellationToken, requireFresh: true); + var retained = primaryManager.CurrentSnapshot; + Assert.Equal(3, retained.ActiveNodeCount); var originalSilos = cluster.GetActiveSilos().ToArray(); var restarted = Assert.IsType( From db36740f56647cace09fb46917df7801bbde490c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 11:52:56 -0700 Subject: [PATCH 35/50] fix(dissemination): validate older full values and isolate metrics --- .../implementation/runtime-dissemination.md | 2 + .../DisseminationBroadcastQueue.cs | 36 +++- .../Dissemination/DisseminationProtocol.cs | 74 ++++++-- .../Dissemination/IDisseminationNamespace.cs | 3 + .../MembershipDisseminationNamespace.cs | 12 +- ...seminationProtocolTests.MembershipReset.cs | 91 ++++++++++ ...eminationProtocolTests.MetricBoundaries.cs | 159 ++++++++++++++++++ ...seminationProtocolTests.PumpDiagnostics.cs | 11 +- .../DisseminationProtocolTests.cs | 7 +- 9 files changed, 371 insertions(+), 24 deletions(-) create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index 861da670476..795bb9e79ca 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -57,6 +57,8 @@ Membership uses the membership-table version plus a liveness fingerprint in its Development clustering recreates its volatile membership table when the primary restarts. Its manager treats received membership snapshots as notifications to perform a fresh authoritative read, so retained peer state from the prior table cannot replace the recreated table's view. Durable membership providers retain snapshot-based application. Dissemination records current owner state rather than delayed publication arguments, and clears retained history and serialized payloads when the owner replaces its view with a lower version. +The membership namespace requests authoritative validation for older full values. Broadcast and anti-entropy let those values reach the owner, and repair selection consults the namespace for numerically greater peer baselines. The owner decides the resulting state; delta chains and monotonic namespaces retain their numeric version guards. + Deployment-load versions are sample timestamps. Each repair is a full latest value. Application runs on the deployment-load publisher's scheduler and accepts samples only for Active silo generations in the membership oracle. Terminal membership transitions remove placement statistics, and the oracle's monotonic membership state rejects later samples for departed generations. After the owner accepts a load sample, its serialized payload is reused for forwarding and repair. Inventory maintenance enumerates namespace keys directly; membership exposes its single key, and deployment load uses the active-silo oracle. Each broadcast queue retains one reusable scratch inventory; individual pruning calls have exclusive ownership and clear all keys before returning it, including on failure. Version and fingerprint construction is reserved for operations which use that information. Receive batches which fit all item, byte, and namespace limits are processed directly from their existing collections; truncated batches retain bounded cursor selection. diff --git a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs index e87e43711f7..3f260dc19b6 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs @@ -467,7 +467,15 @@ private void EmitNotification(IDisseminationNamespace disseminationNamespace, in { for (var index = 0; index < rejections; index++) { - DisseminationInstruments.OnQueueAdmissionRejected(disseminationNamespace.Name); + try + { + DisseminationInstruments.OnQueueAdmissionRejected(disseminationNamespace.Name); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + try { DisseminationEvents.EmitQueueAdmissionRejected( @@ -1182,7 +1190,15 @@ private async ValueTask SendValues( try { var response = await sendTask.WaitAsync(sendCancellation.Token); - DisseminationInstruments.OnBroadcastSent(batch.Values, "tree"); + try + { + DisseminationInstruments.OnBroadcastSent(batch.Values, "tree"); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + return response; } catch (OperationCanceledException) when (sendCancellation.IsCancellationRequested) @@ -1197,18 +1213,30 @@ private async ValueTask SendValues( } catch (OperationCanceledException) when (lifetimeCancellation.IsCancellationRequested) { - DisseminationInstruments.OnBroadcastSendFailure(DisseminationFailureReason.Timeout); + EmitSendFailure(DisseminationFailureReason.Timeout); LogDebugBroadcastTransportLifetimeExpired(_owner._logger, Peer, transportLifetime); return null; } catch (Exception exception) { - DisseminationInstruments.OnBroadcastSendFailure(DisseminationFailureReason.Error); + EmitSendFailure(DisseminationFailureReason.Error); LogDebugDisseminationSendFailed(_owner._logger, exception, Peer); return null; } } + private void EmitSendFailure(DisseminationFailureReason reason) + { + try + { + DisseminationInstruments.OnBroadcastSendFailure(reason); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + } + private void ObserveLateSend(Task sendTask) { // Fault observation follows transport completion after the operation's cancellation. diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index 24eea719fe7..f39705425f1 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -82,14 +82,14 @@ public async ValueTask Publish( using var admission = _admission.TryEnter(); if (!admission.Entered) { - DisseminationInstruments.OnPublication(disseminationNamespace.Name, accepted: false, reason: "stopping"); + EmitPublication(disseminationNamespace.Name, accepted: false, reason: "stopping"); return false; } var options = _options.CurrentValue; if (!options.Enabled || !disseminationNamespace.Options.Enabled) { - DisseminationInstruments.OnPublication( + EmitPublication( disseminationNamespace.Name, accepted: false, reason: "disabled"); @@ -105,7 +105,7 @@ public async ValueTask Publish( out var publishedVersion, out var reason)) { - DisseminationInstruments.OnPublication( + EmitPublication( disseminationNamespace.Name, accepted: false, reason: reason); @@ -118,7 +118,7 @@ public async ValueTask Publish( cancellationToken); if (membership is null) { - DisseminationInstruments.OnPublication( + EmitPublication( disseminationNamespace.Name, accepted: false, "membership-unavailable"); @@ -135,7 +135,7 @@ public async ValueTask Publish( immediate: disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree && !membership.IsAggregationRoot); } - DisseminationInstruments.OnPublication( + EmitPublication( disseminationNamespace.Name, accepted, reason: accepted ? "none" : "queue-rejected"); @@ -174,7 +174,15 @@ public async Task ReceiveBroadcast( foreach (var (namespaceName, values) in selectedValues) { var disseminationNamespace = _namespaces[namespaceName]; - DisseminationInstruments.OnBroadcastReceived(disseminationNamespace.Name, "tree", values.Count); + try + { + DisseminationInstruments.OnBroadcastReceived(disseminationNamespace.Name, "tree", values.Count); + } + catch (Exception exception) + { + LogDebugProtocolDiagnosticFailed(_logger, exception, batch.Sender, "broadcast-receive"); + } + ConfirmPeerNamespaces(batch.Sender, [namespaceName]); var namespaceKeys = new Dictionary(); receivedKeys.Add(disseminationNamespace, namespaceKeys); @@ -571,7 +579,8 @@ private Dictionary> CreateAntiEntropyR exchangeTask.Ignore(); var response = await exchangeTask.WaitAsync(cancellationToken).ConfigureAwait(false); // Record exchange metrics after the peer returns so truncation and repair counts reflect the response. - DisseminationInstruments.OnAntiEntropyExchange( + EmitAntiEntropyExchange( + peer, "out", requestDigestCount, GetValueCount(response.Values), @@ -584,13 +593,13 @@ private Dictionary> CreateAntiEntropyR } catch (OperationCanceledException) when (lifetimeCancellationToken.IsCancellationRequested) { - DisseminationInstruments.OnAntiEntropyFailure(DisseminationFailureReason.Timeout); + EmitAntiEntropyFailure(peer, DisseminationFailureReason.Timeout); return null; } catch (Exception exception) { // Anti-entropy transport failures are isolated to the peer; random peer selection naturally spreads retries. - DisseminationInstruments.OnAntiEntropyFailure(DisseminationFailureReason.Error); + EmitAntiEntropyFailure(peer, DisseminationFailureReason.Error); LogDebugDisseminationSendFailed(_logger, exception, peer); return null; } @@ -955,7 +964,7 @@ private DisseminationAntiEntropyResponse CreateAntiEntropyResponse( continue; } - if (localDigest.Version < peerDigest.Version + if (localDigest.Version < peerDigest.Version && !requestedNamespace.ValidateOlderFullValues || localDigest.Version == peerDigest.Version && localDigest.Fingerprint == peerDigest.Fingerprint) { @@ -1051,7 +1060,7 @@ private DisseminationAntiEntropyResponse CreateAntiEntropyResponse( ClearAntiEntropyResponseCursor(request.Sender); } - DisseminationInstruments.OnAntiEntropyExchange("in", GetDigestCount(request.Digests), valueCount, truncated); + EmitAntiEntropyExchange(request.Sender, "in", GetDigestCount(request.Digests), valueCount, truncated); return new DisseminationAntiEntropyResponse { Sender = _localSilo, @@ -1572,7 +1581,7 @@ private static bool TryGetTerminalApplyResult( } var localVersion = disseminationNamespace.GetVersion(value.Key); - if (value.ToVersion < localVersion) + if (value.ToVersion < localVersion && !(value.FromVersion == 0 && disseminationNamespace.ValidateOlderFullValues)) { result = DisseminationApplyResult.Obsolete; return true; @@ -1620,6 +1629,42 @@ private void EmitApplyResult(DisseminationNamespace namespaceName, Dissemination private static int GetDigestCount(Dictionary> digest) => digest.Values.Sum(entries => entries.Count); + private void EmitPublication(DisseminationNamespace namespaceName, bool accepted, string reason) + { + try + { + DisseminationInstruments.OnPublication(namespaceName, accepted, reason); + } + catch (Exception exception) + { + LogDebugProtocolDiagnosticFailed(_logger, exception, _localSilo, "publication"); + } + } + + private void EmitAntiEntropyExchange(SiloAddress peer, string direction, int digests, int values, bool truncated) + { + try + { + DisseminationInstruments.OnAntiEntropyExchange(direction, digests, values, truncated); + } + catch (Exception exception) + { + LogDebugProtocolDiagnosticFailed(_logger, exception, peer, "anti-entropy-exchange"); + } + } + + private void EmitAntiEntropyFailure(SiloAddress peer, DisseminationFailureReason reason) + { + try + { + DisseminationInstruments.OnAntiEntropyFailure(reason); + } + catch (Exception exception) + { + LogDebugProtocolDiagnosticFailed(_logger, exception, peer, "anti-entropy-failure"); + } + } + private static Dictionary CreateDigestLookup(List digest) { var result = new Dictionary(digest.Count); @@ -1670,6 +1715,11 @@ private readonly record struct AntiEntropyResponseCandidate( DigestEntry LocalDigest, DigestEntry PeerDigest); + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination protocol diagnostic for {Peer} during {Operation} failed.")] + private static partial void LogDebugProtocolDiagnosticFailed(ILogger logger, Exception exception, SiloAddress peer, string operation); + [LoggerMessage( Level = LogLevel.Debug, Message = "Dissemination send to {Peer} failed.")] diff --git a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs index 47c7d06f8ea..34022041052 100644 --- a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs @@ -12,6 +12,9 @@ internal interface IDisseminationNamespace DisseminationRoutingMode RoutingMode => DisseminationRoutingMode.BroadcastTree; + // Full values can be authority-refresh hints even when their numeric version is older. + bool ValidateOlderFullValues => false; + DisseminationNamespaceOptions Options { get; } IEnumerable Digests { get; } diff --git a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs index 3082664341c..11af67fa82a 100644 --- a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs @@ -25,6 +25,8 @@ internal sealed class MembershipDisseminationNamespace( public DisseminationMembershipScope MembershipScope => DisseminationMembershipScope.AllMembers; + public bool ValidateOlderFullValues => true; + public DisseminationNamespaceOptions Options => options.CurrentValue.Dissemination; public IEnumerable Keys => MembershipKeys; @@ -163,11 +165,6 @@ public async ValueTask ApplyValueAsync( // The membership manager merges maximum per-entry IAmAliveTime before publishing a full snapshot. var currentSnapshot = membershipManager.CurrentSnapshot; - if (snapshot.Version < currentSnapshot.Version) - { - return DisseminationApplyResult.Obsolete; - } - if (snapshot.Version == currentSnapshot.Version) { // Same-version snapshots can advance liveness or remove inactive entries. @@ -177,7 +174,10 @@ public async ValueTask ApplyValueAsync( } } - return await ApplySnapshot(currentSnapshot, snapshot, cancellationToken); + var result = await ApplySnapshot(currentSnapshot, snapshot, cancellationToken); + return snapshot.Version < currentSnapshot.Version && result == DisseminationApplyResult.Duplicate + ? DisseminationApplyResult.Obsolete + : result; } private DisseminationValue CreateSnapshotValue(MembershipTableSnapshot snapshot) => new( diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs index 09d3656af23..7ed06b4fd31 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs @@ -8,6 +8,97 @@ namespace UnitTests.Dissemination; public partial class DisseminationProtocolTests { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OlderFullMembershipValueReachesAuthoritativeOwner(bool antiEntropy) + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41011); + var peer = CreateSilo(41012); + var current = CreateMembershipSnapshot(100, CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + var notification = CreateMembershipSnapshot(2, current.Entries.Values.ToArray()); + var authority = CreateMembershipSnapshot(101, current.Entries.Values.Select(entry => + { + var result = entry.Copy(); + result.HostName = "authoritative"; + return result; + }).ToArray()); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(current); + var validations = 0; + manager.ProcessGossipSnapshotHandler = (snapshot, token) => + { + token.ThrowIfCancellationRequested(); + Assert.Equal(notification.Version, snapshot.Version); + validations++; + manager.CurrentSnapshot = authority; + return Task.CompletedTask; + }; + var ns = CreateMembershipNamespace(manager, serializer); + var transport = new FakeTransport(local, peer); + var protocol = CreateProtocol(transport, [ns]); + var values = CreateValueGroups(ns.Name, CreateDisseminationValue(peer, new( + DisseminationKey.Default, 0, 2, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = notification })))); + try + { + if (antiEntropy) + { + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult( + new DisseminationAntiEntropyResponse { Sender = peer, Values = values }); + await protocol.RunAntiEntropyRound(cancellationToken); + } + else + { + var response = await protocol.ReceiveBroadcast(new() { Sender = peer, Values = values }, cancellationToken); + Assert.Equal(101, Assert.Single(response.Acknowledgments[ns.Name]).Version); + } + + Assert.Equal(1, validations); + Assert.Same(authority, manager.CurrentSnapshot); + Assert.Equal("authoritative", manager.CurrentSnapshot.Entries[local].HostName); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task AntiEntropyConsultsMembershipNamespaceWhenPeerVersionIsAhead() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41013); + var peer = CreateSilo(41014); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var ns = CreateMembershipNamespace(new FakeMembershipManager(CreateMembershipSnapshot( + 2, CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch))), serializer); + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns]); + try + { + var response = await protocol.ReceiveAntiEntropy(new() + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = new() { [ns.Name] = [new(DisseminationKey.Default, 100)] }, + }, cancellationToken); + + var value = Assert.Single(GetAntiEntropyResponseValues(response)).Value; + Assert.Equal(0, value.FromVersion); + Assert.Equal(2, value.ToVersion); + Assert.NotNull(Assert.IsType( + serializer.Deserialize(value.Payload)).Snapshot); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + [Fact] public void MembershipRepairSendsFullStateWhenPeerVersionIsAhead() { diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs new file mode 100644 index 00000000000..82f545c0588 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs @@ -0,0 +1,159 @@ +using System.Diagnostics.Metrics; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData("broadcast-receive")] + [InlineData("broadcast-send")] + [InlineData("repair-receive")] + [InlineData("repair-send")] + [InlineData("publication")] + [InlineData("repair-failure")] + [InlineData("broadcast-failure")] + public async Task MetricFailuresPreserveProtocolResults(string operation) + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41021); + var peer = CreateSilo(41022); + var healthy = CreateSilo(41023); + var transport = operation == "repair-failure" ? new FakeTransport(local, peer, healthy) : new FakeTransport(local, peer); + var ns = new FakeNamespace(local, new DisseminationNamespace("metric-boundary-" + operation)); + ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); + var logger = new Phase6ProtocolLogger(); + var protocol = CreatePhase6Protocol(transport, ns, logger, options => options.Overlay.AntiEntropyPeerCount = 2); + var metric = operation switch + { + "broadcast-receive" => "orleans-dissemination-broadcast-received", + "broadcast-send" => "orleans-dissemination-broadcast-sent", + "publication" => DisseminationInstruments.PublicationsName, + "repair-failure" => DisseminationInstruments.AntiEntropyFailuresName, + "broadcast-failure" => DisseminationInstruments.BroadcastSendFailuresName, + _ => "orleans-dissemination-anti-entropy-exchanges", + }; + var failure = new InvalidOperationException("Metric observer failure."); + var failures = 0; + var listenerArmed = 1; + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, owner) => + { + if (Volatile.Read(ref listenerArmed) != 0 + && instrument.Meter.Name == DisseminationInstruments.MeterName && instrument.Name == metric) + { + owner.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((_, _, tags, _) => + { + if (Volatile.Read(ref listenerArmed) == 0) + { + return; + } + + if (operation is "broadcast-receive" or "broadcast-send" or "publication") + { + var matches = false; + foreach (var tag in tags) + { + matches |= tag.Key == "namespace" && Equals(tag.Value, ns.Name); + } + if (!matches) + { + return; + } + } + + Interlocked.Increment(ref failures); + throw failure; + }); + listener.Start(); + try + { + switch (operation) + { + case "broadcast-receive": + { + var response = await protocol.ReceiveBroadcast(new() + { + Sender = peer, + Values = CreateValueGroups(ns.Name, ns.CreateItem(peer, "value", 2)), + }, cancellationToken); + Assert.Equal(2, ns.GetVersion("value")); + Assert.Equal(2, Assert.Single(response.Acknowledgments[ns.Name]).Version); + break; + } + case "broadcast-send": + case "publication": + ns.SetValue("value", 1); + Assert.True(await protocol.Publish(ns, "value", 1, cancellationToken)); + await protocol.FlushPendingBroadcast(cancellationToken); + Assert.Single(transport.BroadcastBatches); + await protocol.FlushPendingBroadcast(cancellationToken); + Assert.Single(transport.BroadcastBatches); + break; + case "repair-receive": + { + ns.SetValue("value", 2); + var response = await protocol.ReceiveAntiEntropy(new() + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = new() { [ns.Name] = [new("value", 1)] }, + }, cancellationToken); + Assert.Equal(2, Assert.Single(GetAntiEntropyResponseValues(response)).Value.ToVersion); + break; + } + case "repair-send": + case "repair-failure": + ns.SetValue("value", 1); + transport.ExchangeAntiEntropyHandler = (target, _, _) => + operation == "repair-failure" && target.Equals(peer) + ? ValueTask.FromException(new InvalidOperationException("Transport failure.")) + : ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(ns.Name, ns.CreateItem(target, "value", 2)), + }); + await protocol.RunAntiEntropyRound(cancellationToken); + Assert.Equal(2, ns.GetVersion("value")); + break; + case "broadcast-failure": + ns.SetValue("value", 1); + transport.SendBroadcastResponseHandler = (target, batch, _) => + { + transport.BroadcastBatches.Add((target, batch)); + return transport.BroadcastBatches.Count == 1 + ? Task.FromException(new InvalidOperationException("Transport failure.")) + : Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + Assert.True(await protocol.Publish(ns, "value", 1, cancellationToken)); + await protocol.FlushPendingBroadcast(cancellationToken); + await protocol.FlushPendingBroadcast(cancellationToken); + Assert.Equal(2, transport.BroadcastBatches.Count); + Assert.All(transport.BroadcastBatches, batch => + Assert.Equal(1, Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion)); + await protocol.FlushPendingBroadcast(cancellationToken); + Assert.Equal(2, transport.BroadcastBatches.Count); + break; + default: + throw new InvalidOperationException(operation); + } + + Assert.Equal(1, Volatile.Read(ref failures)); + if (operation is not ("broadcast-send" or "broadcast-failure")) + { + Assert.Contains(logger.Entries, entry => ReferenceEquals(entry.Exception, failure)); + } + } + finally + { + Volatile.Write(ref listenerArmed, 0); + listener.Dispose(); + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs index 95dd362c8ab..2862cc31ea3 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs @@ -87,9 +87,11 @@ private static async Task VerifyPumpFailureMetricIsolation(int failedTimerChange var scheduleMetricFailure = new InvalidOperationException("Retry schedule metric callback fails."); var pumpStatuses = new ConcurrentQueue(); var scheduleMetricFailures = 0; + var listenerArmed = 0; listener.InstrumentPublished = (instrument, meterListener) => { - if (instrument.Meter.Name == DisseminationInstruments.MeterName + if (Volatile.Read(ref listenerArmed) != 0 + && instrument.Meter.Name == DisseminationInstruments.MeterName && (instrument.Name == DisseminationInstruments.PumpFailuresName || failScheduleMetric && instrument.Name == DisseminationInstruments.BroadcastScheduledName)) { @@ -98,6 +100,11 @@ private static async Task VerifyPumpFailureMetricIsolation(int failedTimerChange }; listener.SetMeasurementEventCallback((instrument, _, tags, _) => { + if (Volatile.Read(ref listenerArmed) == 0) + { + return; + } + if (instrument.Name == DisseminationInstruments.PumpFailuresName) { pumpStatuses.Enqueue((string)Assert.Single(tags.ToArray(), static tag => tag.Key == "status").Value!); @@ -163,6 +170,7 @@ private static async Task VerifyPumpFailureMetricIsolation(int failedTimerChange && value.Reason == DisseminationBroadcastScheduleReason.Retry, TimeSpan.FromSeconds(5), cancellationToken) : null; + Volatile.Write(ref listenerArmed, 1); listener.Start(); clock.ThrowOnNextTimerChanges(failedTimerChanges); release[0].TrySetResult(); @@ -261,6 +269,7 @@ private static async Task VerifyPumpFailureMetricIsolation(int failedTimerChange } finally { + Volatile.Write(ref listenerArmed, 0); listener.Dispose(); clock.ThrowOnNextTimerChanges(0); foreach (var completion in release) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index e1a85ed96b8..31e825ed1cf 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -4553,7 +4553,12 @@ public Task ProcessGossipSnapshot(MembershipTableSnapshot snapshot, Cancellation return handler(snapshot, cancellationToken); } - CurrentSnapshot = snapshot; + var previous = CurrentSnapshot; + if (snapshot.IsSuccessorTo(previous)) + { + CurrentSnapshot = MembershipTableSnapshot.Update(previous, snapshot); + } + return Task.CompletedTask; } From e958c36d30a87ead26b02e34f93602095c6c20aa Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 12:13:21 -0700 Subject: [PATCH 36/50] style(dissemination): format metric boundary regression cases --- ...eminationProtocolTests.MetricBoundaries.cs | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs index 82f545c0588..6e00d6b2406 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs @@ -75,16 +75,16 @@ public async Task MetricFailuresPreserveProtocolResults(string operation) switch (operation) { case "broadcast-receive": - { - var response = await protocol.ReceiveBroadcast(new() { - Sender = peer, - Values = CreateValueGroups(ns.Name, ns.CreateItem(peer, "value", 2)), - }, cancellationToken); - Assert.Equal(2, ns.GetVersion("value")); - Assert.Equal(2, Assert.Single(response.Acknowledgments[ns.Name]).Version); - break; - } + var response = await protocol.ReceiveBroadcast(new() + { + Sender = peer, + Values = CreateValueGroups(ns.Name, ns.CreateItem(peer, "value", 2)), + }, cancellationToken); + Assert.Equal(2, ns.GetVersion("value")); + Assert.Equal(2, Assert.Single(response.Acknowledgments[ns.Name]).Version); + break; + } case "broadcast-send": case "publication": ns.SetValue("value", 1); @@ -95,17 +95,17 @@ public async Task MetricFailuresPreserveProtocolResults(string operation) Assert.Single(transport.BroadcastBatches); break; case "repair-receive": - { - ns.SetValue("value", 2); - var response = await protocol.ReceiveAntiEntropy(new() { - Sender = peer, - SupportedNamespaces = [ns.Name], - Digests = new() { [ns.Name] = [new("value", 1)] }, - }, cancellationToken); - Assert.Equal(2, Assert.Single(GetAntiEntropyResponseValues(response)).Value.ToVersion); - break; - } + ns.SetValue("value", 2); + var response = await protocol.ReceiveAntiEntropy(new() + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = new() { [ns.Name] = [new("value", 1)] }, + }, cancellationToken); + Assert.Equal(2, Assert.Single(GetAntiEntropyResponseValues(response)).Value.ToVersion); + break; + } case "repair-send": case "repair-failure": ns.SetValue("value", 1); From 4acf6f35122d9b2f16eecb6e34ebbd04f3cefd8f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 12:25:08 -0700 Subject: [PATCH 37/50] fix(membership): stop invalidated development table incarnations --- .../implementation/runtime-dissemination.md | 2 + .../MembershipTableManager.cs | 34 +++++++-- .../Membership/MembershipTableManagerTests.cs | 71 +++++++++++++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index 795bb9e79ca..18d5905f9d6 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -57,6 +57,8 @@ Membership uses the membership-table version plus a liveness fingerprint in its Development clustering recreates its volatile membership table when the primary restarts. Its manager treats received membership snapshots as notifications to perform a fresh authoritative read, so retained peer state from the prior table cannot replace the recreated table's view. Durable membership providers retain snapshot-based application. Dissemination records current owner state rather than delayed publication arguments, and clears retained history and serialized payloads when the owner replaces its view with a lower version. +If an existing development silo observes a table counter below the version it knew when it started the read, it requests local shutdown and reports that it must restart against the recreated table. This preserves the monotonic membership-version contract shared with directory and other runtime consumers. A read which began before a newer local observation can finish normally; its older result is ignored. Restart the affected silos after recreating development membership. + The membership namespace requests authoritative validation for older full values. Broadcast and anti-entropy let those values reach the owner, and repair selection consults the namespace for numerically greater peer baselines. The owner decides the resulting state; delta chains and monotonic namespaces retain their numeric version guards. Deployment-load versions are sample timestamps. Each repair is a full latest value. Application runs on the deployment-load publisher's scheduler and accepts samples only for Active silo generations in the membership oracle. Terminal membership transitions remove placement statistics, and the oracle's monotonic membership state rejects later samples for departed generations. diff --git a/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs b/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs index 9d08d66ae62..2d88346886d 100644 --- a/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs +++ b/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs @@ -193,7 +193,7 @@ public async Task RefreshFromSnapshot(MembershipTableSnapshot snapshot, Cancella private async Task RefreshInternal(bool requireCleanup, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var table = await this.membershipTableProvider.ReadAllAsync(cancellationToken); + var table = await this.ReadMembershipTable(cancellationToken); bool success; try @@ -215,6 +215,28 @@ private async Task RefreshInternal(bool requireCleanup, CancellationToken return !requireCleanup || success; } + private Task ReadMembershipTable(CancellationToken cancellationToken) => + this.membershipTableProvider is SystemTargetBasedMembershipTable + ? ReadDevelopmentMembershipTable(cancellationToken) + : this.membershipTableProvider.ReadAllAsync(cancellationToken); + + private async Task ReadDevelopmentMembershipTable(CancellationToken cancellationToken) + { + // Capture before issuing the read: an older read completing after a newer one is not a reset. + var observedVersion = this.MembershipTableSnapshot.Version; + var table = await this.membershipTableProvider.ReadAllAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + if (table.Version.Version < observedVersion.Value) + { + var reason = $"The development membership table version decreased from {observedVersion} to {table.Version.Version}. " + + "Its previous incarnation is no longer authoritative. Restart this silo against the recreated table."; + this.KillMyselfLocally(reason); + throw new OrleansException(reason); + } + + return table; + } + private async Task Start(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -455,7 +477,7 @@ async Task UpdateMyStatusTask(int counter, CancellationToken token) private async Task TryUpdateMyStatusGlobalOnce(SiloStatus newStatus, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var table = await membershipTableProvider.ReadAllAsync(cancellationToken); + var table = await ReadMembershipTable(cancellationToken); LogDebugTryUpdateMyStatusGlobalOnce(this.log, newStatus.Equals(SiloStatus.Active) ? "All" : " my entry from", table.ToString()); LogMissedIAmAlives(table); @@ -915,7 +937,7 @@ private void PublishSuspectOrKillRequestCompletion(SuspectOrKillRequest request, private async Task InnerTryKill(SiloAddress silo, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var table = await membershipTableProvider.ReadAllAsync(cancellationToken); + var table = await ReadMembershipTable(cancellationToken); LogDebugTryKillReadMembershipTable(this.log, table.ToString()); @@ -962,7 +984,7 @@ public async Task TryToSuspectOrKill(SiloAddress silo, SiloAddress? indire private async Task InnerTryToSuspectOrKill(SiloAddress silo, SiloAddress? indirectProbingSilo, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var table = await membershipTableProvider.ReadAllAsync(cancellationToken); + var table = await ReadMembershipTable(cancellationToken); var now = GetDateTimeUtcNow(); LogDebugTryToSuspectOrKillReadMembershipTable(this.log, table.ToString()); @@ -1037,7 +1059,7 @@ private async Task InnerTryToSuspectOrKill(SiloAddress silo, SiloAddress? var ok = await membershipTableProvider.UpdateRowAsync(entry, eTag, table.Version.Next(), cancellationToken); if (ok) { - table = await membershipTableProvider.ReadAllAsync(cancellationToken); + table = await ReadMembershipTable(cancellationToken); this.ProcessTableUpdate(table, "TrySuspectOrKill"); // Gossip using the local silo status, since this is just informational to propagate the suspicion vote. @@ -1069,7 +1091,7 @@ private async Task DeclareDead(MembershipEntry entry, string etag, TableVe { LogDebugSuccessfullyUpdatedStatusToDead(this.log, entry.SiloAddress); - var table = await membershipTableProvider.ReadAllAsync(cancellationToken); + var table = await ReadMembershipTable(cancellationToken); this.ProcessTableUpdate(table, "DeclareDead"); GossipToOthers(entry.SiloAddress, entry.Status, GossipTimeout, cancellationToken).Ignore(); return true; diff --git a/test/Orleans.Core.Tests/Membership/MembershipTableManagerTests.cs b/test/Orleans.Core.Tests/Membership/MembershipTableManagerTests.cs index e4f884e0f92..7eb724c314d 100644 --- a/test/Orleans.Core.Tests/Membership/MembershipTableManagerTests.cs +++ b/test/Orleans.Core.Tests/Membership/MembershipTableManagerTests.cs @@ -1723,6 +1723,77 @@ public override void Post(SendOrPostCallback callback, object? state) => } } + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task MembershipTableResetStopsOnlyTheVolatileProviderIncarnation(bool developmentProvider) + { + var cancellationToken = TestContext.Current.CancellationToken; + var original = await new InMemoryMembershipTable(new TableVersion(100, "old"), + Entry(this.localSilo, SiloStatus.Active, DateTimeOffset.UtcNow)).ReadAllAsync(cancellationToken); + var reset = await new InMemoryMembershipTable(new TableVersion(1, "new")).ReadAllAsync(cancellationToken); + var backing = new LegacyMembershipTable(Substitute.For()); + backing.ConfigureReadAll(original); + IMembershipTable provider = developmentProvider ? CreateSystemTargetBasedMembershipTable(backing) : backing; + await provider.InitializeMembershipTableAsync(true, cancellationToken); + using var manager = CreateMembershipTableManager(provider); + await manager.Refresh(cancellationToken: cancellationToken, requireFresh: true); + var accepted = manager.MembershipTableSnapshot; + Assert.Equal(new MembershipVersion(100), accepted.Version); + backing.ConfigureReadAll(reset); + + if (developmentProvider) + { + var error = await Assert.ThrowsAsync( + () => manager.Refresh(cancellationToken: cancellationToken, requireFresh: true)); + Assert.Contains("version decreased from 100 to 1", error.Message, StringComparison.Ordinal); + this.fatalErrorHandler.Received(1).OnFatalException( + manager, Arg.Is(reason => reason.Contains("Restart this silo", StringComparison.Ordinal)), null); + } + else + { + await manager.Refresh(cancellationToken: cancellationToken, requireFresh: true); + this.fatalErrorHandler.DidNotReceiveWithAnyArgs().OnFatalException(default, default, default); + } + + Assert.Same(accepted, manager.MembershipTableSnapshot); + } + + [Fact] + public async Task OlderDevelopmentReadCompletingAfterNewerReadIsNotATableReset() + { + var cancellationToken = TestContext.Current.CancellationToken; + var original = await new InMemoryMembershipTable(new TableVersion(100, "old"), + Entry(this.localSilo, SiloStatus.Active, DateTimeOffset.UtcNow)).ReadAllAsync(cancellationToken); + var newer = new MembershipTableData(original.Members.ToList(), new TableVersion(101, "new")); + var backing = new LegacyMembershipTable(Substitute.For()); + backing.ConfigureReadAll(original); + var provider = CreateSystemTargetBasedMembershipTable(backing); + await provider.InitializeMembershipTableAsync(true, cancellationToken); + using var manager = CreateMembershipTableManager(provider); + await manager.Refresh(cancellationToken: cancellationToken, requireFresh: true); + var read = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + backing.ConfigureReadAll(read.Task); + var pending = manager.Refresh(cancellationToken: cancellationToken, requireFresh: true); + try + { + Assert.False(pending.IsCompleted); + backing.ConfigureReadAll(newer); + await manager.Refresh(cancellationToken: cancellationToken, requireFresh: true); + Assert.Equal(new MembershipVersion(101), manager.MembershipTableSnapshot.Version); + read.SetResult(original); + await pending.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + + Assert.Equal(new MembershipVersion(101), manager.MembershipTableSnapshot.Version); + this.fatalErrorHandler.DidNotReceiveWithAnyArgs().OnFatalException(default, default, default); + } + finally + { + read.TrySetResult(original); + await pending.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + } + private sealed class BackoffTimeProvider : FakeTimeProvider { public TaskCompletionSource TimerCreated { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); From 15ee0789d76a7ba245d8e45a6e5f8484b2d9a415 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 16:02:22 -0700 Subject: [PATCH 38/50] perf(dissemination): pace root microbatches and bound maintenance work --- .../implementation/runtime-dissemination.md | 19 +- .../Options/DisseminationOptions.cs | 52 +- .../Options/DeploymentLoadPublisherOptions.cs | 10 +- .../Options/DisseminationOptionsValidator.cs | 20 + .../DisseminationMembershipSnapshot.cs | 3 +- .../Dissemination/DisseminationProtocol.cs | 234 ++++- .../Dissemination/DisseminationRootBatcher.cs | 630 +++++++++++++ src/api/Orleans.Core/Orleans.Core.cs | 8 + .../DisseminationAggregationOptionsTests.cs | 278 ++++++ .../DisseminationProtocolTests.Aggregation.cs | 15 +- .../DisseminationProtocolTests.Maintenance.cs | 178 ++++ .../DisseminationProtocolTests.RootPacing.cs | 170 ++++ .../DisseminationProtocolTests.cs | 3 +- .../DisseminationRootBatcherTests.cs | 883 ++++++++++++++++++ 14 files changed, 2449 insertions(+), 54 deletions(-) create mode 100644 src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Maintenance.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.RootPacing.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index 18d5905f9d6..cfb15186597 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -31,7 +31,7 @@ flowchart LR Repair --> Authority ``` -`DisseminationProtocol` coordinates routing, peer capability evidence, anti-entropy, and application isolation. `DisseminationBroadcastQueue` owns per-peer scheduling, coalescing, retry, drain, and acknowledged-version ledgers. `DisseminationMembership` projects one membership snapshot into topology-specific member sets. Each `IDisseminationNamespace` owns serialization, current versions, retained history, payload limits, repair construction, and application semantics. +`DisseminationProtocol` coordinates routing, peer capability evidence, anti-entropy, and application isolation. `DisseminationRootBatcher` owns the aggregation root's bounded, latest-key pending set and broadcast cadence. `DisseminationBroadcastQueue` owns per-peer scheduling, retry, drain, and acknowledged-version ledgers. `DisseminationMembership` projects one membership snapshot into topology-specific member sets. Each `IDisseminationNamespace` owns serialization, current versions, retained history, payload limits, repair construction, and application semantics. Queue entries contain `(namespace, key)` identities. A pump acquires both destination ownership and a global broadcast slot before materializing a repair. Coalesced notifications therefore use the latest namespace state and acknowledged baseline after admission, without retaining serialized payloads for waiting peers. @@ -71,11 +71,13 @@ Every silo derives routing from the same ordered membership projection. Members Membership uses the broadcast forest: for fanout `f` and zero-based member index `i`, a forwarding node selects children starting at `f * (i + 1)` and continuing for at most `f` members. An originator sends to the first `f` members, excluding itself, plus its normal forwarding children. -Deployment load uses root aggregation followed by tree distribution. Node `i > 0` has parent `(i - 1) / f`; its children start at `f * i + 1`. Each non-root producer sends its own update directly to the root immediately. The root merges updates and batches them for 500 ms. Relays atomically enqueue every key from a received batch before waking their send loops, then forward to their children immediately. The batching delay is paid at the root once, rather than at every hop. Queues retain the latest version of a repeatedly published key, and acknowledgments suppress reflected duplicates. +Deployment load uses root aggregation followed by tree distribution. Node `i > 0` has parent `(i - 1) / f`; its children start at `f * i + 1`. Its fanout defaults to eight, independently of the membership forest. Each non-root producer sends its own update directly to the root immediately. The root retains one latest notification per key and collects for 25 ms. It starts at most five normal broadcast waves per second, so the next deadline is the later of the first pending notification plus the collection window and the previous wave start plus 200 ms. New arrivals keep that deadline. An idle root sends after the collection window, while sustained traffic combines updates between paced waves. + +Admitted waves enter every child queue atomically. Relays forward immediately. The root's pending set stays outside the per-peer queues, so urgent membership traffic can proceed without prematurely flushing unadmitted load values. Explicit flush and shutdown drain can bypass normal pacing within their caller's cancellation budget. A former root hands pending state to the current root, including when it has left Active membership. Aggregation ledgers advance on broadcast acknowledgments, after the receiver processes the batch and attempts downstream queue admission. An ingress producer which is also a child receives its own update in the root's distribution batch and forwards it to its descendants. Passive evidence of a peer's local value serves repair; broadcast acknowledgments establish distribution progress. Anti-entropy repairs gaps left by bounded queue admission or changed topology. -At one publication per silo per second, root ingress contributes `N - 1` requests per second. Up to two 500 ms root batches traverse `N - 1` distribution edges, targeting roughly `3 * (N - 1)` requests per second under stable, healthy, fitting-batch conditions. One fully collected publication round requires `2 * (N - 1)` requests. Acknowledgment responses add the corresponding return messages. Batch splitting, repair, topology transitions, and other runtime traffic are accounted separately. Payload delivery still includes each recipient's copy of the statistics; the savings come from aggregating those copies into fewer messages. +At one publication per silo per second, root ingress contributes `N - 1` requests per second. With `R` root waves per second, healthy fitting batches add at most roughly `R * (N - 1)` distribution requests. At the default five waves per second, the model is `6 * (N - 1)`, or 11,994 requests per second for 2,000 silos. An eight-child relay sends up to approximately 40 distribution requests per second. These are protocol projections, not measured multi-host capacity. Acknowledgment replies, retries, repair, topology transitions, forced drains, and wire-message splitting add separate work. Payload delivery still includes each recipient's copy of the statistics. Active members are ordered by silo address. Ingress moves toward a smaller root; distribution moves toward larger children. A non-root node receiving an ingress from a higher-address sender redirects it to its current root immediately, accommodating a changed root without adding another batching window. Membership repair reconciles differing views. @@ -86,7 +88,7 @@ Namespaces select a membership scope before topology construction: | Deployment load | Active members, root aggregation | Producers send to one root; already-aggregated batches traverse the distribution tree immediately. | | Membership | All dissemination members | Joining and graceful-shutdown transitions can propagate. | -Fanout is derived from the target hop count and bounded by the configured minimum and maximum, or selected by the code-configured callback. A membership or fanout change creates a new topology from the next snapshot; acknowledged ledgers and anti-entropy repair convergence across the transition. +Membership fanout is derived from the target hop count and configured bounds, or selected by the code-configured callback. Aggregation uses its independent fanout setting. At 2,000 silos, fanout eight gives at most four distribution hops. A membership change creates a new topology from the next snapshot; acknowledged ledgers and anti-entropy repair convergence across the transition. The projection cache tracks its authoritative source snapshot. On a cache miss it re-reads the owner under the cache lock, so a delayed reader follows the current view, including an owner-authorized version replacement. Eligible membership and status changes refresh routing and peer selection. Heartbeat-only updates and cleanup of non-participants reuse the existing topology, while rebuilt projections retain rotation progress. @@ -101,9 +103,12 @@ Dissemination is opt-in. Enable | 1 MiB | Serialized payload bytes in an outgoing batch or admitted from an incoming batch or repair response. | | | 2 | Target depth used to derive fanout. | | / | 4 / 32 | Bounds for derived fanout. | +| | 8 | Maximum children per load-distribution relay. | +| | 5 | Normal root broadcast-wave frequency. | | | 5 seconds | Repair-round cadence and retry-delay ceiling. | | | 3 | Maximum peers selected in one repair round and independent per-silo active local repair attempt limit. | -| | 1,024 | Distinct retained keys per namespace and peer. | +| / | 8,192 / 1 MiB | Independent repair limits, capped by the global batch limits and negotiated with the peer. | +| | 1,024; load uses 8,192 | Distinct retained keys per namespace and peer, and in a root's pending set. | | | 100 ms | Normal-priority batching window. | | | 30 seconds | Independent local transport and application budgets for each hop. | | | 10 seconds | Quiet period before a digest is offered for repair. | @@ -111,7 +116,9 @@ Dissemination is opt-in. Enable . Operators can enable and tune membership and deployment-load dissemination independently while retaining the local concurrency and per-message bounds. -Deployment load uses a 500 ms root batching window and a 5-second expected update cadence. Producer ingress and relay forwarding bypass that window; relays preserve whole batches atomically. Membership retains high priority and bypasses coalescing. +Deployment load samples remain on the one-second publication cadence. Root collection defaults to 25 ms with a five-wave-per-second admission limit, and the expected repair-update cadence remains five seconds. Producer ingress and relay forwarding bypass collection delays. Membership retains high priority and bypasses coalescing. Configure the load pending-key bound to cover the intended active inventory and stalls; each newer notification replaces the retained value for that key rather than adding another sample. + +Inventory and peer-ledger maintenance runs on membership changes and bounded one-second maintenance opportunities instead of every received update. Failed passes remain eligible for retry. This keeps full inventory scans out of the normal per-message root path while promptly retiring peers when the routing view changes. The anti-entropy loop waits without periodic timer wakeups while the subsystem is disabled. An options-change notification wakes the loop when enablement changes; disabling it returns the loop to the dormant wait. Shutdown removes the options subscription and observes the loop's completion. diff --git a/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs index c6110a78d99..5dc39d7696e 100644 --- a/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs +++ b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs @@ -49,14 +49,15 @@ public sealed class DisseminationOptions public sealed class DisseminationOverlayOptions { /// - /// Gets or sets the code-configured fanout selector. + /// Gets or sets the code-configured fanout selector for the membership broadcast forest. /// /// - /// The argument is the current member count for the selected dissemination topology. + /// The argument is the current member count for the membership broadcast forest. /// The selector must return the same fanout on every silo for a given member count so that forwarding /// routes agree across the cluster. /// When this value is , , , /// and are used to derive a fanout factor. + /// Aggregation trees use instead. /// public Func? FanOutFactor { get; set; } @@ -75,6 +76,28 @@ public sealed class DisseminationOverlayOptions /// public int MaxFanOutFactor { get; set; } = 32; + /// + /// Gets or sets the maximum number of children per node in the aggregation distribution tree. + /// + /// + /// This value is independent of the membership broadcast forest's selector + /// and is clamped to the member count. Configure the same value on every silo so that distribution + /// routes agree across the cluster. Producers continue to send updates directly to the aggregation root. + /// + /// The default is 8 and the value must be greater than zero. + public int AggregationFanOutFactor { get; set; } = 8; + + /// + /// Gets or sets the maximum number of logical aggregation distribution waves admitted by the root per second. + /// + /// + /// Rate admission can add a wait beyond the namespace's collection window. Relays forward admitted waves + /// immediately without another collection window or rate admission wait. The batch item and byte limits + /// can split one logical wave into multiple wire messages, which do not each consume a separate wave allowance. + /// + /// The default is 5 and the value must be between 1 and 1000, inclusive. + public int AggregationBroadcastsPerSecond { get; set; } = 5; + /// /// Gets or sets the interval between anti-entropy repair rounds. /// @@ -90,6 +113,28 @@ public sealed class DisseminationOverlayOptions /// public int AntiEntropyPeerCount { get; set; } = 3; + /// + /// Gets or sets the maximum number of items in one anti-entropy repair batch. + /// + /// + /// The effective limit is the minimum of this value, , + /// and the peer's advertised receive budget. This allows repair batches to be tuned independently + /// of broadcast batches. + /// + /// The default is 8192 and the value must be greater than zero. + public int MaxAntiEntropyBatchItems { get; set; } = 8 * 1024; + + /// + /// Gets or sets the maximum total payload bytes in one anti-entropy repair batch. + /// + /// + /// The effective limit is the minimum of this value, , + /// and the peer's advertised receive budget. This allows repair batches to be tuned independently + /// of broadcast batches. + /// + /// The default is 1048576 and the value must be greater than zero. + public int MaxAntiEntropyBatchBytes { get; set; } = 1024 * 1024; + internal int GetFanOutFactor(int memberCount) { var count = Math.Max(1, memberCount); @@ -153,6 +198,9 @@ public sealed class DisseminationNamespaceOptions /// Per-peer batches can contain values from multiple namespaces and use the shortest configured delay among /// enabled namespaces. High-priority namespaces (see ) do not coalesce and are excluded /// from this calculation. + /// For aggregation trees, this is the root's collection window. Admission under + /// can add a separate wait; + /// relays forward admitted waves immediately. Batch limits can split a logical wave into multiple wire messages. /// /// The delay is 100 milliseconds by default and must be between 1 millisecond and approximately 49.7 days. public TimeSpan MaxCoalescingDelay { get; set; } = TimeSpan.FromMilliseconds(100); diff --git a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs index 13ae59021b8..ce1d26884a2 100644 --- a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs @@ -22,8 +22,11 @@ public class DeploymentLoadPublisherOptions /// /// /// When all active peers confirm support, producers send updates directly to the aggregation root. - /// The root batches for 500 milliseconds by default, then relays forward whole batches immediately - /// through the parent/child tree. Confirmation remains valid for that silo generation + /// By default, the root collects updates for 25 milliseconds and admits at most 5 logical distribution + /// waves per second. The root's rate admission wait is separate from the collection window. Relays forward + /// admitted waves immediately through the parent/child tree, without another collection window or rate + /// admission wait. Batch item and byte limits can split one logical wave into multiple wire messages. + /// Confirmation remains valid for that silo generation /// until the peer explicitly rejects the namespace or leaves the eligible membership set. During bootstrap /// or mixed-version operation, direct publication covers all active peers so that an unsupported intermediate /// node cannot interrupt delivery. Direct publication also covers all active peers when dissemination is @@ -32,7 +35,8 @@ public class DeploymentLoadPublisherOptions public DisseminationNamespaceOptions Dissemination { get; set; } = new() { ExpectedUpdateCadence = TimeSpan.FromSeconds(5), - MaxCoalescingDelay = TimeSpan.FromMilliseconds(500), + MaxCoalescingDelay = TimeSpan.FromMilliseconds(25), + MaxPendingItemCount = 8 * 1024, }; } } diff --git a/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs b/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs index 4e364923cbc..2948a7fa168 100644 --- a/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs +++ b/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs @@ -40,6 +40,16 @@ public ValidateOptionsResult Validate(string? name, DisseminationOptions options return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.MaxFanOutFactor)} must be greater than or equal to {nameof(DisseminationOverlayOptions.MinFanOutFactor)}."); } + if (overlay.AggregationFanOutFactor <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.AggregationFanOutFactor)} must be greater than 0."); + } + + if (overlay.AggregationBroadcastsPerSecond is < 1 or > 1000) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.AggregationBroadcastsPerSecond)} must be between 1 and 1000."); + } + if (overlay.AntiEntropyInterval < TimeSpan.FromMilliseconds(1) || overlay.AntiEntropyInterval > MaxPeriodicTimerPeriod) { @@ -52,6 +62,16 @@ public ValidateOptionsResult Validate(string? name, DisseminationOptions options return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.AntiEntropyPeerCount)} must be greater than 0."); } + if (overlay.MaxAntiEntropyBatchItems <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.MaxAntiEntropyBatchItems)} must be greater than 0."); + } + + if (overlay.MaxAntiEntropyBatchBytes <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.MaxAntiEntropyBatchBytes)} must be greater than 0."); + } + return ValidateOptionsResult.Success; } } diff --git a/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs b/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs index 8cc998045ee..7b80a93a592 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs @@ -27,6 +27,7 @@ public DisseminationMembershipSnapshot( // frequent updates cannot keep repair rounds confined to the first few members. _antiEntropySelection = previous?._antiEntropySelection ?? new(); var fanout = Members.Length <= 1 ? 1 : overlayOptions.GetFanOutFactor(Members.Length); + var aggregationFanout = Math.Clamp(overlayOptions.AggregationFanOutFactor, 1, Math.Max(1, Members.Length)); var memberSet = new HashSet(Members.Length); var localIndex = -1; for (var i = 0; i < Members.Length; i++) @@ -45,7 +46,7 @@ public DisseminationMembershipSnapshot( _set = memberSet.ToFrozenSet(); ForwardingTreeTargets = localIndex < 0 ? [] : ComputeChildren((long)fanout * (localIndex + 1), fanout); - AggregationChildren = localIndex < 0 ? [] : ComputeChildren((long)fanout * localIndex + 1, fanout); + AggregationChildren = localIndex < 0 ? [] : ComputeChildren((long)aggregationFanout * localIndex + 1, aggregationFanout); if (localIndex < 0) { diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index f39705425f1..52998eeb045 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -12,6 +12,7 @@ internal sealed partial class DisseminationProtocol { private const int MaxRetainedNonMemberResponseCursors = 64; private static readonly TimeSpan MaxAntiEntropyRoundLifetime = TimeSpan.FromMilliseconds(uint.MaxValue - 1); + private static readonly TimeSpan InventoryMaintenanceInterval = TimeSpan.FromSeconds(1); private readonly SiloAddress _localSilo; private readonly IInternalGrainFactory _grainFactory; private readonly DisseminationMembership _membership; @@ -19,6 +20,11 @@ internal sealed partial class DisseminationProtocol private readonly TimeProvider _timeProvider; private readonly ILogger _logger; private readonly DisseminationBroadcastQueue _broadcastQueue; + private readonly object _rootBatcherLock = new(); + private readonly Dictionary _rootBatchers = []; + private bool _rootBatchersStopped; + private readonly object _maintenanceLock = new(); + private MaintenanceStamp? _lastMaintenance; private readonly AdmissionGate _admission = new(); private readonly DisseminationSendGate _antiEntropySendGate; private readonly CancellationTokenSource _antiEntropyShutdown = new(); @@ -32,6 +38,7 @@ internal sealed partial class DisseminationProtocol private readonly Dictionary _lastValueUpdates = []; private readonly object _peerSupportLock = new(); private readonly Dictionary> _confirmedPeerNamespaces = []; + private DisseminationMembershipSnapshot? _confirmedMembership; private readonly FrozenDictionary _namespaces; public DisseminationProtocol( @@ -128,11 +135,18 @@ public async ValueTask Publish( // Notifications carry identity only; each peer pump asks the namespace for the latest repair at send time. RecordValueUpdate(disseminationNamespace.Name, key, publishedVersion); var accepted = true; - foreach (var peer in membership.GetOriginatorTargets(disseminationNamespace.RoutingMode)) + if (disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree && membership.IsAggregationRoot) { - accepted &= _broadcastQueue.Notify( - peer, disseminationNamespace, key, - immediate: disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree && !membership.IsAggregationRoot); + accepted = NotifyRoot(disseminationNamespace, [new(key, publishedVersion, true)]); + } + else + { + foreach (var peer in membership.GetOriginatorTargets(disseminationNamespace.RoutingMode)) + { + accepted &= _broadcastQueue.Notify( + peer, disseminationNamespace, key, + immediate: disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree); + } } EmitPublication( @@ -219,9 +233,8 @@ public async Task ReceiveBroadcast( } // Membership may be part of this batch, so apply everything before deriving the forwarding tree. + await MaintainInventories(cancellationToken); var membershipSnapshots = _membership.CurrentSnapshots; - PrunePeerNamespaceConfirmations(membershipSnapshots.AllMembers); - await _broadcastQueue.Prune(membershipSnapshots, cancellationToken); // Changed state always wakes children, including same-version liveness updates. Duplicate deliveries // wake only children which still need this version and have no equivalent queued work. foreach (var (disseminationNamespace, keys) in receivedKeys) @@ -240,12 +253,17 @@ public async Task ReceiveBroadcast( } } - foreach (var peer in membership.GetForwardingTargets(disseminationNamespace.RoutingMode, batch.Sender)) + if (membership.IsAggregationRoot) { - // A producer which is also a child needs its update returned in the distribution batch - // so that it forwards the update to its own descendants. - _broadcastQueue.NotifyBatch( - peer, disseminationNamespace, notifications.AsSpan(0, notificationCount), immediate: !membership.IsAggregationRoot); + NotifyRoot(disseminationNamespace, notifications.AsSpan(0, notificationCount)); + } + else + { + foreach (var peer in membership.GetForwardingTargets(disseminationNamespace.RoutingMode, batch.Sender)) + { + _broadcastQueue.NotifyBatch( + peer, disseminationNamespace, notifications.AsSpan(0, notificationCount), immediate: true); + } } continue; @@ -330,9 +348,8 @@ public async Task RunAntiEntropyRound(CancellationToken cancellationToken) return; } + await MaintainInventories(cancellationToken); var membershipSnapshots = _membership.CurrentSnapshots; - PrunePeerNamespaceConfirmations(membershipSnapshots.AllMembers); - await _broadcastQueue.Prune(membershipSnapshots, cancellationToken); // A round never queues behind prior rounds: busy destinations and slots wait for a future rotation. var leases = AcquireAntiEntropyPeers(membershipSnapshots, options.Overlay.AntiEntropyPeerCount); @@ -480,8 +497,8 @@ private Dictionary CreateAntiEntro Sender = _localSilo, Digests = peerDigests, SupportedNamespaces = supportedNamespaces, - MaxResponseItems = options.MaxBatchItems, - MaxResponseBytes = options.MaxBatchBytes, + MaxResponseItems = Math.Min(options.MaxBatchItems, options.Overlay.MaxAntiEntropyBatchItems), + MaxResponseBytes = Math.Min(options.MaxBatchBytes, options.Overlay.MaxAntiEntropyBatchBytes), }); } } @@ -705,6 +722,8 @@ private Dictionary> Se out bool isUnfiltered) { isUnfiltered = false; + var maxItems = antiEntropy ? Math.Min(options.MaxBatchItems, options.Overlay.MaxAntiEntropyBatchItems) : options.MaxBatchItems; + var maxBytes = antiEntropy ? Math.Min(options.MaxBatchBytes, options.Overlay.MaxAntiEntropyBatchBytes) : options.MaxBatchBytes; long totalCount = 0; foreach (var entries in values.Values) { @@ -720,7 +739,7 @@ private Dictionary> Se : 0; } - if (position == 0 && FitsReceiveBudget(values, totalCount, options)) + if (position == 0 && FitsReceiveBudget(values, totalCount, maxItems, maxBytes)) { var fastPathMembers = _membership.CurrentSnapshots.AllMembers; lock (_receivedBatchCursorLock) @@ -755,14 +774,14 @@ private Dictionary> Se for (var index = (int)skip; index < entries.Count; index++) { - if (examined >= options.MaxBatchItems || byteCount >= options.MaxBatchBytes) + if (examined >= maxItems || byteCount >= maxBytes) { goto Complete; } var item = entries[index]; var payloadBytes = item.Value.Payload.Length; - if (!ValidatePayloadSize(disseminationNamespace, item.Value, options)) + if (!ValidatePayloadSize(disseminationNamespace, item.Value, maxBytes)) { // Oversized entries consume inspection capacity, but must not pin the receive cursor. examined++; @@ -770,7 +789,7 @@ private Dictionary> Se continue; } - if (payloadBytes > options.MaxBatchBytes - byteCount) + if (payloadBytes > maxBytes - byteCount) { // Preserve this candidate for a fresh budget instead of discarding a repair-chain suffix. goto Complete; @@ -815,14 +834,15 @@ private Dictionary> Se private bool FitsReceiveBudget( Dictionary> values, long itemCount, - DisseminationOptions options) + int maxItems, + int maxBytes) { - if (itemCount > options.MaxBatchItems) + if (itemCount > maxItems) { return false; } - var remainingBytes = options.MaxBatchBytes; + var remainingBytes = maxBytes; foreach (var (namespaceName, entries) in values) { if (entries.Count == 0 || !TryGetEnabledNamespace(namespaceName, out var disseminationNamespace)) @@ -935,8 +955,10 @@ private DisseminationAntiEntropyResponse CreateAntiEntropyResponse( // Honor the recipient's budget at the responder's existing fair cursor. Independently truncating // rotating responses at the receiver can repeatedly omit the same keys when the limits differ. - var maxResponseItems = Math.Min(options.MaxBatchItems, request.MaxResponseItems ?? int.MaxValue); - var maxResponseBytes = Math.Min(options.MaxBatchBytes, request.MaxResponseBytes ?? int.MaxValue); + var maxResponseItems = Math.Min(Math.Min(options.MaxBatchItems, options.Overlay.MaxAntiEntropyBatchItems), + request.MaxResponseItems ?? int.MaxValue); + var maxResponseBytes = Math.Min(Math.Min(options.MaxBatchBytes, options.Overlay.MaxAntiEntropyBatchBytes), + request.MaxResponseBytes ?? int.MaxValue); var valueCount = 0; var byteCount = 0; var truncated = false; @@ -1202,7 +1224,7 @@ private async ValueTask ApplyReceivedValueCore( { cancellationToken.ThrowIfCancellationRequested(); var namespaceName = disseminationNamespace.Name; - if (!ValidatePayloadSize(disseminationNamespace, item.Value, options)) + if (!ValidatePayloadSize(disseminationNamespace, item.Value, options.MaxBatchBytes)) { return DisseminationApplyResult.Rejected; } @@ -1266,8 +1288,17 @@ private async ValueTask ApplyReceivedValueCore( return result; } - internal Task FlushPendingBroadcast(CancellationToken cancellationToken) => - _broadcastQueue.FlushPendingBroadcast(cancellationToken); + internal async Task FlushPendingBroadcast(CancellationToken cancellationToken) + { + DisseminationRootBatcher[] roots; + lock (_rootBatcherLock) + { + roots = [.. _rootBatchers.Values]; + } + + await Task.WhenAll(roots.Select(root => root.FlushAsync(cancellationToken))); + await _broadcastQueue.FlushPendingBroadcast(cancellationToken); + } internal async Task StopAsync(CancellationToken cancellationToken) { @@ -1281,7 +1312,22 @@ internal async Task StopAsync(CancellationToken cancellationToken) } finally { - await _broadcastQueue.StopAsync(cancellationToken); + DisseminationRootBatcher[] roots; + lock (_rootBatcherLock) + { + _rootBatchersStopped = true; + roots = [.. _rootBatchers.Values]; + _rootBatchers.Clear(); + } + + try + { + await Task.WhenAll(roots.Select(root => root.StopAsync(cancellationToken))); + } + finally + { + await _broadcastQueue.StopAsync(cancellationToken); + } } cancellationToken.ThrowIfCancellationRequested(); @@ -1384,6 +1430,12 @@ private void PrunePeerNamespaceConfirmations(DisseminationMembershipSnapshot mem { lock (_peerSupportLock) { + if (ReferenceEquals(_confirmedMembership, membership)) + { + return; + } + + membership = _membership.CurrentSnapshots.AllMembers; foreach (var peer in _confirmedPeerNamespaces.Keys) { if (!membership.ContainsMember(peer)) @@ -1391,6 +1443,8 @@ private void PrunePeerNamespaceConfirmations(DisseminationMembershipSnapshot mem _confirmedPeerNamespaces.Remove(peer); } } + + _confirmedMembership = membership; } } @@ -1414,11 +1468,123 @@ private void RecordValueUpdate(DisseminationNamespace namespaceName, Disseminati CancellationToken cancellationToken) { // Prune peer ledgers against the same membership view used to choose tree targets. - var memberships = await _membership.GetSnapshotsContainingMember(member, scope, cancellationToken); - await _broadcastQueue.Prune(memberships ?? _membership.CurrentSnapshots, cancellationToken); - return memberships?.GetSnapshot(scope); + await _membership.GetSnapshotsContainingMember(member, scope, cancellationToken); + await MaintainInventories(cancellationToken); + var snapshot = _membership.GetSnapshot(scope); + return snapshot.ContainsMember(member) ? snapshot : null; } + private bool NotifyRoot( + IDisseminationNamespace disseminationNamespace, + ReadOnlySpan notifications) + { + DisseminationRootBatcher root; + lock (_rootBatcherLock) + { + if (_rootBatchersStopped) + { + return false; + } + + if (notifications.IsEmpty) + { + return true; + } + + if (!_rootBatchers.TryGetValue(disseminationNamespace.Name, out root!)) + { + root = new( + _timeProvider, disseminationNamespace, _options, + values => DispatchRootBatch(disseminationNamespace, values), _logger); + _rootBatchers.Add(disseminationNamespace.Name, root); + } + } + + return root.Notify(notifications); + } + + private bool DispatchRootBatch( + IDisseminationNamespace disseminationNamespace, + ReadOnlySpan notifications) + { + var membership = _membership.GetSnapshot(disseminationNamespace.MembershipScope); + if (!membership.IsAggregationRoot) + { + // A departing root hands accepted state to the new root, even after leaving Active membership. + return membership.Members.IsEmpty + || _broadcastQueue.NotifyBatch(membership.Members[0], disseminationNamespace, notifications, immediate: true); + } + + var accepted = true; + foreach (var child in membership.AggregationChildren) + { + // Include a child producer's own value so it reaches that child's descendants. + accepted &= _broadcastQueue.NotifyBatch(child, disseminationNamespace, notifications, immediate: true); + } + + return accepted; + } + + private async ValueTask MaintainInventories(CancellationToken cancellationToken) + { + MaintenanceStamp reservation; + bool membershipChanged; + lock (_maintenanceLock) + { + var membership = _membership.CurrentSnapshots; + var now = _timeProvider.GetTimestamp(); + membershipChanged = !ReferenceEquals(_lastMaintenance?.Membership, membership); + if (!membershipChanged + && _lastMaintenance is { } previous + && _timeProvider.GetElapsedTime(previous.Timestamp, now) < InventoryMaintenanceInterval) + { + return; + } + + // Reserve the pass before enumerating namespace state, keeping callbacks outside this lock. + reservation = new(membership, now); + _lastMaintenance = reservation; + } + + try + { + cancellationToken.ThrowIfCancellationRequested(); + PrunePeerNamespaceConfirmations(reservation.Membership.AllMembers); + await _broadcastQueue.Prune(reservation.Membership, cancellationToken); + KeyValuePair[] roots; + lock (_rootBatcherLock) + { + roots = [.. _rootBatchers]; + } + + foreach (var (name, root) in roots) + { + cancellationToken.ThrowIfCancellationRequested(); + var ns = _namespaces[name]; + var activeKeys = ns.Options.Enabled ? ns.Keys.ToHashSet() : new HashSet(); + root.Prune(activeKeys); + if (membershipChanged && !reservation.Membership.GetSnapshot(ns.MembershipScope).IsAggregationRoot) + { + root.WakeForMembershipChange(); + } + } + } + catch + { + lock (_maintenanceLock) + { + if (ReferenceEquals(_lastMaintenance, reservation)) + { + _lastMaintenance = null; + } + } + + throw; + } + } + + private sealed record MaintenanceStamp(DisseminationMembershipSnapshots Membership, long Timestamp); + private bool TryGetEnabledNamespace(DisseminationNamespace namespaceName, [NotNullWhen(true)] out IDisseminationNamespace? disseminationNamespace) { if (_namespaces.TryGetValue(namespaceName, out disseminationNamespace) @@ -1434,10 +1600,10 @@ private bool TryGetEnabledNamespace(DisseminationNamespace namespaceName, [NotNu private bool ValidatePayloadSize( IDisseminationNamespace disseminationNamespace, DisseminationValue value, - DisseminationOptions options) + int maxBatchBytes) { if (value.Payload.Length <= disseminationNamespace.Options.MaxPayloadBytes - && value.Payload.Length <= options.MaxBatchBytes) + && value.Payload.Length <= maxBatchBytes) { return true; } @@ -1541,7 +1707,7 @@ private bool ValidateRepair( if (value.Key != request.Key || !IsValidVersionRange(value) || value.ToVersion > repair.Version - || !ValidatePayloadSize(disseminationNamespace, value, options)) + || !ValidatePayloadSize(disseminationNamespace, value, options.MaxBatchBytes)) { return false; } diff --git a/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs b/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs new file mode 100644 index 00000000000..c1a140b0e9d --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs @@ -0,0 +1,630 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime.Internal; +using KeyNotification = Orleans.Runtime.Dissemination.DisseminationBroadcastQueue.KeyNotification; + +namespace Orleans.Runtime.Dissemination; + +// This budget counts logical root waves, not wire messages: each peer queue still owns payload +// materialization, byte limits, splitting, and RPC lifetime. Only bounded key identities live here. +internal sealed partial class DisseminationRootBatcher +{ + internal delegate bool DispatchCallback(ReadOnlySpan notifications); + + private readonly object _lock = new(); + private readonly TimeProvider _timeProvider; + private readonly IDisseminationNamespace _namespace; + private readonly DisseminationNamespace _namespaceName; + private readonly IOptionsMonitor _options; + private readonly DispatchCallback _dispatch; + private readonly ILogger _logger; + private readonly WakeTimer _timer; + private readonly Task _worker; + private readonly Dictionary _pending = []; + private readonly LinkedList _ready = []; + private TaskCompletionSource? _drainCompletion; + private Exception? _failure; + private long? _firstPendingTimestamp; + private long? _lastDispatchTimestamp; + private long _generation; + private bool _dispatching; + private bool _workerActive; + private bool _retrying; + private bool _forceHandoff; + private bool _stopping; + private bool _aborted; + private CancellationToken _abortToken; + + public DisseminationRootBatcher( + TimeProvider timeProvider, + IDisseminationNamespace disseminationNamespace, + IOptionsMonitor options, + DispatchCallback dispatch, + ILogger logger) + { + _timeProvider = timeProvider; + _namespace = disseminationNamespace; + _namespaceName = disseminationNamespace.Name; + _options = options; + _dispatch = dispatch; + _logger = logger; + _timer = new(timeProvider); + using var _ = new ExecutionContextSuppressor(); + _worker = RunAsync(); + } + + // False means at least one distinct key was not admitted. Updates to retained keys still succeed + // at the limit, including keys currently inside the synchronous dispatch callback. + public bool Notify(ReadOnlySpan notifications) + { + var settings = GetSettings(); + var accepted = true; + var rejected = 0; + var wake = false; + lock (_lock) + { + ThrowIfFailedUnsafe(); + if (_stopping) + { + return false; + } + + if (!settings.Enabled) + { + // Disabling explicitly abandons queued hints; re-enabling does not replay them. + // Namespace state and anti-entropy, rather than this queue, own durable convergence. + ClearPendingUnsafe(); + accepted = false; + wake = true; + } + else + { + var wasEmpty = _pending.Count == 0; + var now = _timeProvider.GetTimestamp(); + foreach (var notification in notifications) + { + if (_pending.TryGetValue(notification.Key, out var pending)) + { + // Even an unchanged notification can race an inventory/version read. Protect + // it from pruning without making duplicate in-flight values dirty again. + pending.AdmissionGeneration = ++_generation; + if (!notification.Force && notification.Version <= pending.Notification.Version) + { + continue; + } + + pending.Notification = new( + notification.Key, + Math.Max(notification.Version, pending.Notification.Version), + pending.Notification.Force || notification.Force); + pending.Generation = pending.AdmissionGeneration; + pending.PendingSince ??= now; + } + else + { + if (_pending.Count >= settings.MaxPendingItemCount) + { + accepted = false; + rejected++; + continue; + } + + pending = new(notification, ++_generation, now); + _pending.Add(notification.Key, pending); + _ready.AddLast(pending.Node); + } + + _firstPendingTimestamp ??= now; + // Replacements cannot advance the earliest pending timestamp or its deadline. + // Avoid waking/rearming once per key while a root is collecting thousands of keys. + wake |= wasEmpty || settings.HighPriority; + } + } + } + + if (wake) + { + Wake(); + } + + if (rejected > 0) + { + try + { + LogAdmissionRejected(_logger, _namespaceName, rejected, settings.MaxPendingItemCount); + } + catch (Exception exception) + { + Fail(exception); + DisposeTimer(); + } + + for (var index = 0; index < rejected; index++) + { + try + { + DisseminationInstruments.OnQueueAdmissionRejected(_namespaceName); + } + catch (Exception exception) + { + try + { + LogDiagnosticFailed(_logger, exception, _namespaceName); + } + catch (Exception loggingFailure) + { + Fail(new AggregateException(exception, loggingFailure)); + DisposeTimer(); + } + } + } + } + + return accepted; + } + + // Flush forces all currently retained work into peer queues, not onto the wire. Concurrent + // admissions join this drain. A partial admission never turns a forced drain into a busy loop. + public async Task FlushAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Task completion; + lock (_lock) + { + ThrowIfFailedUnsafe(); + ThrowIfAbortedUnsafe(); + if (_pending.Count == 0 && !_dispatching) + { + return; + } + + _drainCompletion ??= new(TaskCreationOptions.RunContinuationsAsynchronously); + completion = _drainCompletion.Task; + } + + Wake(); + await completion.WaitAsync(cancellationToken).ConfigureAwait(false); + lock (_lock) + { + ThrowIfFailedUnsafe(); + ThrowIfAbortedUnsafe(); + } + } + + // Cancellation ends this owner's drain budget and abandons remaining hints. A wave already + // claimed for synchronous dispatch cannot be interrupted; subsequent waves will not be claimed. + public async Task StopAsync(CancellationToken cancellationToken) + { + lock (_lock) + { + _stopping = true; + } + + try + { + cancellationToken.ThrowIfCancellationRequested(); + Wake(); + await _worker.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + lock (_lock) + { + _aborted = true; + _abortToken = cancellationToken; + ClearPendingUnsafe(); + } + + DisposeTimer(); + throw; + } + + lock (_lock) + { + ThrowIfFailedUnsafe(); + ThrowIfAbortedUnsafe(); + } + } + + public void Prune(IReadOnlySet activeKeys) + { + List<(DisseminationKey Key, PendingKey Pending, long Generation)> candidates = []; + lock (_lock) + { + foreach (var (key, pending) in _pending) + { + if (!activeKeys.Contains(key)) + { + candidates.Add((key, pending, pending.AdmissionGeneration)); + } + } + } + + var retiredCount = 0; + for (var i = 0; i < candidates.Count; i++) + { + var candidate = candidates[i]; + // Owner inventory was captured before entering this method. A positive current version + // means the key has since become live; namespace calls must not hold the batcher lock. + if (_namespace.GetVersion(candidate.Key) <= 0) + { + candidates[retiredCount++] = candidate; + } + } + + lock (_lock) + { + for (var i = 0; i < retiredCount; i++) + { + var candidate = candidates[i]; + if (_pending.TryGetValue(candidate.Key, out var pending) + && ReferenceEquals(pending, candidate.Pending) + && pending.AdmissionGeneration == candidate.Generation) + { + _pending.Remove(candidate.Key); + if (pending.Node.List is not null) + { + _ready.Remove(pending.Node); + } + } + } + + RecomputeFirstPendingUnsafe(); + } + + Wake(); + } + + // Called when the local silo loses the root role, not for ordinary membership changes. Request + // Flush-equivalent scheduling without a drain observer; the dispatcher resolves the new root. + // This does not store credits for later work or bypass partial-retry pacing. + public void WakeForMembershipChange() + { + lock (_lock) + { + _forceHandoff |= _pending.Count > 0; + } + + Wake(); + } + + private async Task RunAsync() + { + try + { + while (await _timer.WaitAsync(CancellationToken.None).ConfigureAwait(false)) + { + while (true) + { + var settings = GetSettings(); + lock (_lock) + { + _workerActive = true; + if (_failure is not null || _aborted) + { + return; + } + + if (!settings.Enabled) + { + ClearPendingUnsafe(); + } + + if (_pending.Count == 0) + { + CompleteDrainUnsafe(); + if (_stopping) + { + return; + } + + _workerActive = false; + break; + } + + var delay = GetDelayUnsafe(settings); + if (delay > TimeSpan.Zero) + { + _timer.Change(delay); + _workerActive = false; + break; + } + } + + Dispatch(); + } + } + } + catch (Exception exception) + { + Fail(exception); + } + finally + { + DisposeTimer(); + lock (_lock) + { + ClearPendingUnsafe(); + CompleteDrainUnsafe(); + } + } + } + + private TimeSpan GetDelayUnsafe(in BatchingSettings settings) + { + var pacedDelay = _lastDispatchTimestamp is { } last + ? settings.BroadcastInterval - _timeProvider.GetElapsedTime(last) + : TimeSpan.Zero; + if (_stopping || _drainCompletion is not null || _forceHandoff || settings.HighPriority) + { + return _retrying ? pacedDelay : TimeSpan.Zero; + } + + var collectionDelay = settings.CollectionDelay + - _timeProvider.GetElapsedTime(_firstPendingTimestamp!.Value); + return collectionDelay > pacedDelay ? collectionDelay : pacedDelay; + } + + // Keeping snapshots in a synchronous method prevents the worker state machine from retaining + // dispatched arrays across timer waits. Newer notifications never mutate a captured snapshot. + private void Dispatch() + { + var settings = GetSettings(); + PendingDispatch[] work; + KeyNotification[] notifications; + lock (_lock) + { + if (_failure is not null || _aborted || _ready.Count == 0 + || !settings.Enabled || GetDelayUnsafe(settings) > TimeSpan.Zero) + { + return; + } + + var count = Math.Min(_ready.Count, Math.Max(1, settings.MaxBatchItems)); + work = new PendingDispatch[count]; + notifications = new KeyNotification[count]; + for (var i = 0; i < count; i++) + { + var pending = _ready.First!.Value; + _ready.RemoveFirst(); + notifications[i] = pending.Notification; + work[i] = new(pending, pending.Generation, pending.PendingSince!.Value, pending.Notification); + // A successful attempt consumes Force; a failed admission must preserve it for peers + // which still hold an earlier same-version state. + pending.Notification = pending.Notification with { Force = false }; + pending.PendingSince = null; + } + + _dispatching = true; + _lastDispatchTimestamp = _timeProvider.GetTimestamp(); + } + + var accepted = false; + Exception? dispatchFailure = null; + try + { + accepted = _dispatch(notifications); + } + catch (Exception exception) + { + dispatchFailure = exception; + } + finally + { + lock (_lock) + { + foreach (var item in work) + { + var pending = item.Pending; + if (!_pending.TryGetValue(pending.Notification.Key, out var current) + || !ReferenceEquals(current, pending)) + { + // Pruning can retire an in-flight identity and admit a new one with the same key. + continue; + } + + if (accepted && pending.Generation == item.Generation) + { + _pending.Remove(pending.Notification.Key); + } + else + { + if (!accepted) + { + pending.PendingSince = item.PendingSince; + if (pending.Notification.Version == item.Notification.Version) + { + pending.Notification = pending.Notification with + { + Force = pending.Notification.Force || item.Notification.Force, + }; + } + } + + _ready.AddLast(pending.Node); + } + } + + _retrying = !accepted; + _dispatching = false; + RecomputeFirstPendingUnsafe(); + if (_pending.Count == 0) + { + CompleteDrainUnsafe(); + } + } + } + + if (dispatchFailure is not null) + { + try + { + LogDispatchFailed(_logger, dispatchFailure, _namespaceName); + } + catch (Exception loggingFailure) + { + Fail(new AggregateException("The root dispatch recovery diagnostic failed.", dispatchFailure, loggingFailure)); + } + } + } + + private BatchingSettings GetSettings() + { + var options = _options.CurrentValue; + var namespaceOptions = _namespace.Options; + return new( + options.Enabled && namespaceOptions.Enabled, + namespaceOptions.Priority == DisseminationPriority.High, + namespaceOptions.MaxPendingItemCount, + options.MaxBatchItems, + namespaceOptions.MaxCoalescingDelay, + TimeSpan.FromSeconds(1d / Math.Clamp(options.Overlay.AggregationBroadcastsPerSecond, 1, 1000))); + } + + private void RecomputeFirstPendingUnsafe() + { + _firstPendingTimestamp = null; + foreach (var pending in _ready) + { + var timestamp = pending.PendingSince!.Value; + if (_firstPendingTimestamp is null || timestamp < _firstPendingTimestamp.Value) + { + _firstPendingTimestamp = timestamp; + } + } + } + + private void ClearPendingUnsafe() + { + _pending.Clear(); + _ready.Clear(); + _firstPendingTimestamp = null; + _retrying = false; + _forceHandoff = false; + } + + private void CompleteDrainUnsafe() + { + _retrying = false; + _forceHandoff = false; + _drainCompletion?.TrySetResult(); + _drainCompletion = null; + } + + private void ThrowIfFailedUnsafe() + { + if (_failure is { } failure) + { + throw new InvalidOperationException($"The dissemination root batcher for {_namespaceName} has failed.", failure); + } + } + + private void ThrowIfAbortedUnsafe() + { + if (_aborted) + { + throw new OperationCanceledException("The dissemination root drain was canceled.", _abortToken); + } + } + + private void Wake() + { + lock (_lock) + { + if (_workerActive || _failure is not null || _aborted) + { + return; + } + + _workerActive = true; + } + + try + { + _timer.Wake(); + } + catch (Exception exception) + { + Fail(exception); + DisposeTimer(); + } + } + + private void DisposeTimer() + { + try + { + _timer.Dispose(); + } + catch (Exception exception) + { + Fail(exception); + } + } + + private void Fail(Exception exception) + { + lock (_lock) + { + if (_failure is not null) + { + return; + } + + _failure = exception; + // Complete normally and read _failure after awaiting: no abandoned shared task faults. + _drainCompletion?.TrySetResult(); + } + + try + { + LogWorkerFailed(_logger, exception, _namespaceName); + } + catch + { + // A failing logger must not hide the original failure from admission/drain callers. + } + } + + private sealed class PendingKey + { + public KeyNotification Notification; + public long Generation; + public long AdmissionGeneration; + public long? PendingSince; + public LinkedListNode Node { get; } + + public PendingKey(KeyNotification notification, long generation, long pendingSince) + { + Notification = notification; + Generation = generation; + AdmissionGeneration = generation; + PendingSince = pendingSince; + Node = new(this); + } + } + + private readonly record struct PendingDispatch( + PendingKey Pending, long Generation, long PendingSince, KeyNotification Notification); + + private readonly record struct BatchingSettings( + bool Enabled, + bool HighPriority, + int MaxPendingItemCount, + int MaxBatchItems, + TimeSpan CollectionDelay, + TimeSpan BroadcastInterval); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Dissemination root dispatch for {Namespace} failed and will retry at the root pacing cadence.")] + private static partial void LogDispatchFailed(ILogger logger, Exception exception, DisseminationNamespace @namespace); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Dissemination root for {Namespace} rejected {Count} new keys at its pending-key limit of {Limit}.")] + private static partial void LogAdmissionRejected(ILogger logger, DisseminationNamespace @namespace, int count, int limit); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Dissemination root diagnostic for {Namespace} failed.")] + private static partial void LogDiagnosticFailed(ILogger logger, Exception exception, DisseminationNamespace @namespace); + + [LoggerMessage(Level = LogLevel.Error, Message = "Dissemination root batcher for {Namespace} failed. Admission and drain callers will observe this failure.")] + private static partial void LogWorkerFailed(ILogger logger, Exception exception, DisseminationNamespace @namespace); +} diff --git a/src/api/Orleans.Core/Orleans.Core.cs b/src/api/Orleans.Core/Orleans.Core.cs index ebfc08f7886..cb6702fe16b 100644 --- a/src/api/Orleans.Core/Orleans.Core.cs +++ b/src/api/Orleans.Core/Orleans.Core.cs @@ -566,12 +566,20 @@ public sealed partial class DisseminationOptions public sealed partial class DisseminationOverlayOptions { + public int AggregationBroadcastsPerSecond { get { throw null; } set { } } + + public int AggregationFanOutFactor { get { throw null; } set { } } + public System.TimeSpan AntiEntropyInterval { get { throw null; } set { } } public int AntiEntropyPeerCount { get { throw null; } set { } } public System.Func? FanOutFactor { get { throw null; } set { } } + public int MaxAntiEntropyBatchBytes { get { throw null; } set { } } + + public int MaxAntiEntropyBatchItems { get { throw null; } set { } } + public int MaxFanOutFactor { get { throw null; } set { } } public int MinFanOutFactor { get { throw null; } set { } } diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs new file mode 100644 index 00000000000..ad4c770db63 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs @@ -0,0 +1,278 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Net; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +[TestCategory("BVT"), TestCategory("Dissemination")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Dissemination")] +public class DisseminationAggregationOptionsTests +{ + [Fact] + public void DefaultsAreBoundedAndOptIn() + { + var options = new DisseminationOptions(); + var load = new DeploymentLoadPublisherOptions(); + var membership = new ClusterMembershipOptions(); + + Assert.Equal(8, options.Overlay.AggregationFanOutFactor); + Assert.Equal(5, options.Overlay.AggregationBroadcastsPerSecond); + Assert.Equal(8192, options.Overlay.MaxAntiEntropyBatchItems); + Assert.Equal(1048576, options.Overlay.MaxAntiEntropyBatchBytes); + Assert.Equal(options.MaxBatchItems, options.Overlay.MaxAntiEntropyBatchItems); + Assert.Equal(options.MaxBatchBytes, options.Overlay.MaxAntiEntropyBatchBytes); + Assert.Equal(TimeSpan.FromMilliseconds(25), load.Dissemination.MaxCoalescingDelay); + Assert.Equal(8192, load.Dissemination.MaxPendingItemCount); + Assert.Equal(TimeSpan.FromSeconds(5), load.Dissemination.ExpectedUpdateCadence); + Assert.Equal(TimeSpan.FromSeconds(1), load.DeploymentLoadPublisherRefreshTime); + Assert.False(options.Enabled); + Assert.False(new DisseminationNamespaceOptions().Enabled); + Assert.False(load.Dissemination.Enabled); + Assert.False(membership.Dissemination.Enabled); + Assert.Equal(DisseminationPriority.High, membership.Dissemination.Priority); + Assert.Equal(ValidateOptionsResult.Success, Validate(options)); + Assert.Equal(ValidateOptionsResult.Success, new DeploymentLoadPublisherOptionsValidator().Validate(Options.DefaultName, load)); + } + + [Theory] + [InlineData(1, 1, 1, 1)] + [InlineData(int.MaxValue, 1000, int.MaxValue, int.MaxValue)] + public void ValidatorAcceptsPositiveBounds(int fanout, int broadcastsPerSecond, int items, int bytes) + { + var options = new DisseminationOptions(); + options.Overlay.AggregationFanOutFactor = fanout; + options.Overlay.AggregationBroadcastsPerSecond = broadcastsPerSecond; + options.Overlay.MaxAntiEntropyBatchItems = items; + options.Overlay.MaxAntiEntropyBatchBytes = bytes; + + Assert.Equal(ValidateOptionsResult.Success, Validate(options)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(int.MinValue)] + public void ValidatorRejectsNonPositiveAggregationFanOut(int value) + { + var options = new DisseminationOptions(); + options.Overlay.AggregationFanOutFactor = value; + + AssertValidationFailure(options, "AggregationFanOutFactor must be greater than 0."); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(int.MinValue)] + [InlineData(1001)] + [InlineData(int.MaxValue)] + public void ValidatorRejectsOutOfRangeAggregationBroadcastRate(int value) + { + var options = new DisseminationOptions(); + options.Overlay.AggregationBroadcastsPerSecond = value; + + AssertValidationFailure(options, "AggregationBroadcastsPerSecond must be between 1 and 1000."); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(int.MinValue)] + public void ValidatorRejectsNonPositiveAntiEntropyItemLimit(int value) + { + var options = new DisseminationOptions(); + options.Overlay.MaxAntiEntropyBatchItems = value; + + AssertValidationFailure(options, "MaxAntiEntropyBatchItems must be greater than 0."); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(int.MinValue)] + public void ValidatorRejectsNonPositiveAntiEntropyByteLimit(int value) + { + var options = new DisseminationOptions(); + options.Overlay.MaxAntiEntropyBatchBytes = value; + + AssertValidationFailure(options, "MaxAntiEntropyBatchBytes must be greater than 0."); + } + + [Fact] + public void AntiEntropyLimitsAreIndependentOfBroadcastLimits() + { + var options = new DisseminationOptions { MaxBatchItems = 1, MaxBatchBytes = 1 }; + + Assert.Equal(8192, options.Overlay.MaxAntiEntropyBatchItems); + Assert.Equal(1048576, options.Overlay.MaxAntiEntropyBatchBytes); + Assert.Equal(ValidateOptionsResult.Success, Validate(options)); + + options.MaxBatchItems = 16384; + options.MaxBatchBytes = 2 * 1024 * 1024; + options.Overlay.MaxAntiEntropyBatchItems = 2; + options.Overlay.MaxAntiEntropyBatchBytes = 3; + + Assert.Equal(16384, options.MaxBatchItems); + Assert.Equal(2 * 1024 * 1024, options.MaxBatchBytes); + Assert.Equal(2, options.Overlay.MaxAntiEntropyBatchItems); + Assert.Equal(3, options.Overlay.MaxAntiEntropyBatchBytes); + Assert.Equal(ValidateOptionsResult.Success, Validate(options)); + } + + [Theory] + [InlineData(2000)] + [InlineData(2048)] + public void AggregationTopologyHasEightWayBranchesAndDepthFourAtScale(int memberCount) + { + var members = CreateSilos(memberCount); + var overlay = new DisseminationOverlayOptions(); + var children = new Dictionary>(memberCount); + var roots = new List(); + + Assert.Equal(32, overlay.GetFanOutFactor(memberCount)); + for (var index = 0; index < members.Length; index++) + { + var silo = members[index]; + var snapshot = new DisseminationMembershipSnapshot(new MembershipVersion(1), silo, members, overlay); + children.Add(silo, snapshot.AggregationChildren); + + Assert.InRange(snapshot.AggregationChildren.Length, 0, 8); + Assert.DoesNotContain(silo, snapshot.AggregationChildren); + Assert.Equal(snapshot.AggregationChildren.Length, snapshot.AggregationChildren.Distinct().Count()); + Assert.Equal(members.Skip(32 * (index + 1)).Take(32), snapshot.ForwardingTreeTargets); + Assert.Equal(snapshot.AggregationChildren, snapshot.GetForwardingTargets(DisseminationRoutingMode.AggregationTree)); + if (snapshot.IsAggregationRoot) + { + roots.Add(silo); + Assert.Equal(8, snapshot.AggregationChildren.Length); + Assert.Equal(members.Skip(1).Take(8), snapshot.AggregationChildren); + Assert.Equal(snapshot.AggregationChildren, snapshot.GetOriginatorTargets(DisseminationRoutingMode.AggregationTree)); + } + else + { + Assert.Equal(members[0], Assert.Single(snapshot.GetOriginatorTargets(DisseminationRoutingMode.AggregationTree))); + } + } + + Assert.Equal(members[0], Assert.Single(roots)); + Assert.Equal(memberCount - 1, children.Values.Sum(targets => targets.Length)); + var reached = new HashSet { members[0] }; + var pending = new Queue<(SiloAddress Silo, int Depth)>(); + pending.Enqueue((members[0], 0)); + var maxDepth = 0; + while (pending.TryDequeue(out var current)) + { + maxDepth = Math.Max(maxDepth, current.Depth); + foreach (var child in children[current.Silo]) + { + Assert.True(reached.Add(child), $"Member {child} was reached more than once."); + pending.Enqueue((child, current.Depth + 1)); + } + } + + Assert.Equal(members, reached.Order()); + Assert.Equal(4, maxDepth); + } + + [Theory] + [InlineData(int.MinValue)] + [InlineData(1)] + [InlineData(2)] + [InlineData(32)] + [InlineData(int.MaxValue)] + public void MembershipFanOutSelectorDoesNotChangeAggregationTopology(int selectedFanout) + { + var members = CreateSilos(65); + var calls = 0; + var overlay = new DisseminationOverlayOptions + { + FanOutFactor = count => + { + Assert.Equal(members.Length, count); + calls++; + return selectedFanout; + }, + }; + var membershipFanout = Math.Clamp(selectedFanout, 1, members.Length); + for (var index = 0; index < members.Length; index++) + { + var snapshot = new DisseminationMembershipSnapshot(new MembershipVersion(1), members[index], members, overlay); + + Assert.Equal(members.Skip(index * 8 + 1).Take(8), snapshot.AggregationChildren); + Assert.Equal(members.Skip(membershipFanout * (index + 1)).Take(membershipFanout), snapshot.ForwardingTreeTargets); + var originatorTargets = members.Take(membershipFanout).Where(silo => !silo.Equals(members[index])) + .Concat(snapshot.ForwardingTreeTargets); + Assert.Equal(originatorTargets, snapshot.OriginatorTreeTargets); + Assert.Equal(snapshot.OriginatorTreeTargets.Length, snapshot.OriginatorTreeTargets.Distinct().Count()); + } + + Assert.Equal(members.Length, calls); + } + + [Theory] + [InlineData(1, int.MaxValue)] + [InlineData(2, 8)] + [InlineData(7, int.MaxValue)] + [InlineData(17, 1)] + [InlineData(17, 2)] + public void AggregationTopologyClampsConfiguredFanOutToMembership(int memberCount, int fanout) + { + var members = CreateSilos(memberCount); + var overlay = new DisseminationOverlayOptions { AggregationFanOutFactor = fanout }; + var effectiveFanout = Math.Min(memberCount, fanout); + for (var index = 0; index < members.Length; index++) + { + var snapshot = new DisseminationMembershipSnapshot(new MembershipVersion(1), members[index], members, overlay); + + Assert.Equal(members.Skip(index * effectiveFanout + 1).Take(effectiveFanout), snapshot.AggregationChildren); + Assert.DoesNotContain(members[index], snapshot.AggregationChildren); + Assert.Equal(snapshot.AggregationChildren.Length, snapshot.AggregationChildren.Distinct().Count()); + } + } + + [Theory] + [InlineData(0)] + [InlineData(8)] + public void NonMembersHaveNoAggregationNeighbors(int memberCount) + { + var local = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 30000), 1); + var snapshot = new DisseminationMembershipSnapshot( + new MembershipVersion(1), + local, + CreateSilos(memberCount), + new DisseminationOverlayOptions { AggregationFanOutFactor = int.MaxValue }); + + Assert.False(snapshot.IsAggregationRoot); + Assert.Empty(snapshot.AggregationChildren); + Assert.Empty(snapshot.GetOriginatorTargets(DisseminationRoutingMode.AggregationTree)); + Assert.Empty(snapshot.GetForwardingTargets(DisseminationRoutingMode.AggregationTree)); + } + + private static ValidateOptionsResult Validate(DisseminationOptions options) => + new DisseminationOptionsValidator().Validate(Options.DefaultName, options); + + private static void AssertValidationFailure(DisseminationOptions options, string message) + { + var result = Validate(options); + Assert.True(result.Failed); + Assert.NotNull(result.Failures); + Assert.Equal(message, Assert.Single(result.Failures)); + } + + private static ImmutableArray CreateSilos(int count) => + Enumerable.Range(11111, count) + .Select(port => SiloAddress.New(new IPEndPoint(IPAddress.Loopback, port), port)) + .Order() + .ToImmutableArray(); +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs index ff5cfa80c87..c509ed66e07 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs @@ -24,9 +24,10 @@ public partial class DisseminationProtocolTests public void AggregationTopologyHasOneRootAndLinearDistributionEdges(int count, int fanout) { var members = CreateSilos(count).ToImmutableArray(); + var overlay = new DisseminationOverlayOptions { AggregationFanOutFactor = fanout }; var snapshots = members.ToDictionary( silo => silo, - silo => new DisseminationMembershipSnapshot(new MembershipVersion(1), silo, members, CreateOverlayOptions(fanout))); + silo => new DisseminationMembershipSnapshot(new MembershipVersion(1), silo, members, overlay)); Assert.Equal(count - 1, snapshots.Values.Sum(snapshot => snapshot.AggregationChildren.Length)); Assert.Equal(members.Skip(1), snapshots.Values.SelectMany(snapshot => snapshot.AggregationChildren).Order()); foreach (var (silo, snapshot) in snapshots) @@ -85,7 +86,7 @@ public async Task AggregationTreeDeliversWholeRoundsWithLinearBatchCount(int cou var transport = new FakeTransport(member, members.Where(peer => !peer.Equals(member)).ToArray()); var protocol = CreateProtocol(transport, ns, options => { - options.Overlay.FanOutFactor = _ => fanout; + options.Overlay.AggregationFanOutFactor = fanout; options.MaxConcurrentSends = 1; }, clock); nodes.Add(member, (ns, transport, protocol)); @@ -171,7 +172,7 @@ public async Task RootAggregationBatchesOnceAndCrossesMultipleLevelsWithoutAnoth var transport = new FakeTransport(member, members.Where(peer => !peer.Equals(member)).ToArray()); var protocol = CreateProtocol(transport, ns, options => { - options.Overlay.FanOutFactor = static _ => 2; + options.Overlay.AggregationFanOutFactor = 2; options.MaxConcurrentSends = 1; }, clock); nodes.Add(member, (ns, transport, protocol)); @@ -280,7 +281,7 @@ public async Task FormerAggregationRootRedirectsIngressToCurrentRootImmediately( forwarded.TrySetResult(); return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); }; - var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 2, clock); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.AggregationFanOutFactor = 2, clock); var start = clock.GetTimestamp(); try { @@ -333,7 +334,7 @@ public async Task AggregationRelayForwardsWholeBatchAtomicallyWithoutCoalescing( } return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); }; - var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 2, clock); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.AggregationFanOutFactor = 2, clock); var values = Enumerable.Range(0, 64).Select(index => ns.CreateItem(members[0], $"key-{index}", 1)).ToArray(); var start = clock.GetTimestamp(); try @@ -373,7 +374,7 @@ public void RootAggregationRequiresAddressOrderedActiveMembership() } [Fact] - public async Task AggregationWindowBatchesStaggeredKeysAndPreservesNextWindow() + public async Task QueueCoalescingWindowBatchesStaggeredKeysAndPreservesNextWindow() { var cancellationToken = TestContext.Current.CancellationToken; var local = CreateSilo(40801); @@ -384,7 +385,7 @@ public async Task AggregationWindowBatchesStaggeredKeysAndPreservesNextWindow() RoutingMode = DisseminationRoutingMode.AggregationTree, MembershipScope = DisseminationMembershipScope.ActiveMembers, }; - ns.Options.MaxCoalescingDelay = new DeploymentLoadPublisherOptions().Dissemination.MaxCoalescingDelay; + ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(500); var transport = new FakeTransport(local, peer); var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var secondStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Maintenance.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Maintenance.cs new file mode 100644 index 00000000000..062479c1005 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Maintenance.cs @@ -0,0 +1,178 @@ +using Microsoft.Extensions.Time.Testing; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task ReceiveTrafficSharesInventoryMaintenanceUntilTimeOrMembershipChanges() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41201); + var peer = CreateSilo(41202); + var transport = new FakeTransport(local, peer); + var clock = new FakeTimeProvider(); + var ns = new CompactReviewInventoryNamespace(local); + var inventories = 0; + DisseminationKey[] keys = ["value"]; + ns.ReadKeys = () => + { + inventories++; + return keys; + }; + var protocol = CreateProtocol(transport, [ns], timeProvider: clock); + try + { + for (var version = 1; version <= 2000; version++) + { + await Receive(version); + } + Assert.Equal(1, inventories); + Assert.Equal(2000, ns.GetVersion("value")); + + clock.Advance(TimeSpan.FromMilliseconds(999)); + await Receive(2001); + Assert.Equal(1, inventories); + clock.Advance(TimeSpan.FromMilliseconds(1)); + await Receive(2002); + Assert.Equal(2, inventories); + + transport.Peers.Add(CreateSilo(41203)); + await Receive(2003); + Assert.Equal(3, inventories); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + + Task Receive(long version) => protocol.ReceiveBroadcast(new() + { + Sender = peer, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(peer, "value", version)), + }, cancellationToken); + } + + [Fact] + public async Task FailedInventoryMaintenanceIsRetriedWithoutWaitingForTheInterval() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41211); + var peer = CreateSilo(41212); + var ns = new CompactReviewInventoryNamespace(local); + var inventories = 0; + ns.ReadKeys = () => ++inventories == 1 + ? throw new InvalidOperationException("Inventory failed.") + : new DisseminationKey[] { "value" }; + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns], timeProvider: new FakeTimeProvider()); + var batch = new DisseminationBroadcastBatch + { + Sender = peer, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(peer, "value", 1)), + }; + try + { + await Assert.ThrowsAsync(() => protocol.ReceiveBroadcast(batch, cancellationToken)); + var response = await protocol.ReceiveBroadcast(batch, cancellationToken); + Assert.Equal(2, inventories); + Assert.Equal(1, Assert.Single(response.Acknowledgments[ns.Name]).Version); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task InventoryMaintenanceAllowsSynchronousPublicationReentry() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41221); + var peer = CreateSilo(41222); + var ns = new CompactReviewInventoryNamespace(local); + ns.Inner.SetValue("nested", 1); + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns], timeProvider: new FakeTimeProvider()); + Task? nested = null; + ns.ReadKeys = () => + { + nested = protocol.Publish(ns, "nested", 1, cancellationToken).AsTask(); + Assert.True(nested.IsCompletedSuccessfully); + return new DisseminationKey[] { "nested", "value" }; + }; + try + { + await protocol.ReceiveBroadcast(new() + { + Sender = peer, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(peer, "value", 1)), + }, cancellationToken); + + Assert.NotNull(nested); + Assert.True(await nested); + Assert.Equal(1, ns.GetVersion("value")); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + + [Theory] + [InlineData(1, 64, 1)] + [InlineData(8, 16, 2)] + public async Task AntiEntropyUsesIndependentResponseAndReceiveBudgets(int repairItems, int repairBytes, int expectedItems) + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41231); + var peer = CreateSilo(41232); + var ns = new FakeNamespace(local); + var transport = new FakeTransport(local, peer); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxBatchItems = 8; + options.MaxBatchBytes = 64; + options.Overlay.MaxAntiEntropyBatchItems = repairItems; + options.Overlay.MaxAntiEntropyBatchBytes = repairBytes; + }); + var keys = new DisseminationKey[] { "first", "second", "third", "fourth" }; + ns.ExpectedKeys.UnionWith(keys); + transport.ExchangeAntiEntropyHandler = (target, request, _) => + { + Assert.Equal(repairItems, request.MaxResponseItems); + Assert.Equal(repairBytes, request.MaxResponseBytes); + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(ns.Name, keys.Select(key => ns.CreateItem(peer, key, 1)).ToArray()), + }); + }; + try + { + await protocol.RunAntiEntropyRound(cancellationToken); + Assert.Equal(expectedItems, keys.Count(key => ns.GetVersion(key) == 1)); + + var broadcast = await protocol.ReceiveBroadcast(new() + { + Sender = peer, + Values = CreateValueGroups(ns.Name, keys.Select(key => ns.CreateItem(peer, key, 2)).ToArray()), + }, cancellationToken); + Assert.Equal(keys.Length, broadcast.Acknowledgments[ns.Name].Count); + Assert.All(keys, key => Assert.Equal(2, ns.GetVersion(key))); + + var repair = await protocol.ReceiveAntiEntropy(new() + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = new() { [ns.Name] = keys.Select(key => new DigestEntry(key, 0)).ToList() }, + }, cancellationToken); + Assert.Equal(expectedItems, GetAntiEntropyResponseValues(repair).Count()); + Assert.True(repair.Truncated); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.RootPacing.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.RootPacing.cs new file mode 100644 index 00000000000..af9b6ac5075 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.RootPacing.cs @@ -0,0 +1,170 @@ +using Microsoft.Extensions.Time.Testing; +using Orleans.Configuration; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task MembershipTrafficCannotFlushUnadmittedRootLoadUpdates() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41301); + var peer = CreateSilo(41302); + var clock = new FakeTimeProvider(); + var load = new FakeNamespace(local, new DisseminationNamespace("paced-load")) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + load.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(25); + var membership = new FakeNamespace(local, new DisseminationNamespace("urgent-membership")); + membership.Options.Priority = DisseminationPriority.High; + var transport = new FakeTransport(local, peer); + var firstLoad = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondLoad = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var firstMembership = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondMembership = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = (target, batch, _) => + { + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Add((target, batch)); + } + foreach (var value in GetBroadcastValues(batch)) + { + if (value.Value.Key == new DisseminationKey("load")) + { + (value.Value.ToVersion == 1 ? firstLoad : secondLoad).TrySetResult(); + } + else + { + (value.Value.ToVersion == 1 ? firstMembership : secondMembership).TrySetResult(); + } + } + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var protocol = CreateProtocol(transport, [load, membership], timeProvider: clock); + try + { + load.SetValue("load", 1); + Assert.True(await protocol.Publish(load, "load", 1, cancellationToken)); + membership.SetValue("membership", 1); + Assert.True(await protocol.Publish(membership, "membership", 1, cancellationToken)); + clock.Advance(TimeSpan.Zero); + await firstMembership.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.False(firstLoad.Task.IsCompleted); + clock.Advance(TimeSpan.FromMilliseconds(24)); + Assert.False(firstLoad.Task.IsCompleted); + clock.Advance(TimeSpan.FromMilliseconds(1)); + await firstLoad.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + + load.SetValue("load", 2); + Assert.True(await protocol.Publish(load, "load", 2, cancellationToken)); + membership.SetValue("membership", 2); + Assert.True(await protocol.Publish(membership, "membership", 2, cancellationToken)); + clock.Advance(TimeSpan.Zero); + await secondMembership.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.False(secondLoad.Task.IsCompleted); + clock.Advance(TimeSpan.FromMilliseconds(199)); + Assert.False(secondLoad.Task.IsCompleted); + clock.Advance(TimeSpan.FromMilliseconds(1)); + await secondLoad.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + finally + { + load.Options.Enabled = false; + membership.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task RootPendingStateHandsOffImmediatelyWhenTheRootChanges() + { + var cancellationToken = TestContext.Current.CancellationToken; + var newRoot = CreateSilo(41311); + var local = CreateSilo(41312); + var peer = CreateSilo(41313); + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(25); + var transport = new FakeTransport(local, peer); + var handedOff = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = (target, batch, _) => + { + Assert.Equal(newRoot, target); + if (GetBroadcastValues(batch).Any(value => value.Value.Key == new DisseminationKey("pending"))) + { + handedOff.TrySetResult(); + } + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var protocol = CreateProtocol(transport, ns, timeProvider: clock); + var start = clock.GetTimestamp(); + try + { + ns.SetValue("pending", 1); + Assert.True(await protocol.Publish(ns, "pending", 1, cancellationToken)); + transport.Peers.Add(newRoot); + ns.SetValue("new", 1); + Assert.True(await protocol.Publish(ns, "new", 1, cancellationToken)); + clock.Advance(TimeSpan.Zero); + await handedOff.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(start)); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task RootAcceptsAndDistributesACompleteTwoThousandSiloInventory() + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(2000); + var local = members[0]; + var ns = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + var defaults = new DeploymentLoadPublisherOptions().Dissemination; + ns.Options.MaxPendingItemCount = defaults.MaxPendingItemCount; + ns.Options.MaxCoalescingDelay = defaults.MaxCoalescingDelay; + var transport = new FakeTransport(local, members[1..]); + var protocol = CreateProtocol(transport, ns, timeProvider: new FakeTimeProvider()); + try + { + var values = members.Select(member => ns.CreateItem(member, member, 1)).ToArray(); + var response = await protocol.ReceiveBroadcast(new() + { + Sender = members[1], + Values = CreateValueGroups(values), + }, cancellationToken); + Assert.Equal(members.Length, response.Acknowledgments[ns.Name].Count); + await protocol.FlushPendingBroadcast(cancellationToken); + + Assert.Equal(8, transport.BroadcastBatches.Count); + Assert.Equal(members[1..9], transport.BroadcastBatches.Select(batch => batch.Peer).Order()); + Assert.All(transport.BroadcastBatches, batch => + { + Assert.Equal(members.Length, GetBroadcastValues(batch.Batch).Count()); + Assert.All(GetBroadcastValues(batch.Batch), value => Assert.Equal(1, value.Value.ToVersion)); + }); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index 31e825ed1cf..a582d230599 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -3198,7 +3198,8 @@ public void OptionsValidatorRejectsInvalidExpectedUpdateCadence() public void NamespaceOptionsUseExpectedUpdateCadenceDefaults() { Assert.Equal(TimeSpan.FromSeconds(5), new DeploymentLoadPublisherOptions().Dissemination.ExpectedUpdateCadence); - Assert.Equal(TimeSpan.FromMilliseconds(500), new DeploymentLoadPublisherOptions().Dissemination.MaxCoalescingDelay); + Assert.Equal(TimeSpan.FromMilliseconds(25), new DeploymentLoadPublisherOptions().Dissemination.MaxCoalescingDelay); + Assert.Equal(8192, new DeploymentLoadPublisherOptions().Dissemination.MaxPendingItemCount); Assert.Equal(TimeSpan.FromSeconds(10), new ClusterMembershipOptions().Dissemination.ExpectedUpdateCadence); } diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs new file mode 100644 index 00000000000..495ecc2a3fa --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs @@ -0,0 +1,883 @@ +using System.Collections.Concurrent; +using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; +using KeyNotification = Orleans.Runtime.Dissemination.DisseminationBroadcastQueue.KeyNotification; + +namespace UnitTests.Dissemination; + +[TestCategory("BVT"), TestCategory("Dissemination")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Dissemination")] +public class DisseminationRootBatcherTests +{ + private static CancellationToken TestToken => TestContext.Current.CancellationToken; + private static TimeSpan Milliseconds(int value) => TimeSpan.FromMilliseconds(value); + + [Fact] + public async Task IdleRootCollectsFor25MillisecondsAndReturnsToIdle() + { + await using var rig = new TestRig(); + rig.Clock.Advance(TimeSpan.FromHours(1)); + var start = rig.Clock.GetTimestamp(); + + Assert.Equal(Milliseconds(25), await rig.NotifyAsync(new KeyNotification("a", 1, true))); + rig.Clock.Advance(Milliseconds(24)); + Assert.Empty(rig.Attempts); + var attempt = await rig.AdvanceToDispatchAsync(Milliseconds(1)); + Assert.Equal(new KeyNotification("a", 1, true), Assert.Single(attempt.Notifications)); + Assert.Equal(Milliseconds(25), rig.Clock.GetElapsedTime(start, attempt.Timestamp)); + + rig.Clock.Advance(TimeSpan.FromHours(1)); + Assert.Single(rig.Attempts); + Assert.Equal(1, rig.Clock.TimerCount); + Assert.Empty(rig.Namespace.VersionReads); + } + + [Fact] + public async Task SustainedUpdatesDispatchAt200MillisecondIntervals() + { + await using var rig = new TestRig(); + Assert.Equal(Milliseconds(25), await rig.NotifyAsync(new KeyNotification("a", 1, false))); + var previous = await rig.AdvanceToDispatchAsync(Milliseconds(25)); + + for (var version = 2; version <= 5; version++) + { + Assert.Equal(Milliseconds(200), await rig.NotifyAsync(new KeyNotification("a", version, false))); + rig.Clock.Advance(Milliseconds(199)); + Assert.Equal(version - 1, rig.Attempts.Count); + var current = await rig.AdvanceToDispatchAsync(Milliseconds(1)); + Assert.Equal(Milliseconds(200), rig.Clock.GetElapsedTime(previous.Timestamp, current.Timestamp)); + Assert.Equal(version, Assert.Single(current.Notifications).Version); + previous = current; + } + } + + [Fact] + public async Task CollectionWindowCanOutliveLastDispatchInterval() + { + await using var rig = new TestRig(); + await rig.NotifyAsync(new KeyNotification("a", 1, false)); + var first = await rig.AdvanceToDispatchAsync(Milliseconds(25)); + rig.Clock.Advance(Milliseconds(190)); + Assert.Equal(Milliseconds(25), await rig.NotifyAsync(new KeyNotification("a", 2, false))); + rig.Clock.Advance(Milliseconds(24)); + Assert.Single(rig.Attempts); + + var second = await rig.AdvanceToDispatchAsync(Milliseconds(1)); + Assert.Equal(Milliseconds(215), rig.Clock.GetElapsedTime(first.Timestamp, second.Timestamp)); + } + + [Fact] + public async Task StaggeredAndBurstUpdatesMergeLatestValuesWithoutPushingDeadline() + { + await using var rig = new TestRig(); + Assert.Equal(Milliseconds(25), await rig.NotifyAsync(new KeyNotification("a", 1, true))); + rig.Clock.Advance(Milliseconds(10)); + Assert.True(rig.Batcher.Notify([new("b", 3, false), new("a", 2, false)])); + rig.Clock.Advance(Milliseconds(10)); + Assert.True(rig.Batcher.Notify([new("a", 9, false), new("a", 4, false)])); + Assert.Empty(rig.Attempts); + Assert.Equal(1, rig.Clock.ScheduleCount); + + var attempt = await rig.AdvanceToDispatchAsync(Milliseconds(5)); + Assert.Equal( + new KeyNotification[] { new("a", 9, true), new("b", 3, false) }, + attempt.Notifications); + } + + [Fact] + public async Task LateWakeDoesNotAccumulateCreditsOrProduceCatchupBurst() + { + await using var rig = new TestRig(); + rig.Options.MaxBatchItems = 1; + await rig.NotifyAsync(new("a", 1, false), new("b", 1, false), new("c", 1, false)); + + var first = await rig.AdvanceToDispatchAsync(TimeSpan.FromSeconds(5), morePending: true); + Assert.Single(rig.Attempts); + var second = await rig.AdvanceToDispatchAsync(Milliseconds(200), morePending: true); + Assert.Equal(2, rig.Attempts.Count); + Assert.Equal(Milliseconds(200), rig.Clock.GetElapsedTime(first.Timestamp, second.Timestamp)); + var third = await rig.AdvanceToDispatchAsync(Milliseconds(200)); + Assert.Equal(Milliseconds(200), rig.Clock.GetElapsedTime(second.Timestamp, third.Timestamp)); + } + + [Fact] + public async Task TwoThousandKeysUseBoundedFairChunksAndRetainedKeysUpdateAtCapacity() + { + await using var rig = new TestRig(); + rig.Options.MaxBatchItems = 128; + rig.Namespace.Options.MaxPendingItemCount = 2000; + var notifications = Enumerable.Range(0, 2001) + .Select(static index => new KeyNotification($"key-{index}", 1, false)).ToArray(); + var scheduled = rig.Clock.WhenScheduled(); + Assert.False(rig.Batcher.Notify(notifications)); + Assert.Equal(Milliseconds(25), await Phase(scheduled, "bounded inventory collection")); + Assert.True(rig.Batcher.Notify([new("key-0", 9, false), new("key-1999", 7, false)])); + Assert.False(rig.Batcher.Notify([new("overflow", 1, false)])); + + var first = await rig.AdvanceToDispatchAsync(Milliseconds(25), morePending: true); + Assert.Equal(128, first.Notifications.Length); + Assert.Equal(9, first.Notifications[0].Version); + + // Updating a waiting key must not move it behind later keys. A dispatched key can be + // re-admitted, but joins the tail instead of perpetually jumping ahead of the old inventory. + Assert.True(rig.Batcher.Notify( + [new("key-128", 6, false), new("key-0", 10, false), new("overflow", 1, false)])); + for (var wave = 1; wave < 16; wave++) + { + var attempt = await rig.AdvanceToDispatchAsync(Milliseconds(200), morePending: wave < 15); + Assert.Equal(wave < 15 ? 128 : 82, attempt.Notifications.Length); + } + + var sent = rig.Attempts.SelectMany(static attempt => attempt.Notifications).ToArray(); + Assert.Equal( + notifications.Take(2000).Select(static notification => notification.Key) + .Concat(new DisseminationKey[] { "key-0", "overflow" }), + sent.Select(static notification => notification.Key)); + Assert.Equal(6, sent[128].Version); + Assert.Equal(7, sent[1999].Version); + Assert.Equal(new KeyNotification("key-0", 10, false), sent[2000]); + Assert.DoesNotContain(sent, static notification => notification.Key.Equals(new DisseminationKey("key-2000"))); + } + + [Theory] + [InlineData(true, true)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(false, false)] + public async Task UpdatesDuringDispatchPreserveGenerationForceAndInFlightCapacity(bool firstAccepted, bool newForce) + { + await using var rig = new TestRig(); + rig.Namespace.Options.MaxPendingItemCount = 1; + using var release = new ManualResetEventSlim(); + var cancellationToken = TestToken; + rig.OnDispatch = attempt => + { + if (attempt.Number == 1) + { + Assert.True(release.Wait(TimeSpan.FromSeconds(10), cancellationToken), "First dispatch was not released."); + } + + return attempt.Number > 1 || firstAccepted; + }; + + await rig.NotifyAsync(new KeyNotification("a", 1, true)); + var nextSchedule = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Milliseconds(25)); + var first = await rig.NextAttemptAsync(); + try + { + // Another thread must be able to enter Notify while the synchronous callback is blocked. + // Re-entering on the callback's thread alone would not prove the lock was released. + await Phase(Task.Run(() => + { + Assert.False(rig.Batcher.Notify([new("b", 1, true)])); + Assert.True(rig.Batcher.Notify([new("a", 2, newForce)])); + }, TestToken), "concurrent admission while callback owns the first snapshot"); + Assert.Equal(new KeyNotification("a", 1, true), Assert.Single(first.Notifications)); + } + finally + { + release.Set(); + } + + Assert.Equal(Milliseconds(200), await Phase(nextSchedule, "new generation scheduled after dispatch")); + var second = await rig.AdvanceToDispatchAsync(Milliseconds(200)); + Assert.Equal(new KeyNotification("a", 2, newForce), Assert.Single(second.Notifications)); + Assert.Equal(2, rig.Attempts.Count); + } + + [Theory] + [InlineData(false, 1L)] + [InlineData(true, 1L)] + [InlineData(true, 2L)] + public async Task PartialAdmissionRetriesPreserveOriginalAndNewForce(bool newForce, long version) + { + await using var rig = new TestRig(); + var firstPeerVersions = new Dictionary(); + var firstPeerSends = new List(); + var secondPeerSends = new List(); + rig.OnDispatch = attempt => + { + foreach (var notification in attempt.Notifications) + { + if (notification.Force + || !firstPeerVersions.TryGetValue(notification.Key, out var knownVersion) + || notification.Version > knownVersion) + { + firstPeerVersions[notification.Key] = notification.Version; + firstPeerSends.Add(notification); + } + } + + if (attempt.Number == 1) + { + return false; + } + + secondPeerSends.AddRange(attempt.Notifications); + return true; + }; + await rig.NotifyAsync(new("a", 1, true), new("b", 1, true)); + await rig.AdvanceToDispatchAsync(Milliseconds(25), morePending: true); + if (newForce) + { + Assert.True(rig.Batcher.Notify([new("a", version, true)])); + } + + var retried = await rig.AdvanceToDispatchAsync(Milliseconds(200)); + Assert.Equal( + new KeyNotification[] { new("a", version, true), new("b", 1, true) }, + retried.Notifications); + Assert.Equal( + new KeyNotification[] { new("a", 1, true), new("b", 1, true), new("a", version, true), new("b", 1, true) }, + firstPeerSends); + Assert.Equal(retried.Notifications, secondPeerSends); + } + + [Fact] + public async Task PartialRetryRotatesBehindWaitingKeys() + { + await using var rig = new TestRig(); + rig.Options.MaxBatchItems = 1; + rig.OnDispatch = attempt => attempt.Number != 1; + await rig.NotifyAsync(new("a", 1, true), new("b", 1, true), new("c", 1, true)); + await rig.AdvanceToDispatchAsync(Milliseconds(25), morePending: true); + await rig.AdvanceToDispatchAsync(Milliseconds(200), morePending: true); + await rig.AdvanceToDispatchAsync(Milliseconds(200), morePending: true); + await rig.AdvanceToDispatchAsync(Milliseconds(200)); + + Assert.Equal( + new KeyNotification[] { new("a", 1, true), new("b", 1, true), new("c", 1, true), new("a", 1, true) }, + rig.Attempts.SelectMany(static attempt => attempt.Notifications)); + } + + [Fact] + public async Task CallbackExceptionRetriesAndReportsFailure() + { + await using var rig = new TestRig(); + var failure = new InvalidOperationException("The parent dispatcher failed."); + rig.OnDispatch = attempt => attempt.Number == 1 ? throw failure : true; + await rig.NotifyAsync(new KeyNotification("a", 1, true)); + + await rig.AdvanceToDispatchAsync(Milliseconds(25), morePending: true); + var entry = Assert.Single(rig.Logger.Entries); + Assert.Equal(LogLevel.Warning, entry.Level); + Assert.Same(failure, entry.Exception); + var retry = await rig.AdvanceToDispatchAsync(Milliseconds(200)); + Assert.Equal(new KeyNotification("a", 1, true), Assert.Single(retry.Notifications)); + } + + [Fact] + public async Task CallbackLoggingFailureIsVisibleToAdmissionAndDrain() + { + await using var rig = new TestRig(); + var dispatchFailure = new InvalidOperationException("The parent dispatcher failed."); + rig.Logger.ThrowOnLog = true; + rig.OnDispatch = _ => throw dispatchFailure; + await rig.NotifyAsync(new KeyNotification("a", 1, true)); + rig.Clock.Advance(Milliseconds(25)); + var failure = await Phase(rig.Logger.WorkerFailed.Task, "dispatch logging failure recorded"); + var aggregate = Assert.IsType(failure); + Assert.Same(dispatchFailure, aggregate.InnerExceptions[0]); + + var admission = Assert.Throws(() => rig.Batcher.Notify([new("b", 1, true)])); + Assert.Same(failure, admission.InnerException); + var drain = await Assert.ThrowsAsync(() => rig.Batcher.FlushAsync(TestToken)); + Assert.Same(failure, drain.InnerException); + } + + [Fact] + public async Task WorkerFailureIsObservedByActiveAndLaterFlushNotifyAndStop() + { + await using var rig = new TestRig(); + using var release = new ManualResetEventSlim(); + var cancellationToken = TestToken; + rig.Logger.ThrowOnLog = true; + rig.OnDispatch = _ => + { + Assert.True(release.Wait(TimeSpan.FromSeconds(10), cancellationToken), "Failing dispatch was not released."); + return false; + }; + await rig.NotifyAsync(new KeyNotification("a", 1, true)); + rig.Clock.Advance(Milliseconds(25)); + await rig.NextAttemptAsync(); + Task activeFlush; + try + { + activeFlush = rig.Batcher.FlushAsync(TestToken); + Assert.False(activeFlush.IsCompleted); + rig.Clock.FailNextSchedule = true; + } + finally + { + release.Set(); + } + + var failure = await Phase(rig.Logger.WorkerFailed.Task, "terminal timer failure logged"); + var activeFailure = await Assert.ThrowsAsync( + () => Phase(activeFlush, "active flush observes terminal failure")); + Assert.Same(failure, activeFailure.InnerException); + var notificationFailure = Assert.Throws( + () => rig.Batcher.Notify([new("b", 2, false)])); + Assert.Same(failure, notificationFailure.InnerException); + var flushFailure = await Assert.ThrowsAsync(() => rig.Batcher.FlushAsync(TestToken)); + Assert.Same(failure, flushFailure.InnerException); + var stopFailure = await Assert.ThrowsAsync(() => rig.Batcher.StopAsync(TestToken)); + Assert.Same(failure, stopFailure.InnerException); + Assert.True(rig.Clock.TimerDisposed); + Assert.Single(rig.Attempts); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DisablingDropsPendingHintsAndReenableDoesNotReplay(bool global) + { + await using var rig = new TestRig(); + await rig.NotifyAsync(new KeyNotification("old", 1, true)); + if (global) + { + rig.Options.Enabled = false; + } + else + { + rig.Namespace.Options.Enabled = false; + } + + await Phase(rig.Batcher.FlushAsync(TestToken), "disabled hints retired"); + Assert.False(rig.Batcher.Notify([new("disabled", 2, true)])); + Assert.Empty(rig.Attempts); + rig.Options.Enabled = true; + rig.Namespace.Options.Enabled = true; + rig.Clock.Advance(TimeSpan.FromSeconds(1)); + Assert.Equal(Milliseconds(25), await rig.NotifyAsync(new KeyNotification("new", 3, false))); + + var sent = await rig.AdvanceToDispatchAsync(Milliseconds(25)); + Assert.Equal(new KeyNotification("new", 3, false), Assert.Single(sent.Notifications)); + Assert.Single(rig.Attempts); + } + + [Fact] + public async Task PruneRemovesRetiredKeysAndFreesAdmissionCapacity() + { + await using var rig = new TestRig(); + rig.Namespace.Options.MaxPendingItemCount = 2; + await rig.NotifyAsync(new("retired", 1, true), new("retained", 2, false)); + var scheduled = rig.Clock.WhenScheduled(); + rig.Batcher.Prune(new HashSet { "retained" }); + Assert.True(rig.Batcher.Notify([new("new", 3, false)])); + Assert.Equal(Milliseconds(25), await Phase(scheduled, "pruned inventory rescheduled")); + + var attempt = await rig.AdvanceToDispatchAsync(Milliseconds(25)); + Assert.Equal( + new KeyNotification[] { new("retained", 2, false), new("new", 3, false) }, + attempt.Notifications); + } + + [Fact] + public async Task PrunePreservesLiveKeysPublishedAfterInventoryCapture() + { + await using var rig = new TestRig(); + await rig.NotifyAsync(new("retired", 1, false), new("retained", 2, false)); + var inventory = new HashSet { "retained" }; + rig.Namespace.GetVersionHandler = key => key.Equals(new DisseminationKey("new")) ? 3 : 0; + Assert.True(rig.Batcher.Notify([new("new", 3, false)])); + var scheduled = rig.Clock.WhenScheduled(); + + rig.Batcher.Prune(inventory); + + Assert.Equal(Milliseconds(25), await Phase(scheduled, "fresh owner versions checked before pruning")); + var attempt = await rig.AdvanceToDispatchAsync(Milliseconds(25)); + Assert.Equal( + new KeyNotification[] { new("retained", 2, false), new("new", 3, false) }, + attempt.Notifications); + Assert.Equal(new DisseminationKey[] { "retired", "new" }, rig.Namespace.VersionReads); + } + + [Theory] + [InlineData(1L)] + [InlineData(2L)] + public async Task PruneVersionReadRunsOutsideLockAndPreservesRacingNotifications(long version) + { + await using var rig = new TestRig(); + await rig.NotifyAsync(new KeyNotification("a", 1, false)); + using var release = new ManualResetEventSlim(); + var cancellationToken = TestToken; + var versionRead = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + rig.Namespace.GetVersionHandler = key => + { + Assert.Equal(new DisseminationKey("a"), key); + versionRead.TrySetResult(); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), cancellationToken), "Owner version read was not released."); + return 0; + }; + var scheduled = rig.Clock.WhenScheduled(); + var prune = Task.Run(() => rig.Batcher.Prune(new HashSet()), cancellationToken); + try + { + await Phase(versionRead.Task, "pruning captured the old admission generation"); + await Phase(Task.Run( + () => Assert.True(rig.Batcher.Notify([new("a", version, false)])), cancellationToken), + "notification admitted while the owner version read is blocked"); + } + finally + { + release.Set(); + await Phase(prune, "pruning reconciles the captured generation"); + } + + Assert.Equal(Milliseconds(25), await Phase(scheduled, "racing notification remains scheduled")); + var attempt = await rig.AdvanceToDispatchAsync(Milliseconds(25)); + Assert.Equal(new KeyNotification("a", version, false), Assert.Single(attempt.Notifications)); + } + + [Fact] + public async Task PruneDuringDispatchDoesNotEraseReadmittedKey() + { + await using var rig = new TestRig(); + rig.Namespace.Options.MaxPendingItemCount = 1; + using var release = new ManualResetEventSlim(); + var cancellationToken = TestToken; + rig.OnDispatch = attempt => + { + if (attempt.Number == 1) + { + Assert.True(release.Wait(TimeSpan.FromSeconds(10), cancellationToken), "Pruned snapshot was not released."); + } + + return true; + }; + await rig.NotifyAsync(new KeyNotification("a", 1, true)); + var nextSchedule = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Milliseconds(25)); + await rig.NextAttemptAsync(); + try + { + rig.Batcher.Prune(new HashSet()); + Assert.True(rig.Batcher.Notify([new("a", 7, true)])); + } + finally + { + release.Set(); + } + + Assert.Equal(Milliseconds(200), await Phase(nextSchedule, "readmitted identity scheduled")); + var next = await rig.AdvanceToDispatchAsync(Milliseconds(200)); + Assert.Equal(new KeyNotification("a", 7, true), Assert.Single(next.Notifications)); + } + + [Fact] + public async Task HighPriorityBypassesCollectionAndCadenceButRetriesRemainPaced() + { + await using var rig = new TestRig(); + rig.Namespace.Options.Priority = DisseminationPriority.High; + rig.OnDispatch = attempt => attempt.Number != 1; + var start = rig.Clock.GetTimestamp(); + var retryScheduled = rig.Clock.WhenScheduled(); + Assert.True(rig.Batcher.Notify([new("a", 1, true)])); + var first = await rig.NextAttemptAsync(); + Assert.Equal(start, first.Timestamp); + Assert.Equal(Milliseconds(200), await Phase(retryScheduled, "high-priority retry paced")); + await rig.AdvanceToDispatchAsync(Milliseconds(200)); + + Assert.True(rig.Batcher.Notify([new("b", 2, true)])); + var next = await rig.NextAttemptAsync(); + await Phase(rig.Batcher.FlushAsync(TestToken), "high-priority handoff completes"); + Assert.Equal(rig.Clock.GetTimestamp(), next.Timestamp); + Assert.Equal(new KeyNotification("b", 2, true), Assert.Single(next.Notifications)); + } + + [Fact] + public async Task PruneReevaluatesCurrentRateWithoutGrantingCredits() + { + await using var rig = new TestRig(); + await rig.NotifyAsync(new KeyNotification("a", 1, false)); + var first = await rig.AdvanceToDispatchAsync(Milliseconds(25)); + await rig.NotifyAsync(new KeyNotification("b", 1, false)); + rig.Clock.Advance(Milliseconds(20)); + rig.Options.Overlay.AggregationBroadcastsPerSecond = 10; + var scheduled = rig.Clock.WhenScheduled(); + rig.Batcher.Prune(new HashSet { "b" }); + Assert.Equal(Milliseconds(80), await Phase(scheduled, "updated rate applied to existing deadline")); + rig.Clock.Advance(Milliseconds(79)); + Assert.Single(rig.Attempts); + var second = await rig.AdvanceToDispatchAsync(Milliseconds(1)); + Assert.Equal(Milliseconds(100), rig.Clock.GetElapsedTime(first.Timestamp, second.Timestamp)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MembershipWakeForceHandsOffPendingChunksWithoutCredits(bool previousDispatch) + { + await using var rig = new TestRig(); + var start = rig.Clock.GetTimestamp(); + rig.Options.MaxBatchItems = 2; + if (previousDispatch) + { + await rig.NotifyAsync(new KeyNotification("seed", 1, false)); + await rig.AdvanceToDispatchAsync(Milliseconds(25)); + } + + await rig.NotifyAsync( + new("a", 2, true), new("b", 3, true), new("c", 4, true), new("d", 5, true), new("e", 6, true)); + var handoffTime = rig.Clock.GetTimestamp(); + + rig.Batcher.WakeForMembershipChange(); + + // Await every callback before calling Flush: otherwise Flush could conceal a missing + // membership wake or failure to bypass collection/pacing for later identity chunks. + var handoffs = new[] { await rig.NextAttemptAsync(), await rig.NextAttemptAsync(), await rig.NextAttemptAsync() }; + await Phase(rig.Batcher.FlushAsync(TestToken), "membership handoff reconciled"); + Assert.Equal(new[] { 2, 2, 1 }, handoffs.Select(static attempt => attempt.Notifications.Length)); + Assert.Equal(new long[] { 2, 3, 4, 5, 6 }, handoffs.SelectMany(static attempt => attempt.Notifications).Select(static n => n.Version)); + Assert.All(handoffs, attempt => Assert.Equal(handoffTime, attempt.Timestamp)); + Assert.Equal(previousDispatch ? Milliseconds(25) : TimeSpan.Zero, rig.Clock.GetElapsedTime(start, handoffTime)); + + Assert.Equal(Milliseconds(200), await rig.NotifyAsync(new KeyNotification("later", 7, false))); + rig.Clock.Advance(Milliseconds(199)); + Assert.Equal(previousDispatch ? 4 : 3, rig.Attempts.Count); + var later = await rig.AdvanceToDispatchAsync(Milliseconds(1)); + Assert.Equal(new KeyNotification("later", 7, false), Assert.Single(later.Notifications)); + } + + [Fact] + public async Task MembershipWakeKeepsPartialHandoffRetriesPaced() + { + await using var rig = new TestRig(); + rig.OnDispatch = attempt => attempt.Number != 1; + await rig.NotifyAsync(new KeyNotification("a", 1, true)); + var first = await rig.AdvanceToDispatchAsync(Milliseconds(25), morePending: true); + rig.Clock.Advance(Milliseconds(20)); + var scheduled = rig.Clock.WhenScheduled(); + + rig.Batcher.WakeForMembershipChange(); + + Assert.Equal(Milliseconds(180), await Phase(scheduled, "membership handoff preserves the retry floor")); + rig.Clock.Advance(Milliseconds(179)); + Assert.Single(rig.Attempts); + var retry = await rig.AdvanceToDispatchAsync(Milliseconds(1)); + Assert.Equal(Milliseconds(200), rig.Clock.GetElapsedTime(first.Timestamp, retry.Timestamp)); + Assert.Equal(new KeyNotification("a", 1, true), Assert.Single(retry.Notifications)); + } + + [Fact] + public async Task StopRejectsAdmissionAndForceDrainsBoundedChunksWithoutAdvancingTime() + { + await using var rig = new TestRig(); + rig.Options.MaxBatchItems = 2; + await rig.NotifyAsync( + new("a", 1, true), new("b", 2, true), new("c", 3, true), new("d", 4, true), new("e", 5, true)); + var start = rig.Clock.GetTimestamp(); + + await Phase(rig.Batcher.StopAsync(TestToken), "accepted keys handed to peer queues"); + + Assert.False(rig.Batcher.Notify([new("late", 6, true)])); + Assert.Equal(new[] { 2, 2, 1 }, rig.Attempts.Select(static attempt => attempt.Notifications.Length)); + Assert.Equal(new long[] { 1, 2, 3, 4, 5 }, rig.Attempts.SelectMany(static attempt => attempt.Notifications).Select(static n => n.Version)); + Assert.All(rig.Attempts, attempt => Assert.Equal(start, attempt.Timestamp)); + Assert.True(rig.Clock.TimerDisposed); + await rig.Batcher.StopAsync(TestToken); + } + + [Fact] + public async Task FlushForcesBoundedChunksWithoutStoppingAdmission() + { + await using var rig = new TestRig(); + rig.Options.MaxBatchItems = 2; + await rig.NotifyAsync(new("a", 1, true), new("b", 2, true), new("c", 3, true)); + var start = rig.Clock.GetTimestamp(); + + await Phase(rig.Batcher.FlushAsync(TestToken), "forced chunks handed to peer queues"); + + Assert.Equal(new[] { 2, 1 }, rig.Attempts.Select(static attempt => attempt.Notifications.Length)); + Assert.All(rig.Attempts, attempt => Assert.Equal(start, attempt.Timestamp)); + // Consume the two forced callbacks before awaiting the later, ordinarily paced callback. + await rig.NextAttemptAsync(); + await rig.NextAttemptAsync(); + Assert.Equal(Milliseconds(200), await rig.NotifyAsync(new KeyNotification("later", 4, true))); + var next = await rig.AdvanceToDispatchAsync(Milliseconds(200)); + Assert.Equal(new KeyNotification("later", 4, true), Assert.Single(next.Notifications)); + } + + [Fact] + public async Task StopCancellationEndsRetryBudgetAndPreservesCallerToken() + { + await using var rig = new TestRig(); + using var cancellation = new CancellationTokenSource(); + rig.OnDispatch = _ => false; + await rig.NotifyAsync(new KeyNotification("a", 1, true)); + var retryScheduled = rig.Clock.WhenScheduled(); + var stop = rig.Batcher.StopAsync(cancellation.Token); + await rig.NextAttemptAsync(); + Assert.Equal(Milliseconds(200), await Phase(retryScheduled, "forced stop retry remains paced")); + Assert.False(stop.IsCompleted); + Assert.False(rig.Batcher.Notify([new("late", 2, true)])); + + cancellation.Cancel(); + + var failure = await Assert.ThrowsAnyAsync(() => Phase(stop, "stop caller budget observed")); + Assert.Equal(cancellation.Token, failure.CancellationToken); + var laterFailure = await Assert.ThrowsAnyAsync( + () => rig.Batcher.StopAsync(CancellationToken.None)); + Assert.Equal(cancellation.Token, laterFailure.CancellationToken); + Assert.True(rig.Clock.TimerDisposed); + rig.Clock.Advance(TimeSpan.FromDays(1)); + Assert.Single(rig.Attempts); + } + + [Fact] + public async Task PreCanceledStopRejectsAdmissionWithoutDispatch() + { + await using var rig = new TestRig(); + using var cancellation = new CancellationTokenSource(); + await rig.NotifyAsync(new KeyNotification("a", 1, true)); + cancellation.Cancel(); + + var failure = await Assert.ThrowsAnyAsync( + () => rig.Batcher.StopAsync(cancellation.Token)); + + Assert.Equal(cancellation.Token, failure.CancellationToken); + Assert.False(rig.Batcher.Notify([new("b", 2, true)])); + Assert.Empty(rig.Attempts); + Assert.True(rig.Clock.TimerDisposed); + } + + [Fact] + public async Task FlushCancellationLeavesAcceptedWorkAvailableForPacedRetry() + { + await using var rig = new TestRig(); + using var cancellation = new CancellationTokenSource(); + rig.OnDispatch = attempt => attempt.Number != 1; + await rig.NotifyAsync(new KeyNotification("a", 1, true)); + var retryScheduled = rig.Clock.WhenScheduled(); + var flush = rig.Batcher.FlushAsync(cancellation.Token); + await rig.NextAttemptAsync(); + Assert.Equal(Milliseconds(200), await Phase(retryScheduled, "force flush retry remains paced")); + cancellation.Cancel(); + var failure = await Assert.ThrowsAnyAsync(() => flush); + Assert.Equal(cancellation.Token, failure.CancellationToken); + + var retry = await rig.AdvanceToDispatchAsync(Milliseconds(200)); + Assert.Equal(new KeyNotification("a", 1, true), Assert.Single(retry.Notifications)); + Assert.Equal(2, rig.Attempts.Count); + } + + private static async Task Phase(Task task, string phase) + { + try + { + await task.WaitAsync(TimeSpan.FromSeconds(10), TestToken); + } + catch (TimeoutException exception) + { + throw new TimeoutException($"Root batcher phase did not complete: {phase}.", exception); + } + } + + private static async Task Phase(Task task, string phase) + { + await Phase((Task)task, phase); + return await task; + } + + private sealed record Attempt(int Number, long Timestamp, KeyNotification[] Notifications); + + private sealed class TestRig : IAsyncDisposable + { + private readonly Channel _attempts = Channel.CreateUnbounded(); + private int _attemptNumber; + + public TestRig() + { + Batcher = new(Clock, Namespace, new TestOptionsMonitor(Options), Dispatch, Logger); + } + + public TestClock Clock { get; } = new(); + public TestNamespace Namespace { get; } = new(); + public DisseminationOptions Options { get; } = new() { Enabled = true }; + public TestLogger Logger { get; } = new(); + public DisseminationRootBatcher Batcher { get; } + public ConcurrentQueue Attempts { get; } = new(); + public Func? OnDispatch { get; set; } + + public async Task NotifyAsync(params KeyNotification[] notifications) + { + var scheduled = Clock.WhenScheduled(); + Assert.True(Batcher.Notify(notifications)); + return await Phase(scheduled, "notification timer armed"); + } + + public Task NextAttemptAsync() => + Phase(_attempts.Reader.ReadAsync(TestToken).AsTask(), "dispatch callback entered"); + + public async Task AdvanceToDispatchAsync(TimeSpan elapsed, bool morePending = false) + { + var scheduled = morePending ? Clock.WhenScheduled() : null; + Clock.Advance(elapsed); + var attempt = await NextAttemptAsync(); + if (scheduled is not null) + { + Assert.Equal(Milliseconds(200), await Phase(scheduled, $"next wave armed after attempt {attempt.Number}")); + } + else + { + await Phase(Batcher.FlushAsync(TestToken), $"attempt {attempt.Number} reconciled"); + } + + return attempt; + } + + private bool Dispatch(ReadOnlySpan notifications) + { + var attempt = new Attempt(Interlocked.Increment(ref _attemptNumber), Clock.GetTimestamp(), notifications.ToArray()); + Attempts.Enqueue(attempt); + Assert.True(_attempts.Writer.TryWrite(attempt)); + return OnDispatch?.Invoke(attempt) ?? true; + } + + public async ValueTask DisposeAsync() + { + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + try + { + await Batcher.StopAsync(cleanup.Token); + } + catch (OperationCanceledException) + { + } + catch (InvalidOperationException) when (Logger.WorkerFailed.Task.IsCompletedSuccessfully) + { + } + } + } + + private sealed class TestNamespace : IDisseminationNamespace + { + public DisseminationNamespace Name => new("root-batcher-test"); + public ConcurrentQueue VersionReads { get; } = new(); + public Func? GetVersionHandler { get; set; } + public DisseminationNamespaceOptions Options { get; } = new() + { + Enabled = true, + MaxCoalescingDelay = TimeSpan.FromMilliseconds(25), + MaxPendingItemCount = 8192, + }; + + public IEnumerable Digests => throw new InvalidOperationException("Root batching must not inspect namespace values."); + public long GetVersion(DisseminationKey key) + { + VersionReads.Enqueue(key); + return GetVersionHandler?.Invoke(key) ?? 0; + } + + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) => + throw new InvalidOperationException("Only peer queues may materialize payloads."); + public ValueTask ApplyValueAsync(DisseminationValue value, CancellationToken cancellationToken) => + throw new InvalidOperationException("Root batching must not apply payloads."); + } + + private sealed class TestOptionsMonitor(DisseminationOptions options) : IOptionsMonitor + { + public DisseminationOptions CurrentValue => options; + public DisseminationOptions Get(string? name) => options; + public IDisposable? OnChange(Action listener) => null; + } + + // The test is the sole fake-time driver. A schedule barrier completes only after the actual + // underlying timer was armed, so advancing time never races timer readiness. + private sealed class TestClock : TimeProvider + { + private readonly FakeTimeProvider _inner = new(); + private TaskCompletionSource? _nextSchedule; + private int _failNextSchedule; + private int _timerDisposed; + private int _timerCount; + private int _scheduleCount; + + public int TimerCount => Volatile.Read(ref _timerCount); + public int ScheduleCount => Volatile.Read(ref _scheduleCount); + public bool TimerDisposed => Volatile.Read(ref _timerDisposed) != 0; + public bool FailNextSchedule { set => Volatile.Write(ref _failNextSchedule, value ? 1 : 0); } + public override DateTimeOffset GetUtcNow() => _inner.GetUtcNow(); + public override long GetTimestamp() => _inner.GetTimestamp(); + public override long TimestampFrequency => _inner.TimestampFrequency; + public void Advance(TimeSpan elapsed) => _inner.Advance(elapsed); + + public Task WhenScheduled() + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Assert.Null(Interlocked.CompareExchange(ref _nextSchedule, completion, null)); + return completion.Task; + } + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + Interlocked.Increment(ref _timerCount); + return new TestTimer(this, _inner.CreateTimer(callback, state, dueTime, period)); + } + + private sealed class TestTimer(TestClock owner, ITimer inner) : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) + { + if (dueTime > TimeSpan.Zero && Interlocked.Exchange(ref owner._failNextSchedule, 0) != 0) + { + throw new InvalidOperationException("The root timer cannot schedule another wave."); + } + + var changed = inner.Change(dueTime, period); + if (dueTime > TimeSpan.Zero) + { + Interlocked.Increment(ref owner._scheduleCount); + Interlocked.Exchange(ref owner._nextSchedule, null)?.TrySetResult(dueTime); + } + + return changed; + } + + public void Dispose() + { + inner.Dispose(); + Volatile.Write(ref owner._timerDisposed, 1); + } + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + } + } + + private sealed class TestLogger : ILogger + { + public ConcurrentQueue<(LogLevel Level, Exception? Exception)> Entries { get; } = new(); + public TaskCompletionSource WorkerFailed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public bool ThrowOnLog { get; set; } + public bool IsEnabled(LogLevel logLevel) => true; + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + Entries.Enqueue((logLevel, exception)); + if (logLevel == LogLevel.Error && exception is not null) + { + WorkerFailed.TrySetResult(exception); + } + + if (ThrowOnLog) + { + throw new InvalidOperationException("The test logger throws."); + } + } + } +} From baf402e2229064194c6b999f40663a44660140cd Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 16:16:09 -0700 Subject: [PATCH 39/50] test(dissemination): preserve sequential harness execution on xUnit 4 --- .../Tests/VersionSkewTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs index 45da2189724..645e9a92e4b 100644 --- a/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs +++ b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs @@ -1,8 +1,10 @@ using System.Diagnostics; using System.Text.Json; using Xunit; +using Xunit.Sdk; +using Xunit.v3; -[assembly: CollectionBehavior(DisableTestParallelization = true)] +[assembly: Parallelization(Mode = ParallelMode.None)] namespace Orleans.Dissemination.IntegrationHarness; From 2b13acc4fe96b33c84d902ff64e6124f3b3fa6be Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 20:03:17 -0700 Subject: [PATCH 40/50] perf(dissemination): synchronize load publication with cohort receipts --- .../implementation/runtime-dissemination.md | 23 +- .../Options/DisseminationOptions.cs | 16 +- src/Orleans.Core/OrleansContracts.txt | 1 + .../IDisseminationSystemTarget.cs | 22 + .../Options/DeploymentLoadPublisherOptions.cs | 11 +- .../Options/DisseminationOptionsValidator.cs | 5 - ...entLoadStatisticsDisseminationNamespace.cs | 2 + .../DisseminationProtocol.Cohorts.cs | 166 +++ .../Dissemination/DisseminationProtocol.cs | 54 +- .../Dissemination/DisseminationRootBatcher.cs | 611 +++++--- .../DisseminationSystemTarget.cs | 13 + .../Dissemination/IDisseminationNamespace.cs | 2 + .../Dissemination/IDisseminationService.cs | 6 + .../Placement/DeploymentLoadPublisher.cs | 87 +- src/api/Orleans.Core/Orleans.Core.cs | 2 - .../Silo/NewRuntime.cs | 5 +- .../Silo/Program.cs | 2 +- .../Runtime/DeploymentLoadPublisherTests.cs | 25 +- .../DisseminationAggregationOptionsTests.cs | 24 +- ...seminationProtocolTests.CohortPublisher.cs | 573 ++++++++ .../DisseminationProtocolTests.Cohorts.cs | 366 +++++ .../DisseminationProtocolTests.RootPacing.cs | 170 --- .../DisseminationProtocolTests.cs | 47 +- .../DisseminationRootBatcherTests.cs | 1267 ++++++++++++----- 24 files changed, 2636 insertions(+), 864 deletions(-) create mode 100644 src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Cohorts.cs delete mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.RootPacing.cs diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index cfb15186597..0ccb78325b0 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -1,7 +1,7 @@ --- title: Runtime state dissemination description: Architecture, invariants, repair, failure semantics, and integration boundaries for Orleans runtime dissemination. -ms.date: 09/10/2026 +ms.date: 09/16/2026 ms.topic: concept-article --- @@ -23,6 +23,9 @@ flowchart LR Producer[Runtime state producer] --> Namespace[IDisseminationNamespace] Namespace --> Protocol[DisseminationProtocol] Protocol --> Queue[Per-peer broadcast pumps] + Protocol --> Cohort[Load cohorts and publication receipts] + Cohort --> Queue + Cohort -.-> Producer Queue --> Target[Remote dissemination system target] Target --> Apply[Namespace apply] Apply --> Forward[Deterministic forwarding tree] @@ -31,7 +34,7 @@ flowchart LR Repair --> Authority ``` -`DisseminationProtocol` coordinates routing, peer capability evidence, anti-entropy, and application isolation. `DisseminationRootBatcher` owns the aggregation root's bounded, latest-key pending set and broadcast cadence. `DisseminationBroadcastQueue` owns per-peer scheduling, retry, drain, and acknowledged-version ledgers. `DisseminationMembership` projects one membership snapshot into topology-specific member sets. Each `IDisseminationNamespace` owns serialization, current versions, retained history, payload limits, repair construction, and application semantics. +`DisseminationProtocol` coordinates routing, peer capability evidence, anti-entropy, and application isolation. `DisseminationRootBatcher` owns the aggregation root's bounded latest-key cohorts, producer contributions, and seal receipts. `DisseminationBroadcastQueue` owns per-peer scheduling, retry, drain, and acknowledged-version ledgers. `DisseminationMembership` projects one membership snapshot into topology-specific member sets. Each `IDisseminationNamespace` owns serialization, current versions, retained history, payload limits, repair construction, and application semantics. Queue entries contain `(namespace, key)` identities. A pump acquires both destination ownership and a global broadcast slot before materializing a repair. Coalesced notifications therefore use the latest namespace state and acknowledged baseline after admission, without retaining serialized payloads for waiting peers. @@ -71,15 +74,19 @@ Every silo derives routing from the same ordered membership projection. Members Membership uses the broadcast forest: for fanout `f` and zero-based member index `i`, a forwarding node selects children starting at `f * (i + 1)` and continuing for at most `f` members. An originator sends to the first `f` members, excluding itself, plus its normal forwarding children. -Deployment load uses root aggregation followed by tree distribution. Node `i > 0` has parent `(i - 1) / f`; its children start at `f * i + 1`. Its fanout defaults to eight, independently of the membership forest. Each non-root producer sends its own update directly to the root immediately. The root retains one latest notification per key and collects for 25 ms. It starts at most five normal broadcast waves per second, so the next deadline is the later of the first pending notification plus the collection window and the previous wave start plus 200 ms. New arrivals keep that deadline. An idle root sends after the collection window, while sustained traffic combines updates between paced waves. +Deployment load uses root aggregation followed by tree distribution. Node `i > 0` has parent `(i - 1) / f`; its children start at `f * i + 1`. Its fanout defaults to eight, independently of the membership forest. Each non-root producer sends its own fresh sample directly to the root through a dedicated aggregation RPC. The root's own sample enters the same cohort locally. A cohort captures its expected Active producer incarnations and retains the latest notification per key. It seals after every expected producer contributes, or after the configured publication period elapses from cohort opening. New arrivals preserve that deadline; duplicate messages and ordinary forwarding hints preserve distinct-producer counting. -Admitted waves enter every child queue atomically. Relays forward immediately. The root's pending set stays outside the per-peer queues, so urgent membership traffic can proceed without prematurely flushing unadmitted load values. Explicit flush and shutdown drain can bypass normal pacing within their caller's cancellation budget. A former root hands pending state to the current root, including when it has left Active membership. +Ingress RPCs remain open asynchronously until the cohort seals and its distribution work is admitted. Their receipts carry the remaining delay to the cohort's next publication boundary: `max(0, cohort start + period - receipt time)`. The publisher applies that delay to its sampling timer. Complete, phase-aligned cohorts can seal quickly while preserving approximately one sample per silo per period. A cohort that reaches its deadline schedules its next sample promptly. Local monotonic timing handles these relative delays across machines. + +The ingress receipt path has independent local-attempt admission. Membership uses its existing immediate peer pumps while load receipts are held. Each admitted cohort enters child queues through bounded batch admission, and relays forward immediately. Explicit flush and shutdown seal partial cohorts within the caller's cancellation budget. A former root hands pending state to the current root, including when it has left Active membership. Shutdown seals cohorts before waiting for their held protocol admissions, then drains peer queues. Aggregation ledgers advance on broadcast acknowledgments, after the receiver processes the batch and attempts downstream queue admission. An ingress producer which is also a child receives its own update in the root's distribution batch and forwards it to its descendants. Passive evidence of a peer's local value serves repair; broadcast acknowledgments establish distribution progress. Anti-entropy repairs gaps left by bounded queue admission or changed topology. -At one publication per silo per second, root ingress contributes `N - 1` requests per second. With `R` root waves per second, healthy fitting batches add at most roughly `R * (N - 1)` distribution requests. At the default five waves per second, the model is `6 * (N - 1)`, or 11,994 requests per second for 2,000 silos. An eight-child relay sends up to approximately 40 distribution requests per second. These are protocol projections, not measured multi-host capacity. Acknowledgment replies, retries, repair, topology transitions, forced drains, and wire-message splitting add separate work. Payload delivery still includes each recipient's copy of the statistics. +At one publication and one fitting cohort per second, root ingress contributes `N - 1` requests and tree distribution contributes another `N - 1`. The healthy steady-state model is `2 * (N - 1)`: 18 load requests per second at 10 silos, 198 at 100 silos, and 3,998 at 2,000 silos. An eight-child relay distributes approximately eight batches per second. Including the default periodic anti-entropy budget, the respective projections are 24, 258, and 5,198 requests per second, plus their replies. Root ingress remains concentrated at the root. These are algorithmic projections; production capacity and tail latency require measurement. Retries, topology transitions, forced drains, fallback, and wire-message splitting add work. Payload delivery includes each recipient's copy of the statistics. + +Cohort completeness depends on the slowest expected producer. Healthy aligned arrivals produce a short collection delay; an absent or paused producer makes the cohort use its full deadline. Repeatedly incomplete cohorts can leave samples approaching two publication periods old before replacement. Startup, membership changes, scheduling skew, and synchronized ingress/reply bursts are important latency and capacity cases. -Active members are ordered by silo address. Ingress moves toward a smaller root; distribution moves toward larger children. A non-root node receiving an ingress from a higher-address sender redirects it to its current root immediately, accommodating a changed root without adding another batching window. Membership repair reconciles differing views. +Active members are ordered by silo address. Ingress moves toward a smaller root; distribution moves toward larger children. A former root rejects new contribution requests, allowing the publisher's direct path to preserve delivery while subsequent publications resolve the new root. Already accepted cohort state and ordinary aggregation forwarding hints move toward the current root. Membership repair reconciles differing views. Namespaces select a membership scope before topology construction: @@ -104,7 +111,7 @@ Dissemination is opt-in. Enable | 2 | Target depth used to derive fanout. | | / | 4 / 32 | Bounds for derived fanout. | | | 8 | Maximum children per load-distribution relay. | -| | 5 | Normal root broadcast-wave frequency. | +| | 1 second | Load sampling period, cohort deadline, and receipt-driven next-round target. | | | 5 seconds | Repair-round cadence and retry-delay ceiling. | | | 3 | Maximum peers selected in one repair round and independent per-silo active local repair attempt limit. | | / | 8,192 / 1 MiB | Independent repair limits, capped by the global batch limits and negotiated with the peer. | @@ -116,7 +123,7 @@ Dissemination is opt-in. Enable . Operators can enable and tune membership and deployment-load dissemination independently while retaining the local concurrency and per-message bounds. -Deployment load samples remain on the one-second publication cadence. Root collection defaults to 25 ms with a five-wave-per-second admission limit, and the expected repair-update cadence remains five seconds. Producer ingress and relay forwarding bypass collection delays. Membership retains high priority and bypasses coalescing. Configure the load pending-key bound to cover the intended active inventory and stalls; each newer notification replaces the retained value for that key rather than adding another sample. +Deployment load samples use the configured publication period, with receipt feedback aligning subsequent samples to cohort boundaries. The expected repair-update cadence remains five seconds. Producer ingress and relay forwarding send immediately; root receipts follow cohort sealing. The publication receipt budget is twice the publication period, capped by the platform timer limit: one period for collection and one for transport and scheduling. Ordinary RPC timeouts also apply. Caller cancellation stops the publication; an unavailable or rejecting root, receipt expiry, or a shorter RPC timeout selects the direct delivery path. Membership retains high priority and bypasses coalescing. Configure the load pending-key bound to cover the intended active inventory and stalls; each newer notification replaces the retained value for that key rather than adding another sample. Inventory and peer-ledger maintenance runs on membership changes and bounded one-second maintenance opportunities instead of every received update. Failed passes remain eligible for retry. This keeps full inventory scans out of the normal per-message root path while promptly retiring peers when the routing view changes. diff --git a/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs index 5dc39d7696e..13cadec9e6f 100644 --- a/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs +++ b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs @@ -87,17 +87,6 @@ public sealed class DisseminationOverlayOptions /// The default is 8 and the value must be greater than zero. public int AggregationFanOutFactor { get; set; } = 8; - /// - /// Gets or sets the maximum number of logical aggregation distribution waves admitted by the root per second. - /// - /// - /// Rate admission can add a wait beyond the namespace's collection window. Relays forward admitted waves - /// immediately without another collection window or rate admission wait. The batch item and byte limits - /// can split one logical wave into multiple wire messages, which do not each consume a separate wave allowance. - /// - /// The default is 5 and the value must be between 1 and 1000, inclusive. - public int AggregationBroadcastsPerSecond { get; set; } = 5; - /// /// Gets or sets the interval between anti-entropy repair rounds. /// @@ -198,9 +187,8 @@ public sealed class DisseminationNamespaceOptions /// Per-peer batches can contain values from multiple namespaces and use the shortest configured delay among /// enabled namespaces. High-priority namespaces (see ) do not coalesce and are excluded /// from this calculation. - /// For aggregation trees, this is the root's collection window. Admission under - /// can add a separate wait; - /// relays forward admitted waves immediately. Batch limits can split a logical wave into multiple wire messages. + /// Deployment-load aggregation uses its publisher's refresh interval as the cohort deadline. + /// Producer ingress and distribution relays send immediately. Batch limits can split a cohort into multiple wire messages. /// /// The delay is 100 milliseconds by default and must be between 1 millisecond and approximately 49.7 days. public TimeSpan MaxCoalescingDelay { get; set; } = TimeSpan.FromMilliseconds(100); diff --git a/src/Orleans.Core/OrleansContracts.txt b/src/Orleans.Core/OrleansContracts.txt index 02e96509eba..5a30a3f33cd 100644 --- a/src/Orleans.Core/OrleansContracts.txt +++ b/src/Orleans.Core/OrleansContracts.txt @@ -59,6 +59,7 @@ interface [GrainInterfaceType("Orleans.Runtime.IDeploymentLoadPublisher")] Orlea interface [GrainInterfaceType("Orleans.Runtime.IDisseminationSystemTarget")] Orleans.Runtime.IDisseminationSystemTarget [Version(0)] EF58CB3D: ExchangeAntiEntropy(Orleans.Runtime.DisseminationAntiEntropyRequest, System.Threading.CancellationToken) -> Task + 8C810F59: PublishAggregated(Orleans.Runtime.DisseminationPublicationRequest, System.Threading.CancellationToken) -> Task 017AA907: PushBroadcast(Orleans.Runtime.DisseminationBroadcastBatch, System.Threading.CancellationToken) -> Task interface [GrainInterfaceType("Orleans.Runtime.IGrainCallCancellationExtension")] Orleans.Runtime.IGrainCallCancellationExtension [Version(0)] diff --git a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs index caafcc97ca5..cd0544e3485 100644 --- a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs +++ b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs @@ -8,8 +8,30 @@ internal interface IDisseminationSystemTarget : ISystemTarget [Alias("EF58CB3D")] Task ExchangeAntiEntropy(DisseminationAntiEntropyRequest request, CancellationToken cancellationToken); + + [Alias("8C810F59")] + Task PublishAggregated(DisseminationPublicationRequest request, CancellationToken cancellationToken); } +[GenerateSerializer, Immutable] +internal sealed class DisseminationPublicationRequest +{ + [Id(0)] + public required SiloAddress Sender { get; init; } + + [Id(1)] + public DisseminationNamespace Namespace { get; init; } + + [Id(2)] + public required DisseminationBroadcastValue Value { get; init; } +} + +// Admission covers the sealed cohort's distribution work. The delay targets its next sampling boundary. +[GenerateSerializer, Immutable] +internal readonly record struct DisseminationPublicationReceipt( + [property: Id(0)] bool Accepted, + [property: Id(1)] TimeSpan NextPublicationDelay); + [GenerateSerializer, Immutable] internal readonly struct DisseminationNamespace : IEquatable, IComparable, IComparable, ISpanFormattable { diff --git a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs index ce1d26884a2..b7e9f58967d 100644 --- a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs @@ -22,20 +22,19 @@ public class DeploymentLoadPublisherOptions /// /// /// When all active peers confirm support, producers send updates directly to the aggregation root. - /// By default, the root collects updates for 25 milliseconds and admits at most 5 logical distribution - /// waves per second. The root's rate admission wait is separate from the collection window. Relays forward - /// admitted waves immediately through the parent/child tree, without another collection window or rate - /// admission wait. Batch item and byte limits can split one logical wave into multiple wire messages. + /// The root seals a cohort after one fresh contribution from each active silo, including itself, or after + /// elapses. Publication receipts complete at sealing and + /// align subsequent samples with the cohort's next publication boundary. Relays forward admitted cohorts + /// immediately through the parent/child tree. Batch item and byte limits can split a cohort into multiple wire messages. /// Confirmation remains valid for that silo generation /// until the peer explicitly rejects the namespace or leaves the eligible membership set. During bootstrap /// or mixed-version operation, direct publication covers all active peers so that an unsupported intermediate /// node cannot interrupt delivery. Direct publication also covers all active peers when dissemination is - /// unavailable, declines the update, throws, or cannot accept it within the refresh interval. + /// unavailable, declines the update, throws, or exceeds the bounded publication receipt wait. /// public DisseminationNamespaceOptions Dissemination { get; set; } = new() { ExpectedUpdateCadence = TimeSpan.FromSeconds(5), - MaxCoalescingDelay = TimeSpan.FromMilliseconds(25), MaxPendingItemCount = 8 * 1024, }; } diff --git a/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs b/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs index 2948a7fa168..9600121f3d5 100644 --- a/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs +++ b/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs @@ -45,11 +45,6 @@ public ValidateOptionsResult Validate(string? name, DisseminationOptions options return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.AggregationFanOutFactor)} must be greater than 0."); } - if (overlay.AggregationBroadcastsPerSecond is < 1 or > 1000) - { - return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.AggregationBroadcastsPerSecond)} must be between 1 and 1000."); - } - if (overlay.AntiEntropyInterval < TimeSpan.FromMilliseconds(1) || overlay.AntiEntropyInterval > MaxPeriodicTimerPeriod) { diff --git a/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs index fec048e4e18..9d7da5cca32 100644 --- a/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs @@ -19,6 +19,8 @@ internal sealed class DeploymentLoadStatisticsDisseminationNamespace( public DisseminationRoutingMode RoutingMode => DisseminationRoutingMode.AggregationTree; + public TimeSpan AggregationPeriod => options.CurrentValue.DeploymentLoadPublisherRefreshTime; + public DisseminationNamespaceOptions Options => options.CurrentValue.Dissemination; public DisseminationValue CreateValue(SiloAddress origin, SiloRuntimeStatistics statistics) diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs new file mode 100644 index 00000000000..bc1adc1d877 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs @@ -0,0 +1,166 @@ +using Orleans.Internal; + +namespace Orleans.Runtime.Dissemination; + +internal sealed partial class DisseminationProtocol +{ + private readonly DisseminationSendGate _publicationSendGate = new(1); + private readonly CancellationTokenSource _publicationShutdown = new(); + + public async ValueTask PublishAggregated( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + using var admission = _admission.TryEnter(); + if (!admission.Entered) + { + return Reject("stopping"); + } + + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _publicationShutdown.Token); + try + { + // Held load receipts have independent admission, keeping membership's peer pumps available. + using var lease = await _publicationSendGate.AcquireAsync(_localSilo, cancellation.Token); + var options = _options.CurrentValue; + if (!options.Enabled || !disseminationNamespace.Options.Enabled) + { + return Reject("disabled"); + } + + if (disseminationNamespace.RoutingMode != DisseminationRoutingMode.AggregationTree + || !Equals(key.Value, _localSilo) + || version <= 0 + || disseminationNamespace.GetVersion(key) < version) + { + return Reject("invalid-contribution"); + } + + var membership = await GetMembershipSnapshotForRouting( + disseminationNamespace.MembershipScope, _localSilo, cancellation.Token); + if (membership is null) + { + return Reject("membership-unavailable"); + } + + var repairRequest = new DisseminationRepairRequest( + key, fromVersion: null, toVersion: null, maxItemCount: 1, + options.MaxBatchBytes, disseminationNamespace.Options.MaxPayloadBytes); + var repair = disseminationNamespace.CreateRepair(repairRequest); + if (repair.Status != DisseminationRepairStatus.Produced + || !repair.IsComplete || repair.Values.Length != 1 + || repair.Version < version + || repair.Values[0].FromVersion != 0 + || !ValidateRepair(disseminationNamespace, repairRequest, repair, options)) + { + return Reject("invalid-repair"); + } + + var value = repair.Values[0]; + RecordValueUpdate(disseminationNamespace.Name, key, value.ToVersion); + DisseminationPublicationReceipt receipt; + if (membership.IsAggregationRoot) + { + var root = GetRootBatcher(disseminationNamespace); + if (root is null) + { + return Reject("stopping"); + } + + receipt = await root.PublishAsync(new(key, value.ToVersion, true), cancellation.Token); + } + else + { + var rootAddress = membership.Members[0]; + var target = _grainFactory.GetSystemTarget( + Constants.DisseminationSystemTargetType, rootAddress); + var request = new DisseminationPublicationRequest + { + Sender = _localSilo, + Namespace = disseminationNamespace.Name, + Value = new() { Value = value, TimeToLive = disseminationNamespace.Options.StaleItemTtl }, + }; + var response = target.PublishAggregated(request, cancellation.Token); + response.Ignore(); + receipt = await response.WaitAsync(cancellation.Token); + if (receipt.Accepted) + { + ConfirmPeerNamespaces(rootAddress, [disseminationNamespace.Name]); + } + } + + EmitPublication(disseminationNamespace.Name, receipt.Accepted, + receipt.Accepted ? "none" : "cohort-rejected"); + return receipt; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + cancellationToken.ThrowIfCancellationRequested(); + throw; + } + + DisseminationPublicationReceipt Reject(string reason) + { + EmitPublication(disseminationNamespace.Name, accepted: false, reason); + return default; + } + } + + public async Task ReceivePublication( + DisseminationPublicationRequest request, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + using var admission = _admission.TryEnter(); + ObjectDisposedException.ThrowIf(!admission.Entered, this); + var receivedTimestamp = _timeProvider.GetTimestamp(); + var options = _options.CurrentValue; + if (!options.Enabled || !TryGetEnabledNamespace(request.Namespace, out var ns)) + { + return Reject("disabled"); + } + + if (ns.RoutingMode != DisseminationRoutingMode.AggregationTree + || !Equals(request.Value.Value.Key.Value, request.Sender) + || request.Value.Value.FromVersion != 0 + || options.MaxBatchItems < 1 + || !ValidatePayloadSize(ns, request.Value.Value, options.MaxBatchBytes)) + { + return Reject("invalid-contribution"); + } + + var membership = await GetMembershipSnapshotForRouting(ns.MembershipScope, request.Sender, cancellationToken); + if (membership is null || !membership.IsAggregationRoot) + { + return Reject("root-changed"); + } + + var result = await ApplyReceivedValue(ns, request.Value, request.Sender, options, receivedTimestamp, cancellationToken); + if (result is not (DisseminationApplyResult.Applied or DisseminationApplyResult.Duplicate)) + { + return Reject("application-rejected"); + } + + ConfirmPeerNamespaces(request.Sender, [ns.Name]); + var root = GetRootBatcher(ns); + if (root is null) + { + return Reject("stopping"); + } + + var receipt = await root.PublishAsync( + new(request.Value.Value.Key, request.Value.Value.ToVersion, result == DisseminationApplyResult.Applied), + cancellationToken); + EmitPublication(ns.Name, receipt.Accepted, receipt.Accepted ? "cohort-sealed" : "cohort-rejected"); + return receipt; + + DisseminationPublicationReceipt Reject(string reason) + { + EmitPublication(request.Namespace, accepted: false, reason); + return default; + } + } +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index 52998eeb045..711b884aa32 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -1304,30 +1304,26 @@ internal async Task StopAsync(CancellationToken cancellationToken) { var admittedOperations = _admission.CloseAsync(); _antiEntropySendGate.Stop(); + _publicationSendGate.Stop(); + DisseminationRootBatcher[] roots; + lock (_rootBatcherLock) + { + _rootBatchersStopped = true; + roots = [.. _rootBatchers.Values]; + _rootBatchers.Clear(); + } + + // Sealing releases held ingress receipts, so it precedes waiting for their protocol admissions. + var rootStops = Task.WhenAll(roots.Select(root => root.StopAsync(cancellationToken))); try { - await _antiEntropyShutdown.CancelAsync(); - // Admitted publishers and receivers finish enqueueing before the accepted broadcasts drain. - await admittedOperations.WaitAsync(cancellationToken); + await Task.WhenAll(_antiEntropyShutdown.CancelAsync(), _publicationShutdown.CancelAsync()); + await Task.WhenAll(rootStops, admittedOperations.WaitAsync(cancellationToken)); } finally { - DisseminationRootBatcher[] roots; - lock (_rootBatcherLock) - { - _rootBatchersStopped = true; - roots = [.. _rootBatchers.Values]; - _rootBatchers.Clear(); - } - - try - { - await Task.WhenAll(roots.Select(root => root.StopAsync(cancellationToken))); - } - finally - { - await _broadcastQueue.StopAsync(cancellationToken); - } + rootStops.Ignore(); + await _broadcastQueue.StopAsync(cancellationToken); } cancellationToken.ThrowIfCancellationRequested(); @@ -1478,29 +1474,29 @@ private bool NotifyRoot( IDisseminationNamespace disseminationNamespace, ReadOnlySpan notifications) { - DisseminationRootBatcher root; + return notifications.IsEmpty || GetRootBatcher(disseminationNamespace)?.Notify(notifications) == true; + } + + private DisseminationRootBatcher? GetRootBatcher(IDisseminationNamespace disseminationNamespace) + { lock (_rootBatcherLock) { if (_rootBatchersStopped) { - return false; - } - - if (notifications.IsEmpty) - { - return true; + return null; } - if (!_rootBatchers.TryGetValue(disseminationNamespace.Name, out root!)) + if (!_rootBatchers.TryGetValue(disseminationNamespace.Name, out var root)) { root = new( _timeProvider, disseminationNamespace, _options, + () => _membership.GetSnapshot(disseminationNamespace.MembershipScope), values => DispatchRootBatch(disseminationNamespace, values), _logger); _rootBatchers.Add(disseminationNamespace.Name, root); } - } - return root.Notify(notifications); + return root; + } } private bool DispatchRootBatch( diff --git a/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs b/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs index c1a140b0e9d..bb59a9ddc43 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs @@ -6,8 +6,8 @@ namespace Orleans.Runtime.Dissemination; -// This budget counts logical root waves, not wire messages: each peer queue still owns payload -// materialization, byte limits, splitting, and RPC lifetime. Only bounded key identities live here. +// Cohorts contain identities, not values. Peer queues still own payload materialization, byte +// limits, splitting, and RPC lifetime. A receipt acknowledges admission there, not delivery. internal sealed partial class DisseminationRootBatcher { internal delegate bool DispatchCallback(ReadOnlySpan notifications); @@ -17,21 +17,24 @@ internal sealed partial class DisseminationRootBatcher private readonly IDisseminationNamespace _namespace; private readonly DisseminationNamespace _namespaceName; private readonly IOptionsMonitor _options; + private readonly Func _getMembership; private readonly DispatchCallback _dispatch; private readonly ILogger _logger; private readonly WakeTimer _timer; private readonly Task _worker; private readonly Dictionary _pending = []; private readonly LinkedList _ready = []; + private readonly Dictionary _producers = []; + private DisseminationMembershipSnapshot? _membership; + private Cohort? _cohort; private TaskCompletionSource? _drainCompletion; private Exception? _failure; - private long? _firstPendingTimestamp; - private long? _lastDispatchTimestamp; private long _generation; + private long? _retryTimestamp; + private TimeSpan _retryPeriod; private bool _dispatching; private bool _workerActive; - private bool _retrying; - private bool _forceHandoff; + private bool _handoff; private bool _stopping; private bool _aborted; private CancellationToken _abortToken; @@ -40,6 +43,7 @@ public DisseminationRootBatcher( TimeProvider timeProvider, IDisseminationNamespace disseminationNamespace, IOptionsMonitor options, + Func getMembership, DispatchCallback dispatch, ILogger logger) { @@ -47,6 +51,7 @@ public DisseminationRootBatcher( _namespace = disseminationNamespace; _namespaceName = disseminationNamespace.Name; _options = options; + _getMembership = getMembership; _dispatch = dispatch; _logger = logger; _timer = new(timeProvider); @@ -54,12 +59,11 @@ public DisseminationRootBatcher( _worker = RunAsync(); } - // False means at least one distinct key was not admitted. Updates to retained keys still succeed - // at the limit, including keys currently inside the synchronous dispatch callback. + // Hints (including owner inventory and forwarded values) never count as fresh contributions. public bool Notify(ReadOnlySpan notifications) { var settings = GetSettings(); - var accepted = true; + var membership = _getMembership(); var rejected = 0; var wake = false; lock (_lock) @@ -70,56 +74,26 @@ public bool Notify(ReadOnlySpan notifications) return false; } + membership = RefreshMembershipUnsafe(membership); if (!settings.Enabled) { - // Disabling explicitly abandons queued hints; re-enabling does not replay them. - // Namespace state and anti-entropy, rather than this queue, own durable convergence. ClearPendingUnsafe(); - accepted = false; wake = true; + rejected = notifications.Length; } else { - var wasEmpty = _pending.Count == 0; - var now = _timeProvider.GetTimestamp(); + var wasEmpty = _ready.Count == 0; foreach (var notification in notifications) { - if (_pending.TryGetValue(notification.Key, out var pending)) + if (!IsEligible(notification, membership) + || !TryAddUnsafe(notification, settings, membership, out _)) { - // Even an unchanged notification can race an inventory/version read. Protect - // it from pruning without making duplicate in-flight values dirty again. - pending.AdmissionGeneration = ++_generation; - if (!notification.Force && notification.Version <= pending.Notification.Version) - { - continue; - } - - pending.Notification = new( - notification.Key, - Math.Max(notification.Version, pending.Notification.Version), - pending.Notification.Force || notification.Force); - pending.Generation = pending.AdmissionGeneration; - pending.PendingSince ??= now; + rejected++; } - else - { - if (_pending.Count >= settings.MaxPendingItemCount) - { - accepted = false; - rejected++; - continue; - } - - pending = new(notification, ++_generation, now); - _pending.Add(notification.Key, pending); - _ready.AddLast(pending.Node); - } - - _firstPendingTimestamp ??= now; - // Replacements cannot advance the earliest pending timestamp or its deadline. - // Avoid waking/rearming once per key while a root is collecting thousands of keys. - wake |= wasEmpty || settings.HighPriority; } + + wake = _ready.Count > 0 && (wasEmpty || settings.HighPriority); } } @@ -130,42 +104,103 @@ public bool Notify(ReadOnlySpan notifications) if (rejected > 0) { - try + ReportRejection(rejected, settings.MaxPendingItemCount); + } + + return settings.Enabled && rejected == 0; + } + + public ValueTask PublishAsync( + KeyNotification notification, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var settings = GetSettings(); + var membership = _getMembership(); + Task completion; + var wake = false; + lock (_lock) + { + ThrowIfFailedUnsafe(); + membership = RefreshMembershipUnsafe(membership); + if (_stopping || !settings.Enabled || !membership.IsAggregationRoot + || notification.Version <= 0 + || notification.Key.Value is not SiloAddress silo || !membership.ContainsMember(silo)) { - LogAdmissionRejected(_logger, _namespaceName, rejected, settings.MaxPendingItemCount); + return new(new DisseminationPublicationReceipt(false, settings.Period)); } - catch (Exception exception) + + if (_producers.TryGetValue(silo, out var producer) && notification.Version <= producer.Version) { - Fail(exception); - DisposeTimer(); - } + // A retry must neither contribute again nor start another publication period. + // Force on a retry is not a new invalidation; independent invalidations use Notify. + if (_pending.TryGetValue(notification.Key, out var retained)) + { + retained.AdmissionGeneration = ++_generation; + } - for (var index = 0; index < rejected; index++) + if (notification.Version <= producer.CompletedVersion && producer.Outcome is { } outcome) + { + return new(CreateReceipt(outcome)); + } + + completion = notification.Version <= producer.InFlightVersion && producer.InFlightReceipt is { } inFlight + ? inFlight.Task : producer.Receipt!.Task; + } + else { - try + var wasEmpty = _ready.Count == 0; + if (!TryAddUnsafe(notification, settings, membership, out var pending, contribution: true)) { - DisseminationInstruments.OnQueueAdmissionRejected(_namespaceName); + return new(new DisseminationPublicationReceipt(false, settings.Period)); } - catch (Exception exception) + + producer ??= new(); + _producers[silo] = producer; + producer.Version = notification.Version; + // All newer versions before sealing share one signal and occupy one producer slot. + pending.Receipt ??= new(TaskCreationOptions.RunContinuationsAsynchronously); + producer.Receipt = pending.Receipt; + pending.Producer = producer; + pending.ProducerVersion = notification.Version; + _cohort!.HasPublications = true; + if (_cohort!.Membership.ContainsMember(silo)) { - try - { - LogDiagnosticFailed(_logger, exception, _namespaceName); - } - catch (Exception loggingFailure) - { - Fail(new AggregateException(exception, loggingFailure)); - DisposeTimer(); - } + _cohort.Contributors.Add(silo); } + + completion = pending.Receipt.Task; + wake = wasEmpty + || _cohort.Contributors.Count == _cohort.Membership.Members.Length; } } - return accepted; + if (wake) + { + Wake(); + } + + return AwaitReceiptAsync(completion, cancellationToken); + } + + private async ValueTask AwaitReceiptAsync( + Task completion, CancellationToken cancellationToken) + { + // Cancellation detaches this RPC only. Other callers and the admitted contribution survive. + var outcome = await completion.WaitAsync(cancellationToken).ConfigureAwait(false); + lock (_lock) + { + ThrowIfFailedUnsafe(); + } + + return CreateReceipt(outcome); + } + + private DisseminationPublicationReceipt CreateReceipt(ReceiptOutcome outcome) + { + var delay = outcome.Period - _timeProvider.GetElapsedTime(outcome.Started); + return new(outcome.Accepted, delay > TimeSpan.Zero ? delay : TimeSpan.Zero); } - // Flush forces all currently retained work into peer queues, not onto the wire. Concurrent - // admissions join this drain. A partial admission never turns a forced drain into a busy loop. public async Task FlushAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -192,8 +227,6 @@ public async Task FlushAsync(CancellationToken cancellationToken) } } - // Cancellation ends this owner's drain budget and abandons remaining hints. A wave already - // claimed for synchronous dispatch cannot be interrupted; subsequent waves will not be claimed. public async Task StopAsync(CancellationToken cancellationToken) { lock (_lock) @@ -214,6 +247,7 @@ public async Task StopAsync(CancellationToken cancellationToken) _aborted = true; _abortToken = cancellationToken; ClearPendingUnsafe(); + CompleteDrainUnsafe(); } DisposeTimer(); @@ -241,54 +275,79 @@ public void Prune(IReadOnlySet activeKeys) } } - var retiredCount = 0; - for (var i = 0; i < candidates.Count; i++) + // Namespace callbacks may reenter and may block. Reconcile against admission generations. + foreach (var candidate in candidates) { - var candidate = candidates[i]; - // Owner inventory was captured before entering this method. A positive current version - // means the key has since become live; namespace calls must not hold the batcher lock. - if (_namespace.GetVersion(candidate.Key) <= 0) + if (_namespace.GetVersion(candidate.Key) > 0) { - candidates[retiredCount++] = candidate; + continue; } - } - lock (_lock) - { - for (var i = 0; i < retiredCount; i++) + lock (_lock) { - var candidate = candidates[i]; if (_pending.TryGetValue(candidate.Key, out var pending) && ReferenceEquals(pending, candidate.Pending) && pending.AdmissionGeneration == candidate.Generation) { - _pending.Remove(candidate.Key); - if (pending.Node.List is not null) - { - _ready.Remove(pending.Node); - } + RemoveUnsafe(pending); } } - - RecomputeFirstPendingUnsafe(); } Wake(); } - // Called when the local silo loses the root role, not for ordinary membership changes. Request - // Flush-equivalent scheduling without a drain observer; the dispatcher resolves the new root. - // This does not store credits for later work or bypass partial-retry pacing. public void WakeForMembershipChange() { + var membership = _getMembership(); lock (_lock) { - _forceHandoff |= _pending.Count > 0; + RefreshMembershipUnsafe(membership); + // A new root is a different admission destination, so bypass an old child's retry floor. + _handoff |= _ready.Count > 0; } Wake(); } + private bool TryAddUnsafe( + KeyNotification notification, in Settings settings, + DisseminationMembershipSnapshot membership, out PendingKey pending, bool contribution = false) + { + if (_pending.TryGetValue(notification.Key, out pending!)) + { + pending.AdmissionGeneration = ++_generation; + if (!notification.Force && notification.Version <= pending.Notification.Version + && (!contribution || pending.Node.List is not null)) + { + return true; + } + + pending.Notification = new(notification.Key, + Math.Max(notification.Version, pending.Notification.Version), + pending.Notification.Force || notification.Force); + pending.Generation = pending.AdmissionGeneration; + } + else + { + if (_pending.Count >= settings.MaxPendingItemCount) + { + return false; + } + + pending = new(notification, ++_generation); + _pending.Add(notification.Key, pending); + } + + _cohort ??= new(membership, _timeProvider.GetTimestamp(), settings.Period); + if (pending.Node.List is null) + { + _ready.AddLast(pending.Node); + } + + return true; + } + private async Task RunAsync() { try @@ -298,6 +357,9 @@ private async Task RunAsync() while (true) { var settings = GetSettings(); + var membership = _getMembership(); + PendingDispatch[] work; + Cohort cohort; lock (_lock) { _workerActive = true; @@ -306,13 +368,15 @@ private async Task RunAsync() return; } + membership = RefreshMembershipUnsafe(membership); if (!settings.Enabled) { ClearPendingUnsafe(); } - if (_pending.Count == 0) + if (_ready.Count == 0) { + _cohort = null; CompleteDrainUnsafe(); if (_stopping) { @@ -323,16 +387,41 @@ private async Task RunAsync() break; } - var delay = GetDelayUnsafe(settings); + var delay = GetDelayUnsafe(settings.HighPriority); if (delay > TimeSpan.Zero) { _timer.Change(delay); _workerActive = false; break; } + + cohort = _cohort!; + work = new PendingDispatch[_ready.Count]; + for (var i = 0; i < work.Length; i++) + { + var pending = _ready.First!.Value; + _ready.RemoveFirst(); + work[i] = new(pending, pending.Generation, pending.Notification, + pending.Receipt, pending.Producer, pending.ProducerVersion); + if (pending.Producer is { } producer) + { + producer.InFlightVersion = pending.ProducerVersion; + producer.InFlightReceipt = pending.Receipt; + } + + pending.InFlight = pending.Receipt; + pending.Receipt = null; + pending.Producer = null; + pending.Notification = pending.Notification with { Force = false }; + } + + _cohort = null; + _handoff = false; + _retryTimestamp = null; + _dispatching = true; } - Dispatch(); + Dispatch(work, cohort, settings); } } } @@ -351,163 +440,226 @@ private async Task RunAsync() } } - private TimeSpan GetDelayUnsafe(in BatchingSettings settings) + private TimeSpan GetDelayUnsafe(bool highPriority) { - var pacedDelay = _lastDispatchTimestamp is { } last - ? settings.BroadcastInterval - _timeProvider.GetElapsedTime(last) - : TimeSpan.Zero; - if (_stopping || _drainCompletion is not null || _forceHandoff || settings.HighPriority) + var cohort = _cohort!; + if (_handoff) { - return _retrying ? pacedDelay : TimeSpan.Zero; + return TimeSpan.Zero; } - var collectionDelay = settings.CollectionDelay - - _timeProvider.GetElapsedTime(_firstPendingTimestamp!.Value); - return collectionDelay > pacedDelay ? collectionDelay : pacedDelay; + var retryDelay = _retryTimestamp is { } retry + ? _retryPeriod - _timeProvider.GetElapsedTime(retry) : TimeSpan.Zero; + var complete = cohort.Membership.Members.Length > 0 + && cohort.Contributors.Count == cohort.Membership.Members.Length; + var collectionDelay = _stopping || _drainCompletion is not null + || highPriority && !cohort.HasPublications || complete + ? TimeSpan.Zero : cohort.Period - _timeProvider.GetElapsedTime(cohort.Started); + return collectionDelay > retryDelay ? collectionDelay : retryDelay; } - // Keeping snapshots in a synchronous method prevents the worker state machine from retaining - // dispatched arrays across timer waits. Newer notifications never mutate a captured snapshot. - private void Dispatch() + private void Dispatch(PendingDispatch[] work, Cohort cohort, in Settings settings) { - var settings = GetSettings(); - PendingDispatch[] work; - KeyNotification[] notifications; - lock (_lock) + var notifications = new KeyNotification[Math.Min(work.Length, settings.MaxBatchItems)]; + for (var offset = 0; offset < work.Length; offset += notifications.Length) { - if (_failure is not null || _aborted || _ready.Count == 0 - || !settings.Enabled || GetDelayUnsafe(settings) > TimeSpan.Zero) + // Revalidate before every chunk: a callback can change membership or exhaust Stop's budget. + var membership = _getMembership(); + var count = Math.Min(notifications.Length, work.Length - offset); + var sendCount = 0; + lock (_lock) { - return; + membership = RefreshMembershipUnsafe(membership); + if (_aborted || _failure is not null) + { + break; + } + + for (var i = offset; i < offset + count; i++) + { + var item = work[i]; + if (IsCurrentUnsafe(item.Pending) && IsEligible(item.Notification, membership)) + { + notifications[sendCount++] = item.Notification; + } + } } - var count = Math.Min(_ready.Count, Math.Max(1, settings.MaxBatchItems)); - work = new PendingDispatch[count]; - notifications = new KeyNotification[count]; - for (var i = 0; i < count; i++) + var accepted = false; + try { - var pending = _ready.First!.Value; - _ready.RemoveFirst(); - notifications[i] = pending.Notification; - work[i] = new(pending, pending.Generation, pending.PendingSince!.Value, pending.Notification); - // A successful attempt consumes Force; a failed admission must preserve it for peers - // which still hold an earlier same-version state. - pending.Notification = pending.Notification with { Force = false }; - pending.PendingSince = null; + accepted = sendCount > 0 && _dispatch(notifications.AsSpan(0, sendCount)); + } + catch (Exception exception) + { + try + { + LogDispatchFailed(_logger, exception, _namespaceName); + } + catch (Exception loggingFailure) + { + Fail(new AggregateException("The root dispatch recovery diagnostic failed.", exception, loggingFailure)); + } } - _dispatching = true; - _lastDispatchTimestamp = _timeProvider.GetTimestamp(); - } - - var accepted = false; - Exception? dispatchFailure = null; - try - { - accepted = _dispatch(notifications); - } - catch (Exception exception) - { - dispatchFailure = exception; - } - finally - { lock (_lock) { - foreach (var item in work) + for (var i = offset; i < offset + count; i++) { + var item = work[i]; var pending = item.Pending; - if (!_pending.TryGetValue(pending.Notification.Key, out var current) - || !ReferenceEquals(current, pending)) + var admitted = accepted && IsEligible(item.Notification, membership) + && !_aborted && _failure is null && IsCurrentUnsafe(pending); + CompleteReceiptUnsafe(item, new(admitted, cohort.Started, cohort.Period)); + pending.InFlight = null; + if (!IsCurrentUnsafe(pending)) { - // Pruning can retire an in-flight identity and admit a new one with the same key. continue; } - if (accepted && pending.Generation == item.Generation) + if (admitted && pending.Generation == item.Generation) { _pending.Remove(pending.Notification.Key); } - else + else if (!admitted && !_aborted && _failure is null) { - if (!accepted) + // Receipts are final even after partial child admission. Only bounded hints + // retry; publishers can use direct fallback without waiting on a congested child. + if (pending.Notification.Version == item.Notification.Version) { - pending.PendingSince = item.PendingSince; - if (pending.Notification.Version == item.Notification.Version) + pending.Notification = pending.Notification with { - pending.Notification = pending.Notification with - { - Force = pending.Notification.Force || item.Notification.Force, - }; - } + Force = pending.Notification.Force || item.Notification.Force, + }; + } + + _cohort ??= new(membership, _timeProvider.GetTimestamp(), settings.Period); + if (pending.Node.List is null) + { + _ready.AddLast(pending.Node); } - _ready.AddLast(pending.Node); + _retryTimestamp = _timeProvider.GetTimestamp(); + _retryPeriod = settings.Period; } } + } + } - _retrying = !accepted; - _dispatching = false; - RecomputeFirstPendingUnsafe(); - if (_pending.Count == 0) - { - CompleteDrainUnsafe(); - } + lock (_lock) + { + _dispatching = false; + if (_pending.Count == 0) + { + CompleteDrainUnsafe(); } } + } - if (dispatchFailure is not null) + private void CompleteReceiptUnsafe(PendingDispatch item, ReceiptOutcome outcome) + { + if (item.Receipt is null) { - try + return; + } + + if (item.Producer is { } producer) + { + if (item.ProducerVersion >= producer.CompletedVersion) { - LogDispatchFailed(_logger, dispatchFailure, _namespaceName); + producer.CompletedVersion = item.ProducerVersion; + producer.Outcome = outcome; } - catch (Exception loggingFailure) + + if (ReferenceEquals(producer.InFlightReceipt, item.Receipt)) { - Fail(new AggregateException("The root dispatch recovery diagnostic failed.", dispatchFailure, loggingFailure)); + producer.InFlightReceipt = null; } } + + item.Receipt.TrySetResult(outcome); + } + + private DisseminationMembershipSnapshot RefreshMembershipUnsafe(DisseminationMembershipSnapshot membership) + { + if (_membership is { } current + && (ReferenceEquals(current, membership) || membership.MembershipVersion.Value < current.MembershipVersion.Value)) + { + return current; + } + + _membership = membership; + foreach (var silo in _producers.Keys.Where(silo => !membership.ContainsMember(silo)).ToArray()) + { + _producers.Remove(silo); + } + + foreach (var pending in _pending.Values.Where(pending => !IsEligible(pending.Notification, membership)).ToArray()) + { + RemoveUnsafe(pending); + } + + return membership; } - private BatchingSettings GetSettings() + private static bool IsEligible(KeyNotification notification, DisseminationMembershipSnapshot membership) => + notification.Key.Value is not SiloAddress silo || membership.ContainsMember(silo); + + private bool IsCurrentUnsafe(PendingKey pending) => + _pending.TryGetValue(pending.Notification.Key, out var current) && ReferenceEquals(current, pending); + + private void RemoveUnsafe(PendingKey pending) + { + var outcome = new ReceiptOutcome(false, _cohort?.Started ?? _timeProvider.GetTimestamp(), _cohort?.Period ?? TimeSpan.Zero); + CompleteReceiptUnsafe(new(pending, pending.Generation, pending.Notification, + pending.Receipt, pending.Producer, pending.ProducerVersion), outcome); + pending.InFlight?.TrySetResult(outcome); + if (pending.Receipt is not null && pending.Notification.Key.Value is SiloAddress silo) + { + _cohort?.Contributors.Remove(silo); + } + + _pending.Remove(pending.Notification.Key); + if (pending.Node.List is not null) + { + _ready.Remove(pending.Node); + } + + if (_ready.Count == 0) + { + _cohort = null; + } + } + + private Settings GetSettings() { var options = _options.CurrentValue; var namespaceOptions = _namespace.Options; + var period = _namespace.AggregationPeriod; return new( options.Enabled && namespaceOptions.Enabled, namespaceOptions.Priority == DisseminationPriority.High, - namespaceOptions.MaxPendingItemCount, - options.MaxBatchItems, - namespaceOptions.MaxCoalescingDelay, - TimeSpan.FromSeconds(1d / Math.Clamp(options.Overlay.AggregationBroadcastsPerSecond, 1, 1000))); + Math.Max(1, namespaceOptions.MaxPendingItemCount), + Math.Max(1, options.MaxBatchItems), + TimeSpan.FromMilliseconds(Math.Clamp(period.TotalMilliseconds, 1, uint.MaxValue - 1))); } - private void RecomputeFirstPendingUnsafe() + private void ClearPendingUnsafe() { - _firstPendingTimestamp = null; - foreach (var pending in _ready) + foreach (var pending in _pending.Values.ToArray()) { - var timestamp = pending.PendingSince!.Value; - if (_firstPendingTimestamp is null || timestamp < _firstPendingTimestamp.Value) - { - _firstPendingTimestamp = timestamp; - } + RemoveUnsafe(pending); } - } - private void ClearPendingUnsafe() - { - _pending.Clear(); - _ready.Clear(); - _firstPendingTimestamp = null; - _retrying = false; - _forceHandoff = false; + _cohort = null; + _retryTimestamp = null; + _handoff = false; } private void CompleteDrainUnsafe() { - _retrying = false; - _forceHandoff = false; + _retryTimestamp = null; + _handoff = false; _drainCompletion?.TrySetResult(); _drainCompletion = null; } @@ -563,6 +715,30 @@ private void DisposeTimer() } } + private void ReportRejection(int count, int limit) + { + try + { + LogAdmissionRejected(_logger, _namespaceName, count, limit); + for (var i = 0; i < count; i++) + { + try + { + DisseminationInstruments.OnQueueAdmissionRejected(_namespaceName); + } + catch (Exception exception) + { + LogDiagnosticFailed(_logger, exception, _namespaceName); + } + } + } + catch (Exception exception) + { + Fail(exception); + DisposeTimer(); + } + } + private void Fail(Exception exception) { lock (_lock) @@ -573,7 +749,8 @@ private void Fail(Exception exception) } _failure = exception; - // Complete normally and read _failure after awaiting: no abandoned shared task faults. + // Signals complete normally: a canceled caller must not leave an unobserved task fault. + ClearPendingUnsafe(); _drainCompletion?.TrySetResult(); } @@ -583,7 +760,7 @@ private void Fail(Exception exception) } catch { - // A failing logger must not hide the original failure from admission/drain callers. + // Preserve the original failure for admission and drain callers even if logging fails. } } @@ -592,31 +769,47 @@ private sealed class PendingKey public KeyNotification Notification; public long Generation; public long AdmissionGeneration; - public long? PendingSince; + public long ProducerVersion; + public Producer? Producer; + public TaskCompletionSource? Receipt; + public TaskCompletionSource? InFlight; public LinkedListNode Node { get; } - public PendingKey(KeyNotification notification, long generation, long pendingSince) + public PendingKey(KeyNotification notification, long generation) { Notification = notification; Generation = generation; AdmissionGeneration = generation; - PendingSince = pendingSince; Node = new(this); } } - private readonly record struct PendingDispatch( - PendingKey Pending, long Generation, long PendingSince, KeyNotification Notification); + private sealed class Producer + { + public long Version; + public long CompletedVersion = long.MinValue; + public long InFlightVersion; + public ReceiptOutcome? Outcome; + public TaskCompletionSource? Receipt; + public TaskCompletionSource? InFlightReceipt; + } + + private sealed class Cohort(DisseminationMembershipSnapshot membership, long started, TimeSpan period) + { + public DisseminationMembershipSnapshot Membership { get; } = membership; + public long Started { get; } = started; + public TimeSpan Period { get; } = period; + public HashSet Contributors { get; } = []; + public bool HasPublications; + } - private readonly record struct BatchingSettings( - bool Enabled, - bool HighPriority, - int MaxPendingItemCount, - int MaxBatchItems, - TimeSpan CollectionDelay, - TimeSpan BroadcastInterval); + private readonly record struct ReceiptOutcome(bool Accepted, long Started, TimeSpan Period); + private readonly record struct PendingDispatch( + PendingKey Pending, long Generation, KeyNotification Notification, + TaskCompletionSource? Receipt, Producer? Producer, long ProducerVersion); + private readonly record struct Settings(bool Enabled, bool HighPriority, int MaxPendingItemCount, int MaxBatchItems, TimeSpan Period); - [LoggerMessage(Level = LogLevel.Warning, Message = "Dissemination root dispatch for {Namespace} failed and will retry at the root pacing cadence.")] + [LoggerMessage(Level = LogLevel.Warning, Message = "Dissemination root dispatch for {Namespace} failed. Receipts are rejected and retained hints will retry after the publication period.")] private static partial void LogDispatchFailed(ILogger logger, Exception exception, DisseminationNamespace @namespace); [LoggerMessage(Level = LogLevel.Debug, Message = "Dissemination root for {Namespace} rejected {Count} new keys at its pending-key limit of {Limit}.")] diff --git a/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs b/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs index 571742b8377..ff2818f8a0d 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs @@ -44,6 +44,14 @@ await this.RunOrQueueTask(token => _protocol.Publish( version, token).AsTask(), cancellationToken); + async ValueTask IDisseminationService.PublishAggregated( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken) => + await this.RunOrQueueTask(token => _protocol.PublishAggregated( + disseminationNamespace, key, version, token).AsTask(), cancellationToken); + IReadOnlyList IDisseminationService.GetUnconfirmedPeers( IDisseminationNamespace disseminationNamespace) => _protocol.GetUnconfirmedPeers(disseminationNamespace); @@ -53,6 +61,11 @@ Task IDisseminationSystemTarget.PushBroadcast( CancellationToken cancellationToken) => _protocol.ReceiveBroadcast(batch, cancellationToken); + Task IDisseminationSystemTarget.PublishAggregated( + DisseminationPublicationRequest request, + CancellationToken cancellationToken) => + _protocol.ReceivePublication(request, cancellationToken); + async Task IDisseminationSystemTarget.ExchangeAntiEntropy( DisseminationAntiEntropyRequest request, CancellationToken cancellationToken) => diff --git a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs index 34022041052..7041db3714e 100644 --- a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs @@ -12,6 +12,8 @@ internal interface IDisseminationNamespace DisseminationRoutingMode RoutingMode => DisseminationRoutingMode.BroadcastTree; + TimeSpan AggregationPeriod => Options.MaxCoalescingDelay; + // Full values can be authority-refresh hints even when their numeric version is older. bool ValidateOlderFullValues => false; diff --git a/src/Orleans.Runtime/Dissemination/IDisseminationService.cs b/src/Orleans.Runtime/Dissemination/IDisseminationService.cs index 19652a5768a..59b89290e28 100644 --- a/src/Orleans.Runtime/Dissemination/IDisseminationService.cs +++ b/src/Orleans.Runtime/Dissemination/IDisseminationService.cs @@ -9,5 +9,11 @@ ValueTask Publish( long version, CancellationToken cancellationToken); + ValueTask PublishAggregated( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken); + IReadOnlyList GetUnconfirmedPeers(IDisseminationNamespace disseminationNamespace); } diff --git a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs index f600b474d2a..37a5dd4a3ca 100644 --- a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs +++ b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs @@ -36,6 +36,8 @@ internal sealed partial class DeploymentLoadPublisher : SystemTarget, IDeploymen private long _lastUpdateDateTimeTicks; private IGrainTimer? _publishTimer; + private Task? _publicationTask; + private Task? _disseminationPublicationTask; public ConcurrentDictionary PeriodicStatistics => _periodicStats; @@ -84,7 +86,7 @@ private async Task StartAsync(CancellationToken cancellationToken) await this.RunOrQueueTask(() => { cancellationToken.ThrowIfCancellationRequested(); - _publishTimer = RegisterGrainTimer(PublishStatistics, randomTimerOffset, _statisticsRefreshTime); + _publishTimer = RegisterGrainTimer(PublishStatisticsOnTimer, randomTimerOffset, _statisticsRefreshTime); return Task.CompletedTask; }); } @@ -98,9 +100,40 @@ await this.RunOrQueueTask(async token => LogDebugStartedDeploymentLoadPublisher(_logger); } - internal async Task PublishStatistics(CancellationToken cancellationToken) + internal Task PublishStatistics(CancellationToken cancellationToken) => + PublishStatisticsAndGetReceipt(cancellationToken); + + private async Task PublishStatisticsOnTimer(CancellationToken cancellationToken) + { + var publication = await PublishStatisticsAndGetReceipt(cancellationToken); + if (publication.Receipt.Accepted && !cancellationToken.IsCancellationRequested) + { + // Change during a grain timer callback applies its due time after the callback + // completes. Account for direct delivery and notifications since receipt arrival, + // rather than adding another full period after the root's cohort wait. + // Startup and explicit publications must not change a paused/disposed timer. + var remaining = publication.Receipt.NextPublicationDelay + - publication.TimeProvider!.GetElapsedTime(publication.ReceiptTimestamp); + _publishTimer?.Change( + remaining > TimeSpan.Zero ? remaining : TimeSpan.Zero, + _statisticsRefreshTime); + } + } + + private async Task PublishStatisticsAndGetReceipt(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + // Startup and the interleaving timer can overlap. Join the pending sample instead of + // submitting another source publication while its cohort is still open. + var publication = _publicationTask is { IsCompleted: false } pending + ? pending + : _publicationTask = PublishStatisticsCore(cancellationToken); + return await publication.WaitAsync(cancellationToken); + } + + private async Task PublishStatisticsCore(CancellationToken cancellationToken) + { + StatisticsPublication publication = default; try { LogTracePublishStatistics(_logger); @@ -123,7 +156,8 @@ internal async Task PublishStatistics(CancellationToken cancellationToken) // Inform other cluster members about our refreshed statistics. var members = _siloStatusOracle.GetApproximateSiloStatuses(true).Keys.ToArray(); IReadOnlyCollection directRecipients = members; - if (await TryPublishStatisticsViaDissemination(myStats, cancellationToken)) + publication = await PublishStatisticsViaDissemination(myStats, cancellationToken); + if (publication.Receipt.Accepted) { try { @@ -156,12 +190,15 @@ internal async Task PublishStatistics(CancellationToken cancellationToken) } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { + cancellationToken.ThrowIfCancellationRequested(); throw; } catch (Exception exc) { LogWarningRuntimeStatisticsUpdateFailure2(_logger, exc); } + + return publication; } public Task UpdateRuntimeStatistics( @@ -194,50 +231,80 @@ internal Dictionary GetActiveSiloStatusesForStatisticsD _siloStatusOracle.GetApproximateSiloStatuses(onlyActive: true); internal async Task TryPublishStatisticsViaDissemination( + SiloRuntimeStatistics myStats, + CancellationToken cancellationToken) => + (await PublishStatisticsViaDissemination(myStats, cancellationToken)).Receipt.Accepted; + + private async Task PublishStatisticsViaDissemination( + SiloRuntimeStatistics myStats, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var publication = _disseminationPublicationTask is { IsCompleted: false } pending + ? pending + : _disseminationPublicationTask = PublishStatisticsViaDisseminationCore(myStats, cancellationToken); + return await publication.WaitAsync(cancellationToken); + } + + private async Task PublishStatisticsViaDisseminationCore( SiloRuntimeStatistics myStats, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); if (_statisticsRefreshTime <= TimeSpan.Zero) { - return false; + return default; } var disseminationNamespace = _serviceProvider.GetRequiredService(); if (!disseminationNamespace.Options.Enabled) { - return false; + return default; } var timeProvider = _serviceProvider.GetRequiredService(); - using var timeoutCancellation = new CancellationTokenSource(_statisticsRefreshTime, timeProvider); + // A cohort may intentionally stay open for P. Allow another P for transport and + // distribution admission; the receipt's delay, not this budget, controls sampling. + // For long periods, the ordinary RPC timeout may expire first and take the same + // logged direct-publication fallback without changing cluster-wide RPC settings. + var receiptTimeout = TimeSpan.FromMilliseconds(Math.Min(disseminationNamespace.AggregationPeriod.TotalMilliseconds * 2, uint.MaxValue - 1)); + using var timeoutCancellation = new CancellationTokenSource(receiptTimeout, timeProvider); using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellation.Token); try { var dissemination = _serviceProvider.GetRequiredService(); - return await dissemination.Publish( + var receipt = await dissemination.PublishAggregated( disseminationNamespace, _siloDetails.SiloAddress, myStats.DateTime.Ticks, cancellation.Token); + var receiptTimestamp = timeProvider.GetTimestamp(); + cancellationToken.ThrowIfCancellationRequested(); + return new(receipt, timeProvider, receiptTimestamp); } catch (OperationCanceledException) when ( timeoutCancellation.IsCancellationRequested && !cancellationToken.IsCancellationRequested) { - LogDebugRuntimeStatisticsDisseminationTimedOut(_logger, _statisticsRefreshTime); - return false; + LogDebugRuntimeStatisticsDisseminationTimedOut(_logger, receiptTimeout); + return default; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { + cancellationToken.ThrowIfCancellationRequested(); throw; } catch (Exception exception) { LogWarningRuntimeStatisticsUpdateFailure1(_logger, exception); - return false; + return default; } } + private readonly record struct StatisticsPublication( + DisseminationPublicationReceipt Receipt, + TimeProvider? TimeProvider, + long ReceiptTimestamp); + private async Task PublishStatisticsDirectly( SiloRuntimeStatistics myStats, IReadOnlyCollection members, diff --git a/src/api/Orleans.Core/Orleans.Core.cs b/src/api/Orleans.Core/Orleans.Core.cs index cb6702fe16b..62eec4aa7be 100644 --- a/src/api/Orleans.Core/Orleans.Core.cs +++ b/src/api/Orleans.Core/Orleans.Core.cs @@ -566,8 +566,6 @@ public sealed partial class DisseminationOptions public sealed partial class DisseminationOverlayOptions { - public int AggregationBroadcastsPerSecond { get { throw null; } set { } } - public int AggregationFanOutFactor { get { throw null; } set { } } public System.TimeSpan AntiEntropyInterval { get { throw null; } set { } } diff --git a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs index d35da8c3206..13ef9d5e62a 100644 --- a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs +++ b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs @@ -201,8 +201,7 @@ public static async Task Execute(IServiceProvider services, Comman } } -// System targets use the normal incoming-call filter pipeline. This filter blocks only tree -// application; native ExchangeAntiEntropy requests/responses continue over the real connection. +// Gate cohort ingress and tree application while native anti-entropy uses the real connection. internal sealed class DisseminationTreeGate : IIncomingGrainCallFilter { private readonly InFlightCallGate _gate = new(); @@ -215,7 +214,7 @@ internal sealed class DisseminationTreeGate : IIncomingGrainCallFilter public Task Invoke(IIncomingGrainCallContext context) => context.InterfaceMethod.DeclaringType == typeof(IDisseminationSystemTarget) - && context.InterfaceMethod.Name == nameof(IDisseminationSystemTarget.PushBroadcast) + && context.InterfaceMethod.Name is nameof(IDisseminationSystemTarget.PushBroadcast) or nameof(IDisseminationSystemTarget.PublishAggregated) ? _gate.Invoke(context.Invoke) : context.Invoke(); } diff --git a/test/Dissemination.IntegrationHarness/Silo/Program.cs b/test/Dissemination.IntegrationHarness/Silo/Program.cs index 7b9fdcaa5ff..f68de2fd5c5 100644 --- a/test/Dissemination.IntegrationHarness/Silo/Program.cs +++ b/test/Dissemination.IntegrationHarness/Silo/Program.cs @@ -76,7 +76,7 @@ public static async Task Main(string[] args) options.MaxJoinAttemptTime = TimeSpan.FromSeconds(60); }); services.Configure(options => - options.DeploymentLoadPublisherRefreshTime = TimeSpan.FromHours(1)); + options.DeploymentLoadPublisherRefreshTime = TimeSpan.FromSeconds(1)); services.Configure(options => options.ResponseTimeout = TimeSpan.FromSeconds(5)); services.Configure(options => options.ShutdownTimeout = TimeSpan.FromSeconds(15)); services.Configure(options => diff --git a/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs b/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs index 54d106bc486..77cac0e21dd 100644 --- a/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs +++ b/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs @@ -52,7 +52,7 @@ public async Task PublishStatistics_ExplicitlyEnabled_ConfirmedPeersUseDissemina await rig.Publisher.PublishStatistics(TestContext.Current.CancellationToken); - await rig.Dissemination.Received(1).Publish( + await rig.Dissemination.Received(1).PublishAggregated( Arg.Any(), rig.LocalSilo, rig.Publisher.LocalRuntimeStatistics.DateTime.Ticks, Arg.Any()); await rig.DirectTarget.Received(confirmed ? 0 : 1).UpdateRuntimeStatistics( @@ -65,9 +65,9 @@ public async Task PublishStatistics_DisseminationCancelsIndependently_FallsBackT using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: true); using var cancellation = new CancellationTokenSource(); cancellation.Cancel(); - rig.Dissemination.Publish( + rig.Dissemination.PublishAggregated( Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(_ => ValueTask.FromCanceled(cancellation.Token)); + .Returns(_ => ValueTask.FromCanceled(cancellation.Token)); await rig.Publisher.PublishStatistics(TestContext.Current.CancellationToken); @@ -80,12 +80,12 @@ public async Task PublishStatistics_CallerCancels_DoesNotFallBackToDirectPublica { using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: true); using var cancellation = new CancellationTokenSource(); - rig.Dissemination.Publish( + rig.Dissemination.PublishAggregated( Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(call => { cancellation.Cancel(); - return ValueTask.FromCanceled(call.ArgAt(3)); + return ValueTask.FromCanceled(call.ArgAt(3)); }); await Assert.ThrowsAnyAsync(() => rig.Publisher.PublishStatistics(cancellation.Token)); @@ -100,12 +100,15 @@ public async Task PublishStatistics_DeadlineCancelsNativePublicationBeforeDirect var timeProvider = (FakeTimeProvider)rig.ServiceProvider.GetRequiredService(); var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var completed = false; - rig.Dissemination.Publish( + rig.Dissemination.PublishAggregated( Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(call => new ValueTask(PublishAsync(call.ArgAt(3)))); + .Returns(call => new ValueTask(PublishAsync(call.ArgAt(3)))); var publication = rig.Publisher.PublishStatistics(TestContext.Current.CancellationToken); var token = await started.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + timeProvider.Advance(TimeSpan.FromSeconds(5)); + Assert.False(token.IsCancellationRequested); + Assert.False(publication.IsCompleted); timeProvider.Advance(TimeSpan.FromSeconds(5)); await publication.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); @@ -114,13 +117,13 @@ public async Task PublishStatistics_DeadlineCancelsNativePublicationBeforeDirect await rig.DirectTarget.Received(1).UpdateRuntimeStatistics( rig.LocalSilo, rig.Publisher.LocalRuntimeStatistics, TestContext.Current.CancellationToken); - async Task PublishAsync(CancellationToken cancellationToken) + async Task PublishAsync(CancellationToken cancellationToken) { started.SetResult(cancellationToken); try { await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); - return true; + return new(true, TimeSpan.FromSeconds(5)); } finally { @@ -458,9 +461,9 @@ private static TestRig CreateTestRig( control.GetRuntimeStatistics(Arg.Any()).Returns(Task.FromResult(initialStatistics)); grainFactory.GetSystemTarget(Constants.SiloControlType, Arg.Any()).Returns(control); var dissemination = Substitute.For(); - dissemination.Publish( + dissemination.PublishAggregated( Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(ValueTask.FromResult(true)); + .Returns(ValueTask.FromResult(new DisseminationPublicationReceipt(true, refreshTime))); var services = new ServiceCollection(); services.AddSerializer(); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs index ad4c770db63..f07fe64af26 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs @@ -27,12 +27,11 @@ public void DefaultsAreBoundedAndOptIn() var membership = new ClusterMembershipOptions(); Assert.Equal(8, options.Overlay.AggregationFanOutFactor); - Assert.Equal(5, options.Overlay.AggregationBroadcastsPerSecond); Assert.Equal(8192, options.Overlay.MaxAntiEntropyBatchItems); Assert.Equal(1048576, options.Overlay.MaxAntiEntropyBatchBytes); Assert.Equal(options.MaxBatchItems, options.Overlay.MaxAntiEntropyBatchItems); Assert.Equal(options.MaxBatchBytes, options.Overlay.MaxAntiEntropyBatchBytes); - Assert.Equal(TimeSpan.FromMilliseconds(25), load.Dissemination.MaxCoalescingDelay); + Assert.Equal(new DisseminationNamespaceOptions().MaxCoalescingDelay, load.Dissemination.MaxCoalescingDelay); Assert.Equal(8192, load.Dissemination.MaxPendingItemCount); Assert.Equal(TimeSpan.FromSeconds(5), load.Dissemination.ExpectedUpdateCadence); Assert.Equal(TimeSpan.FromSeconds(1), load.DeploymentLoadPublisherRefreshTime); @@ -46,13 +45,12 @@ public void DefaultsAreBoundedAndOptIn() } [Theory] - [InlineData(1, 1, 1, 1)] - [InlineData(int.MaxValue, 1000, int.MaxValue, int.MaxValue)] - public void ValidatorAcceptsPositiveBounds(int fanout, int broadcastsPerSecond, int items, int bytes) + [InlineData(1, 1, 1)] + [InlineData(int.MaxValue, int.MaxValue, int.MaxValue)] + public void ValidatorAcceptsPositiveBounds(int fanout, int items, int bytes) { var options = new DisseminationOptions(); options.Overlay.AggregationFanOutFactor = fanout; - options.Overlay.AggregationBroadcastsPerSecond = broadcastsPerSecond; options.Overlay.MaxAntiEntropyBatchItems = items; options.Overlay.MaxAntiEntropyBatchBytes = bytes; @@ -71,20 +69,6 @@ public void ValidatorRejectsNonPositiveAggregationFanOut(int value) AssertValidationFailure(options, "AggregationFanOutFactor must be greater than 0."); } - [Theory] - [InlineData(0)] - [InlineData(-1)] - [InlineData(int.MinValue)] - [InlineData(1001)] - [InlineData(int.MaxValue)] - public void ValidatorRejectsOutOfRangeAggregationBroadcastRate(int value) - { - var options = new DisseminationOptions(); - options.Overlay.AggregationBroadcastsPerSecond = value; - - AssertValidationFailure(options, "AggregationBroadcastsPerSecond must be between 1 and 1000."); - } - [Theory] [InlineData(0)] [InlineData(-1)] diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs new file mode 100644 index 00000000000..58e29faa146 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs @@ -0,0 +1,573 @@ +#nullable enable + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Reflection; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.Scheduler; +using Orleans.Serialization; +using Orleans.Serialization.Invocation; +using Orleans.Timers; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData(25)] + [InlineData(1000)] + public async Task CohortPublisherReceiptReschedulesActualTimer(int sealMilliseconds) + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + var tick = await harness.StartTickAsync(); + var first = await harness.Dissemination.NextPublicationAsync(); + var sealDelay = TimeSpan.FromMilliseconds(sealMilliseconds); + harness.Clock.Advance(sealDelay); + first.Receipt.SetResult(new(true, harness.Period - sealDelay)); + await tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(harness.Period - sealDelay, harness.TimerClock.DueTime); + if (sealDelay < harness.Period) + { + harness.Clock.Advance(harness.Period - sealDelay - TimeSpan.FromMilliseconds(1)); + Assert.Single(harness.Dissemination.Publications); + harness.Clock.Advance(TimeSpan.FromMilliseconds(1)); + } + + var second = await harness.Dissemination.NextPublicationAsync(); + Assert.Equal(harness.Period, harness.Clock.GetElapsedTime(harness.StartTimestamp)); + Assert.True(second.Version > first.Version); + Assert.Equal(second.Version, harness.Publisher.LocalRuntimeStatistics.DateTime.Ticks); + Assert.False(second.Token.IsCancellationRequested); + Assert.False(second.Receipt.Task.IsCompleted); + Assert.Equal(2, harness.Dissemination.Publications.Count); + Assert.Empty(harness.DirectUpdates); + } + + [Fact] + public async Task CohortPublisherReceiptSubtractsDirectDeliveryTime() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + harness.Dissemination.UnconfirmedPeers = [harness.Peer]; + var directStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseDirect = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.DirectHandler = token => + { + directStarted.TrySetResult(); + return releaseDirect.Task.WaitAsync(token); + }; + var tick = await harness.StartTickAsync(); + var publication = await harness.Dissemination.NextPublicationAsync(); + harness.Clock.Advance(TimeSpan.FromMilliseconds(25)); + publication.Receipt.SetResult(new(true, TimeSpan.FromMilliseconds(975))); + await directStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + harness.Clock.Advance(TimeSpan.FromMilliseconds(200)); + releaseDirect.SetResult(); + await tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(TimeSpan.FromMilliseconds(775), harness.TimerClock.DueTime); + Assert.Equal(publication.Version, Assert.Single(harness.DirectUpdates).DateTime.Ticks); + harness.Clock.Advance(TimeSpan.FromMilliseconds(775)); + var next = await harness.Dissemination.NextPublicationAsync(); + Assert.Equal(harness.Period, harness.Clock.GetElapsedTime(harness.StartTimestamp)); + Assert.True(next.Version > publication.Version); + } + + [Theory] + [InlineData("disabled")] + [InlineData("rejected")] + [InlineData("failed")] + public async Task CohortPublisherDirectFallbackRetainsNormalTimerCadence(string outcome) + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Options.Dissemination.Enabled = outcome != "disabled"; + harness.Dissemination.AutomaticReceipt = new(false, TimeSpan.Zero); + if (outcome == "failed") + { + harness.Dissemination.Failure = new InvalidOperationException("Receipt transport failed."); + } + + var directStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseDirect = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.DirectHandler = token => + { + directStarted.TrySetResult(); + return releaseDirect.Task.WaitAsync(token); + }; + var tick = await harness.StartTickAsync(); + await directStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + harness.Clock.Advance(TimeSpan.FromMilliseconds(200)); + releaseDirect.SetResult(); + await tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(harness.Period, harness.TimerClock.DueTime); + Assert.Single(harness.DirectUpdates); + Assert.Equal(outcome == "disabled" ? 0 : 1, harness.Dissemination.Publications.Count); + Assert.Equal(0, harness.Dissemination.QueryCount); + harness.Clock.Advance(harness.Period - TimeSpan.FromMilliseconds(1)); + Assert.Single(harness.DirectUpdates); + harness.Clock.Advance(TimeSpan.FromMilliseconds(1)); + var nextTick = await harness.Timers.NextTickAsync(); + await nextTick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(2, harness.DirectUpdates.Count); + Assert.Equal(TimeSpan.FromMilliseconds(1200), harness.Clock.GetElapsedTime(harness.StartTimestamp)); + } + + [Fact] + public async Task CohortPublisherReceiptBudgetAllowsOnePeriodAndExpiresAfterTwo() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + var tick = await harness.StartTickAsync(); + var publication = await harness.Dissemination.NextPublicationAsync(); + + harness.Clock.Advance(harness.Period); + await harness.OwnerBarrierAsync(); + Assert.False(publication.Token.IsCancellationRequested); + Assert.False(tick.IsCompleted); + Assert.Empty(harness.DirectUpdates); + + harness.Clock.Advance(harness.Period - TimeSpan.FromMilliseconds(1)); + Assert.False(publication.Token.IsCancellationRequested); + harness.Clock.Advance(TimeSpan.FromMilliseconds(1)); + await tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(publication.Token.IsCancellationRequested); + Assert.Single(harness.DirectUpdates); + Assert.Equal(harness.Period, harness.TimerClock.DueTime); + Assert.Equal(0, harness.Dissemination.QueryCount); + } + + [Fact] + public async Task CohortPublisherCallerCancellationPreservesIdentityWithoutFallback() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var publish = harness.Publisher.RunOrQueueTask( + async token => + { + await harness.Publisher.PublishStatistics(token); + return true; + }, + caller.Token); + var publication = await harness.Dissemination.NextPublicationAsync(); + caller.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + () => publish.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + await harness.OwnerBarrierAsync(); + + Assert.Equal(caller.Token, exception.CancellationToken); + Assert.True(publication.Token.IsCancellationRequested); + Assert.Empty(harness.DirectUpdates); + Assert.Equal(0, harness.Dissemination.QueryCount); + } + + [Fact] + public async Task CohortPublisherStartupReceiptWaitKeepsInboundOwnerApplicationLive() + { + await using var harness = new CohortPublisherHarness(); + harness.Dissemination.AutomaticReceipt = null; + var startup = harness.StartAsync(clearStartup: false); + var publication = await harness.Dissemination.NextPublicationAsync(); + var tick = await harness.StartTickAsync(); + await harness.OwnerBarrierAsync(); + + var incoming = CreatePhase5Statistics(42); + var applied = await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Peer, incoming, TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(DisseminationApplyResult.Applied, applied); + Assert.Same(incoming, harness.Publisher.PeriodicStatistics[harness.Peer]); + Assert.Single(harness.Dissemination.Publications); + Assert.Equal(publication.Version, harness.Publisher.LocalRuntimeStatistics.DateTime.Ticks); + Assert.False(startup.IsCompleted); + Assert.False(tick.IsCompleted); + + harness.Clock.Advance(TimeSpan.FromMilliseconds(25)); + publication.Receipt.SetResult(new(true, TimeSpan.FromMilliseconds(975))); + await Task.WhenAll(startup, tick).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Single(harness.Dissemination.Publications); + Assert.Equal(TimeSpan.FromMilliseconds(975), harness.TimerClock.DueTime); + Assert.Empty(harness.DirectUpdates); + } + + [Fact] + public async Task CohortPublisherOverlappingCallerCancellationDoesNotCancelPendingSample() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + var tick = await harness.StartTickAsync(); + var publication = await harness.Dissemination.NextPublicationAsync(); + using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var overlapping = harness.Publisher.RunOrQueueTask( + async token => + { + await harness.Publisher.PublishStatistics(token); + return true; + }, + caller.Token); + await harness.OwnerBarrierAsync(); + caller.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + () => overlapping.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + + Assert.Equal(caller.Token, exception.CancellationToken); + Assert.False(publication.Token.IsCancellationRequested); + Assert.Single(harness.Dissemination.Publications); + publication.Receipt.SetResult(new(true, harness.Period)); + await tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Empty(harness.DirectUpdates); + } + + [Fact] + public async Task CohortPublisherStopRetainsDisposedTimerAndCancelsReceipt() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + var tick = await harness.StartTickAsync(); + var publication = await harness.Dissemination.NextPublicationAsync(); + var timer = harness.Timers.Timer; + await harness.Lifecycle.OnStop(TestContext.Current.CancellationToken); + await Assert.ThrowsAnyAsync( + () => tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + await harness.OwnerBarrierAsync(); + harness.Clock.Advance(harness.Period * 3); + + Assert.True(publication.Token.IsCancellationRequested); + Assert.True(harness.TimerClock.Disposed); + Assert.Same(timer, typeof(DeploymentLoadPublisher).GetField( + "_publishTimer", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(harness.Publisher)); + Assert.Single(harness.Dissemination.Publications); + Assert.Empty(harness.DirectUpdates); + } + + [Fact] + public async Task CohortPublisherExplicitPublicationPreservesDisposedTimerPause() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + // Only timer registration changes the schedule: the explicit startup publication does not. + Assert.Equal(1, harness.TimerClock.ChangeCount); + var timer = harness.Timers.Timer; + var firstVersion = harness.Publisher.LocalRuntimeStatistics.DateTime.Ticks; + await harness.Lifecycle.OnStop(TestContext.Current.CancellationToken); + + await harness.Publisher.RunOrQueueTask( + async token => + { + await harness.Publisher.PublishStatistics(token); + return true; + }, + TestContext.Current.CancellationToken); + harness.Clock.Advance(harness.Period * 3); + + Assert.True(harness.Publisher.LocalRuntimeStatistics.DateTime.Ticks > firstVersion); + Assert.Single(harness.Dissemination.Publications); + Assert.Equal(1, harness.TimerClock.ChangeCount); + Assert.True(harness.TimerClock.Disposed); + Assert.Same(timer, typeof(DeploymentLoadPublisher).GetField( + "_publishTimer", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(harness.Publisher)); + Assert.Empty(harness.Timers.AllTicks); + Assert.Empty(harness.DirectUpdates); + } + + [Fact] + public async Task CohortPublisherCompatibilityWrapperPublishesOneReceipt() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + var result = await harness.Publisher.RunOrQueueTask( + token => harness.Publisher.TryPublishStatisticsViaDissemination( + harness.Publisher.LocalRuntimeStatistics, token), + TestContext.Current.CancellationToken); + + Assert.True(result); + Assert.Equal( + harness.Publisher.LocalRuntimeStatistics.DateTime.Ticks, + Assert.Single(harness.Dissemination.Publications).Version); + Assert.Empty(harness.DirectUpdates); + } + + private sealed class CohortPublisherHarness : IAsyncDisposable + { + private readonly ServiceProvider _serializerServices; + + public CohortPublisherHarness() + { + var local = CreateSilo(40501); + Peer = CreateSilo(40502); + var statusOracle = new Phase4FakeSiloStatusOracle(); + statusOracle.SetStatus(local, SiloStatus.Active); + statusOracle.SetStatus(Peer, SiloStatus.Active); + var details = new FakeLocalSiloDetails(local); + Options = new() { DeploymentLoadPublisherRefreshTime = Period }; + Options.Dissemination.Enabled = true; + _serializerServices = new ServiceCollection().AddSerializer().BuildServiceProvider(); + TimerClock = new(Clock); + Timers = new(new TimerRegistry( + NullLoggerFactory.Instance, + TimerClock, + new MessageFactory(_serializerServices.GetRequiredService(), NullLogger.Instance, null!), + details)); + var shared = new SystemTargetShared( + runtimeClient: null!, + details, + NullLoggerFactory.Instance, + Microsoft.Extensions.Options.Options.Create(new SchedulingOptions()), + grainReferenceActivator: null!, + Timers, + new ActivationDirectory(CreatePhase4Instruments()), + CreatePhase4Instruments(), + CreatePhase4Instruments(), + CreatePhase4Instruments(), + CreatePhase4Instruments()); + var grainFactory = Substitute.For(); + var siloControl = Substitute.For(); + siloControl.GetRuntimeStatistics(Arg.Any()).Returns(CreatePhase5Statistics(0)); + grainFactory.GetSystemTarget( + Constants.SiloControlType, Arg.Any()).Returns(siloControl); + var directTarget = Substitute.For(); + directTarget.UpdateRuntimeStatistics( + Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => + { + DirectUpdates.Enqueue(call.ArgAt(1)); + return DirectHandler(call.ArgAt(2)); + }); + grainFactory.GetSystemTarget( + Constants.DeploymentLoadPublisherSystemTargetType, Peer).Returns(directTarget); + var services = new Phase4MutableServiceProvider(); + services.Add(Clock); + Publisher = new( + details, + statusOracle, + Microsoft.Extensions.Options.Options.Create(Options), + grainFactory, + NullLoggerFactory.Instance, + shared.ActivationDirectory, + new Phase4FakeActivationWorkingSet(), + new Phase4FakeEnvironmentStatisticsProvider(), + Microsoft.Extensions.Options.Options.Create(new LoadSheddingOptions()), + services, + shared); + services.Add(Dissemination); + services.Add(new DeploymentLoadStatisticsDisseminationNamespace( + Publisher, + new TestOptionsMonitor(Options), + _serializerServices.GetRequiredService())); + var lifecycle = Substitute.For(); + lifecycle.Subscribe(Arg.Any(), Arg.Any(), Arg.Any()).Returns(call => + { + Lifecycle = call.ArgAt(2); + return Substitute.For(); + }); + ((ILifecycleParticipant)Publisher).Participate(lifecycle); + StartTimestamp = Clock.GetTimestamp(); + } + + public TimeSpan Period => TimeSpan.FromSeconds(1); + public long StartTimestamp { get; } + public SiloAddress Peer { get; } + public FakeTimeProvider Clock { get; } = new(); + public CohortPublisherTimerClock TimerClock { get; } + public CohortPublisherTimerRegistry Timers { get; } + public DeploymentLoadPublisherOptions Options { get; } + public DeploymentLoadPublisher Publisher { get; } + public CohortPublisherDisseminationService Dissemination { get; } = new(); + public ILifecycleObserver Lifecycle { get; private set; } = null!; + public ConcurrentQueue DirectUpdates { get; } = new(); + public Func DirectHandler { get; set; } = static _ => Task.CompletedTask; + + public async Task StartAsync(bool clearStartup = true) + { + await Lifecycle.OnStart(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (clearStartup) + { + Dissemination.Publications.Clear(); + _ = await Dissemination.NextPublicationAsync(); + Dissemination.QueryCount = 0; + DirectUpdates.Clear(); + } + } + + public async Task StartTickAsync() + { + await Publisher.RunOrQueueTask(() => + { + Timers.Timer.Change(TimeSpan.Zero, Period); + return Task.CompletedTask; + }); + return await Timers.NextTickAsync(); + } + + public Task OwnerBarrierAsync() => Publisher.QueueTask(static () => Task.CompletedTask); + + public async ValueTask DisposeAsync() + { + await Lifecycle.OnStop(CancellationToken.None); + foreach (var tick in Timers.AllTicks) + { + try + { + await tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + catch (OperationCanceledException) when (TimerClock.Disposed) + { + } + } + + await OwnerBarrierAsync(); + _serializerServices.Dispose(); + } + } + + private sealed class CohortPublisherDisseminationService : IDisseminationService + { + private readonly Channel _publications = Channel.CreateUnbounded(); + + public ConcurrentQueue Publications { get; } = new(); + public DisseminationPublicationReceipt? AutomaticReceipt { get; set; } = new(true, TimeSpan.FromSeconds(1)); + public Exception? Failure { get; set; } + public IReadOnlyList UnconfirmedPeers { get; set; } = []; + public int QueryCount { get; set; } + + public ValueTask Publish( + IDisseminationNamespace disseminationNamespace, DisseminationKey key, long version, CancellationToken cancellationToken) => + throw new InvalidOperationException("Load publication must use the receipt-returning path."); + + public ValueTask PublishAggregated( + IDisseminationNamespace disseminationNamespace, DisseminationKey key, long version, CancellationToken cancellationToken) + { + var publication = new CohortPublisherPublication(version, cancellationToken); + Publications.Enqueue(publication); + if (Failure is { } failure) + { + publication.Receipt.SetException(failure); + } + else if (AutomaticReceipt is { } receipt) + { + publication.Receipt.SetResult(receipt); + } + + var result = new ValueTask(publication.Receipt.Task.WaitAsync(cancellationToken)); + _publications.Writer.TryWrite(publication); + return result; + } + + public IReadOnlyList GetUnconfirmedPeers(IDisseminationNamespace disseminationNamespace) + { + QueryCount++; + return UnconfirmedPeers; + } + + public Task NextPublicationAsync() => + _publications.Reader.ReadAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + private sealed record CohortPublisherPublication(long Version, CancellationToken Token) + { + public TaskCompletionSource Receipt { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + // Only message delivery is replaced: timer firing, Change, cancellation and callback completion + // use the real GrainTimer, and its callback runs on the publisher's actual SystemTarget scheduler. + private sealed class CohortPublisherTimerRegistry(TimerRegistry inner) : ITimerRegistry + { + private readonly Channel _ticks = Channel.CreateUnbounded(); + + public IGrainTimer Timer { get; private set; } = null!; + public ConcurrentQueue AllTicks { get; } = new(); + + public IGrainTimer RegisterGrainTimer( + IGrainContext grainContext, Func callback, TState state, GrainTimerCreationOptions options) + { + Assert.True(options.Interleave); + var context = Substitute.For(); + context.GrainId.Returns(grainContext.GrainId); + context.When(value => value.ReceiveMessage(Arg.Any())).Do(call => + { + var message = Assert.IsType(call.Arg()); + var invocation = Assert.IsAssignableFrom(message.BodyObject); + var tick = grainContext.QueueTask(async () => + { + using var response = await invocation.Invoke(); + if (response.Exception is { } exception) + { + ExceptionDispatchInfo.Capture(exception).Throw(); + } + }); + AllTicks.Enqueue(tick); + _ticks.Writer.TryWrite(tick); + }); + // Replace startup jitter with a deterministic offset; keep the runtime timer itself. + return Timer = inner.RegisterGrainTimer(context, callback, state, options with { DueTime = options.Period }); + } + + public IDisposable RegisterTimer(IGrainContext grainContext, Func callback, object? state, TimeSpan dueTime, TimeSpan period) => + throw new NotSupportedException(); + + public Task NextTickAsync() => + _ticks.Reader.ReadAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + private sealed class CohortPublisherTimerClock(FakeTimeProvider inner) : TimeProvider + { + public TimeSpan DueTime { get; private set; } + public int ChangeCount { get; private set; } + public bool Disposed { get; private set; } + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => + new RecordingTimer(this, inner.CreateTimer(callback, state, dueTime, period)); + + private sealed class RecordingTimer(CohortPublisherTimerClock owner, ITimer innerTimer) : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) + { + owner.DueTime = dueTime; + owner.ChangeCount++; + return innerTimer.Change(dueTime, period); + } + + public void Dispose() + { + owner.Disposed = true; + innerTimer.Dispose(); + } + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Cohorts.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Cohorts.cs new file mode 100644 index 00000000000..10c107d59e6 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Cohorts.cs @@ -0,0 +1,366 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task HeldCohortReceiptKeepsMembershipPeerAdmissionAvailable() + { + var cancellationToken = TestContext.Current.CancellationToken; + var peer = CreateSilo(41301); + var local = CreateSilo(41302); + var load = CreateCohortNamespace(local); + var membership = new FakeNamespace(local, new DisseminationNamespace("urgent-membership")); + membership.Options.Priority = DisseminationPriority.High; + var transport = new FakeTransport(local, peer); + var ingressStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var heldReceipt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var membershipSent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.PublishAggregatedHandler = (_, _, _) => + { + ingressStarted.TrySetResult(); + return heldReceipt.Task; + }; + transport.SendBroadcastResponseHandler = (target, batch, _) => + { + Assert.Equal(peer, target); + Assert.Equal(membership.Name, Assert.Single(batch.Values.Keys)); + membershipSent.TrySetResult(); + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var protocol = CreateProtocol(transport, [load, membership], options => options.MaxConcurrentSends = 1); + try + { + load.SetValue(local, 1); + var publication = protocol.PublishAggregated(load, local, 1, cancellationToken).AsTask(); + await ingressStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + membership.SetValue("membership", 1); + Assert.True(await protocol.Publish(membership, "membership", 1, cancellationToken)); + await membershipSent.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.False(publication.IsCompleted); + heldReceipt.SetResult(new(true, TimeSpan.FromMilliseconds(750))); + Assert.Equal(new DisseminationPublicationReceipt(true, TimeSpan.FromMilliseconds(750)), await publication); + } + finally + { + heldReceipt.TrySetResult(default); + load.Options.Enabled = false; + membership.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } + + [Theory] + [InlineData(4)] + [InlineData(10)] + public async Task CompleteCohortIncludesRootAndUsesOneIngressAndDistributionTree(int count) + { + var token = TestContext.Current.CancellationToken; + var members = CreateSilos(count); + var clock = new FakeTimeProvider(); + var startedAt = clock.GetTimestamp(); + var namespaces = members.Select(CreateCohortNamespace).ToArray(); + var transports = members.Select(local => new FakeTransport(local, members.Where(peer => !peer.Equals(local)).ToArray())).ToArray(); + var protocols = transports.Select((transport, index) => CreateProtocol( + transport, namespaces[index], options => options.Overlay.AggregationFanOutFactor = 2, clock)).ToArray(); + for (var index = 0; index < count; index++) + { + transports[index].PublishAggregatedHandler = (peer, request, cancellationToken) => + { + Assert.Equal(members[0], peer); + return protocols[0].ReceivePublication(request, cancellationToken); + }; + var source = transports[index]; + source.SendBroadcastResponseHandler = async (peer, batch, cancellationToken) => + { + lock (source.BroadcastBatches) + { + source.BroadcastBatches.Add((peer, batch)); + } + return await protocols[Array.IndexOf(members, peer)].ReceiveBroadcast(batch, cancellationToken); + }; + namespaces[index].SetValue(members[index], 1); + } + + try + { + var remoteReceipts = Enumerable.Range(1, count - 1) + .Select(index => protocols[index].PublishAggregated(namespaces[index], members[index], 1, token).AsTask()) + .ToArray(); + Assert.All(remoteReceipts, receipt => Assert.False(receipt.IsCompleted)); + var localReceipt = protocols[0].PublishAggregated(namespaces[0], members[0], 1, token).AsTask(); + var receipts = await Task.WhenAll(remoteReceipts.Append(localReceipt)).WaitAsync(TimeSpan.FromSeconds(5), token); + Assert.All(receipts, receipt => + { + Assert.True(receipt.Accepted); + Assert.Equal(TimeSpan.FromSeconds(1), receipt.NextPublicationDelay); + }); + for (var index = 0; index < count; index++) + { + await protocols[index].FlushPendingBroadcast(token); + } + + Assert.Equal(count - 1, transports.Sum(transport => transport.PublicationRequests.Count)); + Assert.Equal(count - 1, transports.Sum(transport => transport.BroadcastBatches.Count)); + Assert.All(namespaces, ns => Assert.All(members, member => Assert.Equal(1, ns.GetVersion(member)))); + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(startedAt)); + } + finally + { + foreach (var ns in namespaces) + { + ns.Options.Enabled = false; + } + await Task.WhenAll(protocols.Select(protocol => protocol.StopAsync(token))); + } + } + + [Fact] + public async Task StoppingRootSealsHeldIngressBeforeDrainingProtocolAdmissions() + { + var token = TestContext.Current.CancellationToken; + var local = CreateSilo(41401); + var peer = CreateSilo(41402); + var ns = CreateCohortNamespace(local); + var clock = new FakeTimeProvider(); + var startedAt = clock.GetTimestamp(); + var protocol = CreateProtocol(new FakeTransport(local, peer), ns, timeProvider: clock); + try + { + var request = new DisseminationPublicationRequest { Sender = peer, Namespace = ns.Name, Value = ns.CreateItem(peer, peer, 1) }; + var receipt = protocol.ReceivePublication(request, token); + Assert.False(receipt.IsCompleted); + Assert.Equal(1, ns.GetVersion(peer)); + await protocol.StopAsync(token).WaitAsync(TimeSpan.FromSeconds(5), token); + Assert.True((await receipt).Accepted); + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(startedAt)); + } + finally + { + await protocol.StopAsync(token); + } + } + + [Fact] + public async Task CanceledPublicationReleasesLocalAttemptBeforeNonCooperativeRpcReturns() + { + var token = TestContext.Current.CancellationToken; + var root = CreateSilo(41411); + var local = CreateSilo(41412); + var ns = CreateCohortNamespace(local); + var transport = new FakeTransport(local, root); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var late = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.PublishAggregatedHandler = (_, request, _) => + { + started.TrySetResult(); + return request.Value.Value.ToVersion == 1 ? late.Task : Task.FromResult(new DisseminationPublicationReceipt(true, TimeSpan.FromSeconds(1))); + }; + var protocol = CreateProtocol(transport, ns); + try + { + using var cancellation = new CancellationTokenSource(); + ns.SetValue(local, 1); + var first = protocol.PublishAggregated(ns, local, 1, cancellation.Token).AsTask(); + await started.Task.WaitAsync(TimeSpan.FromSeconds(5), token); + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync(() => first); + Assert.Equal(cancellation.Token, exception.CancellationToken); + ns.SetValue(local, 2); + Assert.True((await protocol.PublishAggregated(ns, local, 2, token)).Accepted); + Assert.Equal(2, transport.PublicationRequests.Count); + Assert.False(late.Task.IsCompleted); + } + finally + { + late.TrySetResult(default); + await protocol.StopAsync(token); + } + } + + [Fact] + public async Task PublicationToFormerRootRejectsBeforeOwnerMutation() + { + var token = TestContext.Current.CancellationToken; + var currentRoot = CreateSilo(41431); + var local = CreateSilo(41432); + var producer = CreateSilo(41433); + var ns = CreateCohortNamespace(local); + var transport = new FakeTransport(local, currentRoot, producer); + var protocol = CreateProtocol(transport, ns); + try + { + var receipt = await protocol.ReceivePublication(new() + { + Sender = producer, + Namespace = ns.Name, + Value = ns.CreateItem(producer, producer, 1), + }, token); + Assert.False(receipt.Accepted); + Assert.Equal(0, ns.GetVersion(producer)); + Assert.Empty(ns.ApplyCounts); + Assert.Empty(transport.BroadcastBatches); + } + finally + { + await protocol.StopAsync(token); + } + } + + [Fact] + public async Task ShutdownCancelsHeldOutgoingReceiptAndDrainsItsAdmission() + { + var token = TestContext.Current.CancellationToken; + var root = CreateSilo(41441); + var local = CreateSilo(41442); + var ns = CreateCohortNamespace(local); + var transport = new FakeTransport(local, root); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var late = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.PublishAggregatedHandler = (_, _, _) => + { + started.TrySetResult(); + return late.Task; + }; + var protocol = CreateProtocol(transport, ns); + try + { + ns.SetValue(local, 1); + var publication = protocol.PublishAggregated(ns, local, 1, token).AsTask(); + await started.Task.WaitAsync(TimeSpan.FromSeconds(5), token); + await protocol.StopAsync(token).WaitAsync(TimeSpan.FromSeconds(5), token); + await Assert.ThrowsAnyAsync(() => publication); + Assert.False(late.Task.IsCompleted); + Assert.False((await protocol.PublishAggregated(ns, local, 1, token)).Accepted); + } + finally + { + late.TrySetResult(default); + await protocol.StopAsync(token); + } + } + + [Fact] + public void CohortPublicationWireContractPreservesPayloadAndSchedulingReceipt() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var producer = CreateSilo(41421); + var ns = CreateCohortNamespace(producer); + var request = new DisseminationPublicationRequest { Sender = producer, Namespace = ns.Name, Value = ns.CreateItem(producer, producer, 7) }; + var decoded = Assert.IsType(serializer.Deserialize(serializer.SerializeToArray(request))); + Assert.Equal(request.Sender, decoded.Sender); + Assert.Equal(request.Namespace, decoded.Namespace); + Assert.Equal(request.Value.Value.Key, decoded.Value.Value.Key); + Assert.Equal(7, decoded.Value.Value.ToVersion); + Assert.Equal(request.Value.Value.Payload.ToArray(), decoded.Value.Value.Payload.ToArray()); + Assert.Equal(request.Value.TimeToLive, decoded.Value.TimeToLive); + var receipt = new DisseminationPublicationReceipt(true, TimeSpan.FromMilliseconds(975)); + Assert.Equal(receipt, serializer.Deserialize(serializer.SerializeToArray(receipt))); + } + + private static FakeNamespace CreateCohortNamespace(SiloAddress local) + { + var result = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + result.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + return result; + } + + [Fact] + public async Task RootPendingStateHandsOffImmediatelyWhenTheRootChanges() + { + var cancellationToken = TestContext.Current.CancellationToken; + var newRoot = CreateSilo(41311); + var local = CreateSilo(41312); + var peer = CreateSilo(41313); + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(25); + var transport = new FakeTransport(local, peer); + var handedOff = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = (target, batch, _) => + { + Assert.Equal(newRoot, target); + if (GetBroadcastValues(batch).Any(value => value.Value.Key == new DisseminationKey("pending"))) + { + handedOff.TrySetResult(); + } + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var protocol = CreateProtocol(transport, ns, timeProvider: clock); + var start = clock.GetTimestamp(); + try + { + ns.SetValue("pending", 1); + Assert.True(await protocol.Publish(ns, "pending", 1, cancellationToken)); + transport.Peers.Add(newRoot); + ns.SetValue("new", 1); + Assert.True(await protocol.Publish(ns, "new", 1, cancellationToken)); + clock.Advance(TimeSpan.Zero); + await handedOff.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(start)); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task RootAcceptsAndDistributesACompleteTwoThousandSiloInventory() + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(2000); + var local = members[0]; + var ns = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + var defaults = new DeploymentLoadPublisherOptions().Dissemination; + ns.Options.MaxPendingItemCount = defaults.MaxPendingItemCount; + ns.Options.MaxCoalescingDelay = defaults.MaxCoalescingDelay; + var transport = new FakeTransport(local, members[1..]); + var protocol = CreateProtocol(transport, ns, timeProvider: new FakeTimeProvider()); + try + { + var values = members.Select(member => ns.CreateItem(member, member, 1)).ToArray(); + var response = await protocol.ReceiveBroadcast(new() + { + Sender = members[1], + Values = CreateValueGroups(values), + }, cancellationToken); + Assert.Equal(members.Length, response.Acknowledgments[ns.Name].Count); + await protocol.FlushPendingBroadcast(cancellationToken); + + Assert.Equal(8, transport.BroadcastBatches.Count); + Assert.Equal(members[1..9], transport.BroadcastBatches.Select(batch => batch.Peer).Order()); + Assert.All(transport.BroadcastBatches, batch => + { + Assert.Equal(members.Length, GetBroadcastValues(batch.Batch).Count()); + Assert.All(GetBroadcastValues(batch.Batch), value => Assert.Equal(1, value.Value.ToVersion)); + }); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.RootPacing.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.RootPacing.cs deleted file mode 100644 index af9b6ac5075..00000000000 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.RootPacing.cs +++ /dev/null @@ -1,170 +0,0 @@ -using Microsoft.Extensions.Time.Testing; -using Orleans.Configuration; -using Orleans.Runtime.Dissemination; -using Xunit; - -namespace UnitTests.Dissemination; - -public partial class DisseminationProtocolTests -{ - [Fact] - public async Task MembershipTrafficCannotFlushUnadmittedRootLoadUpdates() - { - var cancellationToken = TestContext.Current.CancellationToken; - var local = CreateSilo(41301); - var peer = CreateSilo(41302); - var clock = new FakeTimeProvider(); - var load = new FakeNamespace(local, new DisseminationNamespace("paced-load")) - { - RoutingMode = DisseminationRoutingMode.AggregationTree, - MembershipScope = DisseminationMembershipScope.ActiveMembers, - }; - load.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(25); - var membership = new FakeNamespace(local, new DisseminationNamespace("urgent-membership")); - membership.Options.Priority = DisseminationPriority.High; - var transport = new FakeTransport(local, peer); - var firstLoad = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var secondLoad = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var firstMembership = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var secondMembership = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - transport.SendBroadcastResponseHandler = (target, batch, _) => - { - lock (transport.BroadcastBatches) - { - transport.BroadcastBatches.Add((target, batch)); - } - foreach (var value in GetBroadcastValues(batch)) - { - if (value.Value.Key == new DisseminationKey("load")) - { - (value.Value.ToVersion == 1 ? firstLoad : secondLoad).TrySetResult(); - } - else - { - (value.Value.ToVersion == 1 ? firstMembership : secondMembership).TrySetResult(); - } - } - return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); - }; - var protocol = CreateProtocol(transport, [load, membership], timeProvider: clock); - try - { - load.SetValue("load", 1); - Assert.True(await protocol.Publish(load, "load", 1, cancellationToken)); - membership.SetValue("membership", 1); - Assert.True(await protocol.Publish(membership, "membership", 1, cancellationToken)); - clock.Advance(TimeSpan.Zero); - await firstMembership.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - Assert.False(firstLoad.Task.IsCompleted); - clock.Advance(TimeSpan.FromMilliseconds(24)); - Assert.False(firstLoad.Task.IsCompleted); - clock.Advance(TimeSpan.FromMilliseconds(1)); - await firstLoad.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - - load.SetValue("load", 2); - Assert.True(await protocol.Publish(load, "load", 2, cancellationToken)); - membership.SetValue("membership", 2); - Assert.True(await protocol.Publish(membership, "membership", 2, cancellationToken)); - clock.Advance(TimeSpan.Zero); - await secondMembership.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - Assert.False(secondLoad.Task.IsCompleted); - clock.Advance(TimeSpan.FromMilliseconds(199)); - Assert.False(secondLoad.Task.IsCompleted); - clock.Advance(TimeSpan.FromMilliseconds(1)); - await secondLoad.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - } - finally - { - load.Options.Enabled = false; - membership.Options.Enabled = false; - await protocol.StopAsync(cancellationToken); - } - } - - [Fact] - public async Task RootPendingStateHandsOffImmediatelyWhenTheRootChanges() - { - var cancellationToken = TestContext.Current.CancellationToken; - var newRoot = CreateSilo(41311); - var local = CreateSilo(41312); - var peer = CreateSilo(41313); - var clock = new FakeTimeProvider(); - var ns = new FakeNamespace(local) - { - RoutingMode = DisseminationRoutingMode.AggregationTree, - MembershipScope = DisseminationMembershipScope.ActiveMembers, - }; - ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(25); - var transport = new FakeTransport(local, peer); - var handedOff = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - transport.SendBroadcastResponseHandler = (target, batch, _) => - { - Assert.Equal(newRoot, target); - if (GetBroadcastValues(batch).Any(value => value.Value.Key == new DisseminationKey("pending"))) - { - handedOff.TrySetResult(); - } - return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); - }; - var protocol = CreateProtocol(transport, ns, timeProvider: clock); - var start = clock.GetTimestamp(); - try - { - ns.SetValue("pending", 1); - Assert.True(await protocol.Publish(ns, "pending", 1, cancellationToken)); - transport.Peers.Add(newRoot); - ns.SetValue("new", 1); - Assert.True(await protocol.Publish(ns, "new", 1, cancellationToken)); - clock.Advance(TimeSpan.Zero); - await handedOff.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(start)); - } - finally - { - ns.Options.Enabled = false; - await protocol.StopAsync(cancellationToken); - } - } - - [Fact] - public async Task RootAcceptsAndDistributesACompleteTwoThousandSiloInventory() - { - var cancellationToken = TestContext.Current.CancellationToken; - var members = CreateSilos(2000); - var local = members[0]; - var ns = new FakeNamespace(local) - { - RoutingMode = DisseminationRoutingMode.AggregationTree, - MembershipScope = DisseminationMembershipScope.ActiveMembers, - }; - var defaults = new DeploymentLoadPublisherOptions().Dissemination; - ns.Options.MaxPendingItemCount = defaults.MaxPendingItemCount; - ns.Options.MaxCoalescingDelay = defaults.MaxCoalescingDelay; - var transport = new FakeTransport(local, members[1..]); - var protocol = CreateProtocol(transport, ns, timeProvider: new FakeTimeProvider()); - try - { - var values = members.Select(member => ns.CreateItem(member, member, 1)).ToArray(); - var response = await protocol.ReceiveBroadcast(new() - { - Sender = members[1], - Values = CreateValueGroups(values), - }, cancellationToken); - Assert.Equal(members.Length, response.Acknowledgments[ns.Name].Count); - await protocol.FlushPendingBroadcast(cancellationToken); - - Assert.Equal(8, transport.BroadcastBatches.Count); - Assert.Equal(members[1..9], transport.BroadcastBatches.Select(batch => batch.Peer).Order()); - Assert.All(transport.BroadcastBatches, batch => - { - Assert.Equal(members.Length, GetBroadcastValues(batch.Batch).Count()); - Assert.All(GetBroadcastValues(batch.Batch), value => Assert.Equal(1, value.Value.ToVersion)); - }); - } - finally - { - ns.Options.Enabled = false; - await protocol.StopAsync(cancellationToken); - } - } -} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index a582d230599..582459bee56 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -75,6 +75,12 @@ public void PushBroadcastUsesRequestResponseSemantics() method = typeof(IDisseminationSystemTarget).GetMethod(nameof(IDisseminationSystemTarget.ExchangeAntiEntropy)); Assert.NotNull(method); Assert.Equal("EF58CB3D", method.GetCustomAttribute()?.Alias); + + method = typeof(IDisseminationSystemTarget).GetMethod(nameof(IDisseminationSystemTarget.PublishAggregated)); + Assert.NotNull(method); + Assert.False(method.IsDefined(typeof(Orleans.Concurrency.OneWayAttribute), inherit: false)); + Assert.Equal("8C810F59", method.GetCustomAttribute()?.Alias); + Assert.Equal(typeof(Task), method.ReturnType); } [Fact] @@ -3198,7 +3204,7 @@ public void OptionsValidatorRejectsInvalidExpectedUpdateCadence() public void NamespaceOptionsUseExpectedUpdateCadenceDefaults() { Assert.Equal(TimeSpan.FromSeconds(5), new DeploymentLoadPublisherOptions().Dissemination.ExpectedUpdateCadence); - Assert.Equal(TimeSpan.FromMilliseconds(25), new DeploymentLoadPublisherOptions().Dissemination.MaxCoalescingDelay); + Assert.Equal(TimeSpan.FromMilliseconds(100), new DeploymentLoadPublisherOptions().Dissemination.MaxCoalescingDelay); Assert.Equal(8192, new DeploymentLoadPublisherOptions().Dissemination.MaxPendingItemCount); Assert.Equal(TimeSpan.FromSeconds(10), new ClusterMembershipOptions().Dissemination.ExpectedUpdateCadence); } @@ -4211,6 +4217,10 @@ private sealed class FakeDisseminationService : IDisseminationService public Func>? PublishHandler { get; set; } + public async ValueTask PublishAggregated( + IDisseminationNamespace disseminationNamespace, DisseminationKey key, long version, CancellationToken cancellationToken) => + new(await Publish(disseminationNamespace, key, version, cancellationToken), disseminationNamespace.AggregationPeriod); + public async ValueTask Publish( IDisseminationNamespace disseminationNamespace, DisseminationKey key, @@ -4270,6 +4280,11 @@ public FakeTransport(SiloAddress localSilo, params SiloAddress[] peers) public List<(SiloAddress Peer, DisseminationAntiEntropyRequest Request)> AntiEntropyRequests { get; } = new(); + public List<(SiloAddress Peer, DisseminationPublicationRequest Request)> PublicationRequests { get; } = []; + + public Func> PublishAggregatedHandler { get; set; } = + static (_, _, _) => Task.FromResult(new DisseminationPublicationReceipt(true, TimeSpan.FromSeconds(1))); + public IInternalGrainFactory GrainFactory { get; } public FakeMembershipManager MembershipManager { get; } @@ -4421,6 +4436,17 @@ private static long ComputeMembershipVersion(IEnumerable entrie private sealed class FakeDisseminationSystemTarget(SiloAddress peer, FakeTransport transport) : IDisseminationSystemTarget { + public Task PublishAggregated( + DisseminationPublicationRequest request, CancellationToken cancellationToken) + { + lock (transport.PublicationRequests) + { + transport.PublicationRequests.Add((peer, request)); + } + + return transport.PublishAggregatedHandler(peer, request, cancellationToken); + } + public Task PushBroadcast( DisseminationBroadcastBatch batch, CancellationToken cancellationToken) => @@ -6165,6 +6191,8 @@ await harness.Dissemination.PublishStarted.Task.WaitAsync( TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); timeProvider.Advance(harness.RefreshTime); + Assert.False(publish.IsCompleted); + timeProvider.Advance(harness.RefreshTime); await publish.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); AssertDirectRecipients(harness, harness.ActiveOne, harness.ActiveTwo); @@ -6370,6 +6398,23 @@ private sealed class Phase4FakeDisseminationService : IDisseminationService public TaskCompletionSource PublishStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Func>? AggregatedPublishHandler { get; set; } + + public async ValueTask PublishAggregated( + IDisseminationNamespace disseminationNamespace, DisseminationKey key, long version, CancellationToken cancellationToken) + { + if (AggregatedPublishHandler is null) + { + return new(await Publish(disseminationNamespace, key, version, cancellationToken), disseminationNamespace.AggregationPeriod); + } + + PublishCalls.Add((disseminationNamespace, key, version)); + PublishStarted.TrySetResult(); + var result = await AggregatedPublishHandler(disseminationNamespace, key, version, cancellationToken); + PublishResults.Add(result.Accepted); + return result; + } + public Func> PublishHandler { get; set; } = static (_, _, _, _) => ValueTask.FromResult(true); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs index 495ecc2a3fa..9429c657d26 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs @@ -1,8 +1,13 @@ using System.Collections.Concurrent; +using System.Collections.Immutable; +using System.Net; using System.Threading.Channels; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans; using Orleans.Configuration; using Orleans.Runtime; using Orleans.Runtime.Dissemination; @@ -18,133 +23,329 @@ namespace UnitTests.Dissemination; public class DisseminationRootBatcherTests { private static CancellationToken TestToken => TestContext.Current.CancellationToken; + private static TimeSpan Period => TimeSpan.FromSeconds(1); private static TimeSpan Milliseconds(int value) => TimeSpan.FromMilliseconds(value); [Fact] - public async Task IdleRootCollectsFor25MillisecondsAndReturnsToIdle() + public async Task SingleRootContributionSealsWithoutArmingCollectionTimer() { - await using var rig = new TestRig(); - rig.Clock.Advance(TimeSpan.FromHours(1)); - var start = rig.Clock.GetTimestamp(); + await using var rig = new TestRig(1); + Assert.Equal(new(true, Period), await Phase(rig.Publish(0), "single-silo cohort includes the local root")); + Assert.Equal(rig.Notification(0), Assert.Single(Assert.Single(rig.Attempts).Notifications)); + Assert.Equal(0, rig.Clock.ScheduleCount); + } - Assert.Equal(Milliseconds(25), await rig.NotifyAsync(new KeyNotification("a", 1, true))); - rig.Clock.Advance(Milliseconds(24)); + [Fact] + public async Task AllActiveIncarnationsIncludingRootSealEarlyAtOriginalNextPeriod() + { + await using var rig = new TestRig(); + var first = await rig.StartPublicationAsync(1); + rig.Clock.Advance(Milliseconds(10)); + var second = rig.Publish(2); + rig.Clock.Advance(Milliseconds(15)); + Assert.False(first.IsCompleted); + Assert.False(second.IsCompleted); Assert.Empty(rig.Attempts); - var attempt = await rig.AdvanceToDispatchAsync(Milliseconds(1)); - Assert.Equal(new KeyNotification("a", 1, true), Assert.Single(attempt.Notifications)); - Assert.Equal(Milliseconds(25), rig.Clock.GetElapsedTime(start, attempt.Timestamp)); - rig.Clock.Advance(TimeSpan.FromHours(1)); - Assert.Single(rig.Attempts); - Assert.Equal(1, rig.Clock.TimerCount); + var local = rig.Publish(0); + var receipts = await Phase(Task.WhenAll(first, second, local), "all three producer receipts"); + + Assert.All(receipts, receipt => Assert.Equal(new(true, Milliseconds(975)), receipt)); + Assert.Equal( + new KeyNotification[] { rig.Notification(1), rig.Notification(2), rig.Notification(0) }, + Assert.Single(rig.Attempts).Notifications); + Assert.Equal(1, rig.Clock.ScheduleCount); Assert.Empty(rig.Namespace.VersionReads); } [Fact] - public async Task SustainedUpdatesDispatchAt200MillisecondIntervals() + public async Task MissingProducerTimesOutAtFirstPendingDeadlineDespiteUpdates() { await using var rig = new TestRig(); - Assert.Equal(Milliseconds(25), await rig.NotifyAsync(new KeyNotification("a", 1, false))); - var previous = await rig.AdvanceToDispatchAsync(Milliseconds(25)); + var first = await rig.StartPublicationAsync(0); + rig.Clock.Advance(Milliseconds(400)); + var newer = rig.Publish(0, 9); + var peer = rig.Publish(1); + rig.Clock.Advance(Milliseconds(599)); + Assert.False(first.IsCompleted); + Assert.False(newer.IsCompleted); + Assert.False(peer.IsCompleted); + Assert.Empty(rig.Attempts); + + rig.Clock.Advance(Milliseconds(1)); + var receipts = await Phase(Task.WhenAll(first, newer, peer), "partial cohort at its original deadline"); + + Assert.All(receipts, receipt => Assert.Equal(new(true, TimeSpan.Zero), receipt)); + Assert.Equal(new[] { rig.Notification(0, 9), rig.Notification(1) }, Assert.Single(rig.Attempts).Notifications); + Assert.Equal(1, rig.Clock.ScheduleCount); + } - for (var version = 2; version <= 5; version++) + [Fact] + public async Task ReceiptDelaySubtractsDispatchTimeInsteadOfStartingAnotherPeriod() + { + await using var rig = new TestRig(2); + using var release = new ManualResetEventSlim(); + var token = TestToken; + rig.OnDispatch = _ => + { + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Receipt dispatch was not released."); + return true; + }; + var first = await rig.StartPublicationAsync(0); + rig.Clock.Advance(Milliseconds(25)); + var second = rig.Publish(1); + try { - Assert.Equal(Milliseconds(200), await rig.NotifyAsync(new KeyNotification("a", version, false))); - rig.Clock.Advance(Milliseconds(199)); - Assert.Equal(version - 1, rig.Attempts.Count); - var current = await rig.AdvanceToDispatchAsync(Milliseconds(1)); - Assert.Equal(Milliseconds(200), rig.Clock.GetElapsedTime(previous.Timestamp, current.Timestamp)); - Assert.Equal(version, Assert.Single(current.Notifications).Version); - previous = current; + await rig.NextAttemptAsync(); + rig.Clock.Advance(Milliseconds(75)); + Assert.False(first.IsCompleted); + Assert.False(second.IsCompleted); } + finally + { + release.Set(); + } + + Assert.All(await Phase(Task.WhenAll(first, second), "dispatch time deducted from receipt delay"), + receipt => Assert.Equal(new(true, Milliseconds(900)), receipt)); } [Fact] - public async Task CollectionWindowCanOutliveLastDispatchInterval() + public async Task InventoryHintsAndDuplicateVersionsNeverFillMissingContributions() { await using var rig = new TestRig(); - await rig.NotifyAsync(new KeyNotification("a", 1, false)); - var first = await rig.AdvanceToDispatchAsync(Milliseconds(25)); - rig.Clock.Advance(Milliseconds(190)); - Assert.Equal(Milliseconds(25), await rig.NotifyAsync(new KeyNotification("a", 2, false))); - rig.Clock.Advance(Milliseconds(24)); - Assert.Single(rig.Attempts); + await rig.NotifyAsync(rig.Members.Select(silo => new KeyNotification(silo, 10, true)).ToArray()); + var first = rig.Publish(0, 3); + var duplicate = rig.Publish(0, 3); + var older = rig.Publish(0, 1); + var second = rig.Publish(1); + rig.Clock.Advance(Milliseconds(999)); + Assert.False(first.IsCompleted); + Assert.False(second.IsCompleted); + Assert.Empty(rig.Attempts); - var second = await rig.AdvanceToDispatchAsync(Milliseconds(1)); - Assert.Equal(Milliseconds(215), rig.Clock.GetElapsedTime(first.Timestamp, second.Timestamp)); + rig.Clock.Advance(Milliseconds(1)); + Assert.All(await Phase(Task.WhenAll(first, duplicate, older, second), "hints do not count"), + receipt => Assert.Equal(new(true, TimeSpan.Zero), receipt)); + Assert.All(Assert.Single(rig.Attempts).Notifications, notification => + { + Assert.Equal(10, notification.Version); + Assert.True(notification.Force); + }); + Assert.Empty(rig.Namespace.VersionReads); } [Fact] - public async Task StaggeredAndBurstUpdatesMergeLatestValuesWithoutPushingDeadline() + public async Task HintBeforeFirstProducerAnchorsCohortDeadline() { await using var rig = new TestRig(); - Assert.Equal(Milliseconds(25), await rig.NotifyAsync(new KeyNotification("a", 1, true))); - rig.Clock.Advance(Milliseconds(10)); - Assert.True(rig.Batcher.Notify([new("b", 3, false), new("a", 2, false)])); - rig.Clock.Advance(Milliseconds(10)); - Assert.True(rig.Batcher.Notify([new("a", 9, false), new("a", 4, false)])); - Assert.Empty(rig.Attempts); + await rig.NotifyAsync(rig.Notification(1)); + rig.Clock.Advance(Milliseconds(400)); + var first = rig.Publish(0); + rig.Clock.Advance(Milliseconds(599)); + Assert.False(first.IsCompleted); + rig.Clock.Advance(Milliseconds(1)); + Assert.Equal(new(true, TimeSpan.Zero), await Phase(first, "deadline remains anchored to first pending hint")); Assert.Equal(1, rig.Clock.ScheduleCount); - - var attempt = await rig.AdvanceToDispatchAsync(Milliseconds(5)); - Assert.Equal( - new KeyNotification[] { new("a", 9, true), new("b", 3, false) }, - attempt.Notifications); + Assert.Equal(new[] { rig.Notification(1), rig.Notification(0) }, Assert.Single(rig.Attempts).Notifications); } [Fact] - public async Task LateWakeDoesNotAccumulateCreditsOrProduceCatchupBurst() + public async Task SealedRetriesReuseOutcomeAndRemainingDelayWithoutAnotherCohort() { - await using var rig = new TestRig(); - rig.Options.MaxBatchItems = 1; - await rig.NotifyAsync(new("a", 1, false), new("b", 1, false), new("c", 1, false)); + await using var rig = new TestRig(2); + var first = await rig.StartPublicationAsync(0, 2); + rig.Clock.Advance(Milliseconds(25)); + var peer = rig.Publish(1); + Assert.Equal(new(true, Milliseconds(975)), await Phase(first, "first early receipt")); + await Phase(peer, "second early receipt"); + rig.Clock.Advance(Milliseconds(75)); + + Assert.Equal(new(true, Milliseconds(900)), await rig.Publish(0, 2)); + Assert.Equal(new(true, Milliseconds(900)), await rig.Publish(0, 1)); + Assert.Equal(new(true, Milliseconds(900)), await rig.Publish(0, 2, force: true)); + rig.Clock.Advance(TimeSpan.FromHours(1)); + Assert.Equal(new(true, TimeSpan.Zero), await rig.Publish(0, 2)); + Assert.Single(rig.Attempts); + Assert.Equal(1, rig.Clock.ScheduleCount); + } - var first = await rig.AdvanceToDispatchAsync(TimeSpan.FromSeconds(5), morePending: true); + [Fact] + public async Task NewCohortRequiresNewVersionsFromEveryProducer() + { + await using var rig = new TestRig(2); + var first = await rig.StartPublicationAsync(0); + var peer = rig.Publish(1); + await Phase(Task.WhenAll(first, peer), "first complete cohort"); + rig.Clock.Advance(Period); + + var next = await rig.StartPublicationAsync(0, 2); + Assert.Equal(new(true, TimeSpan.Zero), await rig.Publish(1)); + rig.Clock.Advance(Milliseconds(25)); + Assert.False(next.IsCompleted); Assert.Single(rig.Attempts); - var second = await rig.AdvanceToDispatchAsync(Milliseconds(200), morePending: true); + var fresh = rig.Publish(1, 2); + Assert.All(await Phase(Task.WhenAll(next, fresh), "new versions fill the next cohort"), + receipt => Assert.Equal(new(true, Milliseconds(975)), receipt)); Assert.Equal(2, rig.Attempts.Count); - Assert.Equal(Milliseconds(200), rig.Clock.GetElapsedTime(first.Timestamp, second.Timestamp)); - var third = await rig.AdvanceToDispatchAsync(Milliseconds(200)); - Assert.Equal(Milliseconds(200), rig.Clock.GetElapsedTime(second.Timestamp, third.Timestamp)); } [Fact] - public async Task TwoThousandKeysUseBoundedFairChunksAndRetainedKeysUpdateAtCapacity() + public async Task LateTimerDoesNotCreateEmptyOrCatchupWaves() { await using var rig = new TestRig(); - rig.Options.MaxBatchItems = 128; - rig.Namespace.Options.MaxPendingItemCount = 2000; - var notifications = Enumerable.Range(0, 2001) - .Select(static index => new KeyNotification($"key-{index}", 1, false)).ToArray(); + var first = await rig.StartPublicationAsync(0); + rig.Clock.Advance(TimeSpan.FromSeconds(5)); + Assert.Equal(new(true, TimeSpan.Zero), await Phase(first, "late timer receipt")); + await Phase(rig.Batcher.FlushAsync(TestToken), "late cohort reconciled"); + Assert.Equal(new(true, TimeSpan.Zero), await rig.Publish(0)); + rig.Clock.Advance(TimeSpan.FromDays(1)); + Assert.Single(rig.Attempts); + Assert.Equal(1, rig.Clock.TimerCount); + + var second = await rig.StartPublicationAsync(0, 2); + rig.Clock.Advance(Milliseconds(999)); + Assert.False(second.IsCompleted); + rig.Clock.Advance(Milliseconds(1)); + Assert.Equal(new(true, TimeSpan.Zero), await Phase(second, "new cohort gets its own full period")); + Assert.Equal(2, rig.Attempts.Count); + } + + [Fact] + public async Task CanceledCallerDoesNotEraseContributionOrCancelSharedReceipt() + { + await using var rig = new TestRig(2); + using var cancellation = new CancellationTokenSource(); var scheduled = rig.Clock.WhenScheduled(); - Assert.False(rig.Batcher.Notify(notifications)); - Assert.Equal(Milliseconds(25), await Phase(scheduled, "bounded inventory collection")); - Assert.True(rig.Batcher.Notify([new("key-0", 9, false), new("key-1999", 7, false)])); - Assert.False(rig.Batcher.Notify([new("overflow", 1, false)])); + var canceled = rig.Publish(0, token: cancellation.Token); + Assert.Equal(Period, await Phase(scheduled, "cancelable publication admitted")); + var retry = rig.Publish(0); + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync(() => canceled); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.False(retry.IsCompleted); + rig.Clock.Advance(Milliseconds(25)); + + var peer = rig.Publish(1); + Assert.All(await Phase(Task.WhenAll(retry, peer), "shared receipt survives cancellation"), + receipt => Assert.Equal(new(true, Milliseconds(975)), receipt)); + Assert.Equal(new[] { rig.Notification(0), rig.Notification(1) }, Assert.Single(rig.Attempts).Notifications); + } - var first = await rig.AdvanceToDispatchAsync(Milliseconds(25), morePending: true); - Assert.Equal(128, first.Notifications.Length); - Assert.Equal(9, first.Notifications[0].Version); + [Fact] + public async Task PreCanceledPublicationDoesNotOpenCohort() + { + await using var rig = new TestRig(2); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + async () => await rig.Publish(0, token: cancellation.Token)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(0, rig.Clock.ScheduleCount); + var peer = await rig.StartPublicationAsync(1); + rig.Clock.Advance(Milliseconds(999)); + Assert.False(peer.IsCompleted); + rig.Clock.Advance(Milliseconds(1)); + Assert.True((await Phase(peer, "only admitted producer")).Accepted); + Assert.Equal(rig.Notification(1), Assert.Single(Assert.Single(rig.Attempts).Notifications)); + } - // Updating a waiting key must not move it behind later keys. A dispatched key can be - // re-admitted, but joins the tail instead of perpetually jumping ahead of the old inventory. - Assert.True(rig.Batcher.Notify( - [new("key-128", 6, false), new("key-0", 10, false), new("overflow", 1, false)])); - for (var wave = 1; wave < 16; wave++) + [Theory] + [InlineData(2000)] + [InlineData(2048)] + public async Task ThousandsOfProducersRetainLatestAtCapacityAndDispatchBoundedChunks(int capacity) + { + await using var rig = new TestRig(capacity + 1); + rig.Namespace.Options.MaxPendingItemCount = capacity; + rig.Options.MaxBatchItems = 128; + var receipts = new List> { await rig.StartPublicationAsync(0) }; + for (var i = 1; i < capacity; i++) { - var attempt = await rig.AdvanceToDispatchAsync(Milliseconds(200), morePending: wave < 15); - Assert.Equal(wave < 15 ? 128 : 82, attempt.Notifications.Length); + receipts.Add(rig.Publish(i)); } - var sent = rig.Attempts.SelectMany(static attempt => attempt.Notifications).ToArray(); - Assert.Equal( - notifications.Take(2000).Select(static notification => notification.Key) - .Concat(new DisseminationKey[] { "key-0", "overflow" }), - sent.Select(static notification => notification.Key)); - Assert.Equal(6, sent[128].Version); - Assert.Equal(7, sent[1999].Version); - Assert.Equal(new KeyNotification("key-0", 10, false), sent[2000]); - Assert.DoesNotContain(sent, static notification => notification.Key.Equals(new DisseminationKey("key-2000"))); + Assert.Equal(new(false, Period), await rig.Publish(capacity)); + receipts.Add(rig.Publish(0, 9)); + receipts.Add(rig.Publish(capacity - 1, 7)); + rig.Clock.Advance(Period); + Assert.All(await Phase(Task.WhenAll(receipts), "bounded producer cohort"), receipt => Assert.True(receipt.Accepted)); + + var attempts = rig.Attempts.ToArray(); + Assert.Equal((capacity + 127) / 128, attempts.Length); + Assert.All(attempts.Take(attempts.Length - 1), attempt => Assert.Equal(128, attempt.Notifications.Length)); + Assert.Equal((capacity - 1) % 128 + 1, attempts[^1].Notifications.Length); + var sent = attempts.SelectMany(attempt => attempt.Notifications).ToArray(); + Assert.Equal(rig.Members.Take(capacity).Select(silo => new DisseminationKey(silo)), sent.Select(n => n.Key)); + Assert.Equal(9, sent[0].Version); + Assert.Equal(7, sent[^1].Version); + Assert.DoesNotContain(sent, n => n.Key.Equals(new DisseminationKey(rig.Members[capacity]))); + Assert.All(attempts, attempt => Assert.Equal(attempts[0].Timestamp, attempt.Timestamp)); + Assert.Empty(rig.Namespace.VersionReads); + + var later = await rig.StartPublicationAsync(capacity); + rig.Clock.Advance(Period); + Assert.True((await Phase(later, "capacity released after admission")).Accepted); + } + + [Theory] + [InlineData(2000)] + [InlineData(2048)] + public async Task FullCohortPreservesPeerItemAndByteSplitting(int count) + { + await using var rig = new TestRig(count); + rig.Options.MaxBatchItems = 128; + rig.Options.MaxBatchBytes = 64; + rig.Namespace.Options.MaxPayloadBytes = sizeof(long); + rig.Namespace.Options.MaxPendingItemCount = count; + rig.Namespace.MaterializeValues = true; + var target = Substitute.For(); + var factory = Substitute.For(); + var batches = new ConcurrentQueue(); + target.PushBroadcast(Arg.Any(), Arg.Any()) + .Returns(call => + { + var batch = call.ArgAt(0); + batches.Enqueue(batch); + return Task.FromResult(new DisseminationBroadcastResponse + { + Acknowledgments = batch.Values.ToDictionary( + pair => pair.Key, pair => pair.Value.Select(value => new DigestEntry(value.Value.Key, value.Value.ToVersion)).ToList()), + }); + }); + factory.GetSystemTarget(Constants.DisseminationSystemTargetType, rig.Members[1]).Returns(target); + var queue = new DisseminationBroadcastQueue( + rig.Clock, rig.Members[0], factory, new TestOptionsMonitor(rig.Options), + [rig.Namespace], NullLogger.Instance); + rig.OnDispatch = attempt => queue.NotifyBatch(rig.Members[1], rig.Namespace, attempt.Notifications, immediate: false); + try + { + var receipts = new List> { await rig.StartPublicationAsync(0) }; + for (var i = 1; i < count; i++) + { + receipts.Add(rig.Publish(i)); + } + + Assert.All(await Phase(Task.WhenAll(receipts), "all producers sealed without fake time advancement"), + receipt => Assert.Equal(new(true, Period), receipt)); + await Phase(queue.FlushPendingBroadcast(TestToken), "peer byte-limited chunks acknowledged"); + + var values = batches.SelectMany(batch => batch.Values[rig.Namespace.Name]).Select(value => value.Value).ToArray(); + Assert.Equal(count, values.Length); + Assert.Equal(count, values.Select(value => value.Key).Distinct().Count()); + Assert.Equal(rig.Members.Select(silo => new DisseminationKey(silo)).Order(), values.Select(value => value.Key).Order()); + Assert.All(batches, batch => + { + var payloads = batch.Values[rig.Namespace.Name]; + Assert.InRange(payloads.Count, 1, rig.Options.MaxBatchItems); + Assert.InRange(payloads.Sum(value => value.Value.Payload.Length), 1, rig.Options.MaxBatchBytes); + Assert.All(payloads, value => Assert.Equal(1L, BitConverter.ToInt64(value.Value.Payload.Span))); + }); + Assert.All(rig.Attempts, attempt => Assert.InRange(attempt.Notifications.Length, 1, 128)); + } + finally + { + await queue.StopAsync(TestToken); + } } [Theory] @@ -154,140 +355,211 @@ public async Task TwoThousandKeysUseBoundedFairChunksAndRetainedKeysUpdateAtCapa [InlineData(false, false)] public async Task UpdatesDuringDispatchPreserveGenerationForceAndInFlightCapacity(bool firstAccepted, bool newForce) { - await using var rig = new TestRig(); + await using var rig = new TestRig(2); rig.Namespace.Options.MaxPendingItemCount = 1; using var release = new ManualResetEventSlim(); - var cancellationToken = TestToken; + var token = TestToken; rig.OnDispatch = attempt => { if (attempt.Number == 1) { - Assert.True(release.Wait(TimeSpan.FromSeconds(10), cancellationToken), "First dispatch was not released."); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "First dispatch was not released."); } return attempt.Number > 1 || firstAccepted; }; - - await rig.NotifyAsync(new KeyNotification("a", 1, true)); - var nextSchedule = rig.Clock.WhenScheduled(); - rig.Clock.Advance(Milliseconds(25)); - var first = await rig.NextAttemptAsync(); + var first = await rig.StartPublicationAsync(0, force: true); + var scheduled = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Period); + await rig.NextAttemptAsync(); + Task newer = null!; try { - // Another thread must be able to enter Notify while the synchronous callback is blocked. - // Re-entering on the callback's thread alone would not prove the lock was released. - await Phase(Task.Run(() => + await Phase(Task.Run(async () => { - Assert.False(rig.Batcher.Notify([new("b", 1, true)])); - Assert.True(rig.Batcher.Notify([new("a", 2, newForce)])); - }, TestToken), "concurrent admission while callback owns the first snapshot"); - Assert.Equal(new KeyNotification("a", 1, true), Assert.Single(first.Notifications)); + Assert.Equal(new(false, Period), await rig.Publish(1)); + newer = rig.Publish(0, 2, newForce); + Assert.False(newer.IsCompleted); + }, TestToken), "new generation admitted concurrently with the dispatch callback"); } finally { release.Set(); } - Assert.Equal(Milliseconds(200), await Phase(nextSchedule, "new generation scheduled after dispatch")); - var second = await rig.AdvanceToDispatchAsync(Milliseconds(200)); - Assert.Equal(new KeyNotification("a", 2, newForce), Assert.Single(second.Notifications)); - Assert.Equal(2, rig.Attempts.Count); + Assert.Equal(firstAccepted, (await Phase(first, "first generation receipt")).Accepted); + Assert.Equal(Period, await Phase(scheduled, "new generation deadline")); + rig.Clock.Advance(Period); + Assert.True((await Phase(newer, "new generation receipt")).Accepted); + Assert.Equal(new[] { rig.Notification(0, 1, true), rig.Notification(0, 2, newForce) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); } - [Theory] - [InlineData(false, 1L)] - [InlineData(true, 1L)] - [InlineData(true, 2L)] - public async Task PartialAdmissionRetriesPreserveOriginalAndNewForce(bool newForce, long version) + [Fact] + public async Task SameVersionInvalidationDuringDispatchRemainsForced() { await using var rig = new TestRig(); - var firstPeerVersions = new Dictionary(); - var firstPeerSends = new List(); - var secondPeerSends = new List(); + using var release = new ManualResetEventSlim(); + var token = TestToken; rig.OnDispatch = attempt => { - foreach (var notification in attempt.Notifications) + if (attempt.Number == 1) { - if (notification.Force - || !firstPeerVersions.TryGetValue(notification.Key, out var knownVersion) - || notification.Version > knownVersion) - { - firstPeerVersions[notification.Key] = notification.Version; - firstPeerSends.Add(notification); - } + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Invalidated dispatch was not released."); } + return true; + }; + await rig.NotifyAsync(new KeyNotification("a", 1, false)); + var scheduled = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Period); + await rig.NextAttemptAsync(); + try + { + await Phase(Task.Run(() => + { + Assert.True(rig.Batcher.Notify([new("a", 1, false)])); + Assert.True(rig.Batcher.Notify([new("a", 1, true)])); + }, TestToken), "same-version invalidation admitted during callback"); + } + finally + { + release.Set(); + } + + Assert.Equal(Period, await Phase(scheduled, "invalidation remains pending")); + rig.Clock.Advance(Period); + await Phase(rig.Batcher.FlushAsync(TestToken), "same-version invalidation dispatched"); + Assert.Equal(new KeyNotification[] { new("a", 1, false), new("a", 1, true) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); + } + + [Fact] + public async Task OlderInFlightRetryUsesSealedSignalWhileNewerVersionWaitsForNextCohort() + { + await using var rig = new TestRig(2); + using var release = new ManualResetEventSlim(); + var token = TestToken; + rig.OnDispatch = attempt => + { if (attempt.Number == 1) { - return false; + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "First generation was not released."); } - secondPeerSends.AddRange(attempt.Notifications); return true; }; - await rig.NotifyAsync(new("a", 1, true), new("b", 1, true)); - await rig.AdvanceToDispatchAsync(Milliseconds(25), morePending: true); + var first = await rig.StartPublicationAsync(0); + var scheduled = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Period); + await rig.NextAttemptAsync(); + Task newer; + Task retry; + try + { + newer = rig.Publish(0, 2); + retry = rig.Publish(0); + Assert.False(first.IsCompleted); + Assert.False(newer.IsCompleted); + Assert.False(retry.IsCompleted); + } + finally + { + release.Set(); + } + + Assert.All(await Phase(Task.WhenAll(first, retry), "old callers share the sealed generation"), + receipt => Assert.Equal(new(true, TimeSpan.Zero), receipt)); + Assert.False(newer.IsCompleted); + Assert.Equal(Period, await Phase(scheduled, "only the new generation waits another period")); + rig.Clock.Advance(Period); + Assert.Equal(new(true, TimeSpan.Zero), await Phase(newer, "next generation admitted")); + Assert.Equal(new[] { rig.Notification(0), rig.Notification(0, 2) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); + } + + [Theory] + [InlineData(false, 1L)] + [InlineData(true, 1L)] + [InlineData(true, 2L)] + public async Task PartialAdmissionRejectsReceiptButRetriesNewestHintAndForce(bool newForce, long version) + { + await using var rig = new TestRig(2); + rig.OnDispatch = attempt => attempt.Number != 1; + var first = await rig.StartPublicationAsync(0, force: true); + var scheduled = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Period); + Assert.Equal(new(false, TimeSpan.Zero), await Phase(first, "partial admission receipt")); + Assert.Equal(Period, await Phase(scheduled, "bounded hint retry")); + Assert.Equal(new(false, TimeSpan.Zero), await rig.Publish(0)); if (newForce) { - Assert.True(rig.Batcher.Notify([new("a", version, true)])); + Assert.True(rig.Batcher.Notify([rig.Notification(0, version, true)])); } - var retried = await rig.AdvanceToDispatchAsync(Milliseconds(200)); - Assert.Equal( - new KeyNotification[] { new("a", version, true), new("b", 1, true) }, - retried.Notifications); - Assert.Equal( - new KeyNotification[] { new("a", 1, true), new("b", 1, true), new("a", version, true), new("b", 1, true) }, - firstPeerSends); - Assert.Equal(retried.Notifications, secondPeerSends); + rig.Clock.Advance(Milliseconds(999)); + Assert.Single(rig.Attempts); + rig.Clock.Advance(Milliseconds(1)); + await Phase(rig.Batcher.FlushAsync(TestToken), "retained hint admitted on retry"); + Assert.Equal(new[] { rig.Notification(0, 1, true), rig.Notification(0, version, true) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); + // A later hint retry is not another publication receipt or another producer contribution. + Assert.Equal(new(false, TimeSpan.Zero), await rig.Publish(0)); } [Fact] - public async Task PartialRetryRotatesBehindWaitingKeys() + public async Task PartialChunkAdmissionResolvesOnlyItsOwnProducerReceipts() { - await using var rig = new TestRig(); + await using var rig = new TestRig(3); rig.Options.MaxBatchItems = 1; - rig.OnDispatch = attempt => attempt.Number != 1; - await rig.NotifyAsync(new("a", 1, true), new("b", 1, true), new("c", 1, true)); - await rig.AdvanceToDispatchAsync(Milliseconds(25), morePending: true); - await rig.AdvanceToDispatchAsync(Milliseconds(200), morePending: true); - await rig.AdvanceToDispatchAsync(Milliseconds(200), morePending: true); - await rig.AdvanceToDispatchAsync(Milliseconds(200)); - - Assert.Equal( - new KeyNotification[] { new("a", 1, true), new("b", 1, true), new("c", 1, true), new("a", 1, true) }, - rig.Attempts.SelectMany(static attempt => attempt.Notifications)); + rig.OnDispatch = attempt => attempt.Number != 2; + var first = await rig.StartPublicationAsync(0); + var second = rig.Publish(1); + var retryScheduled = rig.Clock.WhenScheduled(); + var third = rig.Publish(2); + var receipts = await Phase(Task.WhenAll(first, second, third), "per-chunk receipts"); + Assert.Equal(new[] { true, false, true }, receipts.Select(receipt => receipt.Accepted)); + Assert.All(receipts, receipt => Assert.Equal(Period, receipt.NextPublicationDelay)); + Assert.Equal(Period, await Phase(retryScheduled, "only rejected chunk retained")); + rig.Clock.Advance(Period); + await Phase(rig.Batcher.FlushAsync(TestToken), "one rejected chunk retried"); + Assert.Equal(new[] { rig.Notification(0), rig.Notification(1), rig.Notification(2), rig.Notification(1) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); } [Fact] - public async Task CallbackExceptionRetriesAndReportsFailure() + public async Task CallbackExceptionRejectsReceiptsRetriesHintsAndReportsFailure() { await using var rig = new TestRig(); var failure = new InvalidOperationException("The parent dispatcher failed."); rig.OnDispatch = attempt => attempt.Number == 1 ? throw failure : true; - await rig.NotifyAsync(new KeyNotification("a", 1, true)); - - await rig.AdvanceToDispatchAsync(Milliseconds(25), morePending: true); + var publication = await rig.StartPublicationAsync(0, force: true); + var scheduled = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Period); + Assert.Equal(new(false, TimeSpan.Zero), await Phase(publication, "exception rejects the producer receipt")); + Assert.Equal(Period, await Phase(scheduled, "exception retry armed")); var entry = Assert.Single(rig.Logger.Entries); Assert.Equal(LogLevel.Warning, entry.Level); Assert.Same(failure, entry.Exception); - var retry = await rig.AdvanceToDispatchAsync(Milliseconds(200)); - Assert.Equal(new KeyNotification("a", 1, true), Assert.Single(retry.Notifications)); + rig.Clock.Advance(Period); + await Phase(rig.Batcher.FlushAsync(TestToken), "exception hint recovered"); + Assert.Equal(rig.Notification(0, 1, true), Assert.Single(rig.Attempts.Last().Notifications)); + Assert.Equal(2, rig.Attempts.Count); } [Fact] - public async Task CallbackLoggingFailureIsVisibleToAdmissionAndDrain() + public async Task CallbackLoggingFailureIsVisibleToReceiptAdmissionAndDrain() { await using var rig = new TestRig(); var dispatchFailure = new InvalidOperationException("The parent dispatcher failed."); rig.Logger.ThrowOnLog = true; rig.OnDispatch = _ => throw dispatchFailure; - await rig.NotifyAsync(new KeyNotification("a", 1, true)); - rig.Clock.Advance(Milliseconds(25)); + var publication = await rig.StartPublicationAsync(0); + rig.Clock.Advance(Period); var failure = await Phase(rig.Logger.WorkerFailed.Task, "dispatch logging failure recorded"); - var aggregate = Assert.IsType(failure); - Assert.Same(dispatchFailure, aggregate.InnerExceptions[0]); - + Assert.Same(dispatchFailure, Assert.IsType(failure).InnerExceptions[0]); + var receipt = await Assert.ThrowsAsync(() => publication); + Assert.Same(failure, receipt.InnerException); var admission = Assert.Throws(() => rig.Batcher.Notify([new("b", 1, true)])); Assert.Same(failure, admission.InnerException); var drain = await Assert.ThrowsAsync(() => rig.Batcher.FlushAsync(TestToken)); @@ -295,19 +567,19 @@ public async Task CallbackLoggingFailureIsVisibleToAdmissionAndDrain() } [Fact] - public async Task WorkerFailureIsObservedByActiveAndLaterFlushNotifyAndStop() + public async Task WorkerFailureIsObservedByActiveAndLaterFlushNotifyPublishAndStop() { await using var rig = new TestRig(); using var release = new ManualResetEventSlim(); - var cancellationToken = TestToken; + var token = TestToken; rig.Logger.ThrowOnLog = true; rig.OnDispatch = _ => { - Assert.True(release.Wait(TimeSpan.FromSeconds(10), cancellationToken), "Failing dispatch was not released."); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Failing dispatch was not released."); return false; }; await rig.NotifyAsync(new KeyNotification("a", 1, true)); - rig.Clock.Advance(Milliseconds(25)); + rig.Clock.Advance(Period); await rig.NextAttemptAsync(); Task activeFlush; try @@ -322,16 +594,15 @@ public async Task WorkerFailureIsObservedByActiveAndLaterFlushNotifyAndStop() } var failure = await Phase(rig.Logger.WorkerFailed.Task, "terminal timer failure logged"); - var activeFailure = await Assert.ThrowsAsync( - () => Phase(activeFlush, "active flush observes terminal failure")); - Assert.Same(failure, activeFailure.InnerException); - var notificationFailure = Assert.Throws( - () => rig.Batcher.Notify([new("b", 2, false)])); - Assert.Same(failure, notificationFailure.InnerException); - var flushFailure = await Assert.ThrowsAsync(() => rig.Batcher.FlushAsync(TestToken)); - Assert.Same(failure, flushFailure.InnerException); - var stopFailure = await Assert.ThrowsAsync(() => rig.Batcher.StopAsync(TestToken)); - Assert.Same(failure, stopFailure.InnerException); + Assert.Same(failure, (await Assert.ThrowsAsync( + () => Phase(activeFlush, "active flush observes terminal failure"))).InnerException); + Assert.Same(failure, Assert.Throws(() => rig.Batcher.Notify([new("b", 2, false)])).InnerException); + Assert.Same(failure, (await Assert.ThrowsAsync( + async () => await rig.Publish(0))).InnerException); + Assert.Same(failure, (await Assert.ThrowsAsync( + () => rig.Batcher.FlushAsync(TestToken))).InnerException); + Assert.Same(failure, (await Assert.ThrowsAsync( + () => rig.Batcher.StopAsync(TestToken))).InnerException); Assert.True(rig.Clock.TimerDisposed); Assert.Single(rig.Attempts); } @@ -339,10 +610,10 @@ public async Task WorkerFailureIsObservedByActiveAndLaterFlushNotifyAndStop() [Theory] [InlineData(false)] [InlineData(true)] - public async Task DisablingDropsPendingHintsAndReenableDoesNotReplay(bool global) + public async Task DisablingRejectsReceiptsAndReenableDoesNotReplay(bool global) { await using var rig = new TestRig(); - await rig.NotifyAsync(new KeyNotification("old", 1, true)); + var publication = await rig.StartPublicationAsync(0); if (global) { rig.Options.Enabled = false; @@ -352,53 +623,208 @@ public async Task DisablingDropsPendingHintsAndReenableDoesNotReplay(bool global rig.Namespace.Options.Enabled = false; } - await Phase(rig.Batcher.FlushAsync(TestToken), "disabled hints retired"); + await Phase(rig.Batcher.FlushAsync(TestToken), "disabled work retired"); + Assert.False((await publication).Accepted); Assert.False(rig.Batcher.Notify([new("disabled", 2, true)])); + Assert.False((await rig.Publish(1)).Accepted); Assert.Empty(rig.Attempts); rig.Options.Enabled = true; rig.Namespace.Options.Enabled = true; - rig.Clock.Advance(TimeSpan.FromSeconds(1)); - Assert.Equal(Milliseconds(25), await rig.NotifyAsync(new KeyNotification("new", 3, false))); + Assert.False((await rig.Publish(0)).Accepted); + var newer = await rig.StartPublicationAsync(0, 2); + rig.Clock.Advance(Period); + Assert.True((await Phase(newer, "new work after reenabling")).Accepted); + Assert.Equal(rig.Notification(0, 2), Assert.Single(Assert.Single(rig.Attempts).Notifications)); + } - var sent = await rig.AdvanceToDispatchAsync(Milliseconds(25)); - Assert.Equal(new KeyNotification("new", 3, false), Assert.Single(sent.Notifications)); - Assert.Single(rig.Attempts); + [Fact] + public async Task MembershipSnapshotIsStableAndRetiredIncarnationCannotContribute() + { + await using var rig = new TestRig(); + var first = await rig.StartPublicationAsync(0); + var retired = rig.Members[2]; + var replacement = SiloAddress.New(retired.Endpoint, retired.Generation + 1); + rig.SetMembership([rig.Members[0], rig.Members[1], replacement]); + Assert.False((await rig.Batcher.PublishAsync(new(retired, 9, true), TestToken)).Accepted); + Assert.False(rig.Batcher.Notify([new(retired, 9, true)])); + var other = rig.Publish(1); + var joined = rig.Batcher.PublishAsync(new(replacement, 1, false), TestToken).AsTask(); + rig.Clock.Advance(Milliseconds(999)); + Assert.False(first.IsCompleted); + Assert.False(other.IsCompleted); + Assert.False(joined.IsCompleted); + Assert.Empty(rig.Attempts); + rig.Clock.Advance(Milliseconds(1)); + Assert.All(await Phase(Task.WhenAll(first, other, joined), "stable cohort tolerates incarnation replacement"), + receipt => Assert.Equal(new(true, TimeSpan.Zero), receipt)); + Assert.DoesNotContain(Assert.Single(rig.Attempts).Notifications, n => n.Key.Equals(new DisseminationKey(retired))); } [Fact] - public async Task PruneRemovesRetiredKeysAndFreesAdmissionCapacity() + public async Task MembershipChangeRetiresPendingReceiptsAndFreesCapacity() { await using var rig = new TestRig(); - rig.Namespace.Options.MaxPendingItemCount = 2; - await rig.NotifyAsync(new("retired", 1, true), new("retained", 2, false)); + rig.Namespace.Options.MaxPendingItemCount = 1; + var departed = await rig.StartPublicationAsync(2); + rig.SetMembership([rig.Members[0], rig.Members[1]]); var scheduled = rig.Clock.WhenScheduled(); - rig.Batcher.Prune(new HashSet { "retained" }); - Assert.True(rig.Batcher.Notify([new("new", 3, false)])); - Assert.Equal(Milliseconds(25), await Phase(scheduled, "pruned inventory rescheduled")); + var active = rig.Publish(0); + Assert.False((await Phase(departed, "departed incarnation rejected")).Accepted); + Assert.Equal(Period, await Phase(scheduled, "replacement pending capacity")); + rig.Clock.Advance(Period); + Assert.True((await Phase(active, "active contribution accepted")).Accepted); + Assert.Equal(rig.Notification(0), Assert.Single(Assert.Single(rig.Attempts).Notifications)); + } - var attempt = await rig.AdvanceToDispatchAsync(Milliseconds(25)); - Assert.Equal( - new KeyNotification[] { new("retained", 2, false), new("new", 3, false) }, - attempt.Notifications); + [Fact] + public async Task EmptyMembershipReleasesPendingReceiptsWithoutDispatch() + { + await using var rig = new TestRig(); + var first = await rig.StartPublicationAsync(0); + var second = rig.Publish(1); + rig.SetMembership([]); + rig.Batcher.WakeForMembershipChange(); + Assert.All(await Phase(Task.WhenAll(first, second), "root and peers no longer eligible"), + receipt => Assert.False(receipt.Accepted)); + await Phase(rig.Batcher.StopAsync(TestToken), "root loss drains without destinations"); + Assert.Empty(rig.Attempts); + Assert.True(rig.Clock.TimerDisposed); + } + + [Fact] + public async Task RacingMembershipReadCannotReadmitAnAlreadyRetiredIncarnation() + { + await using var rig = new TestRig(); + var first = await rig.StartPublicationAsync(0); + var oldMembership = rig.Membership; + using var release = new ManualResetEventSlim(); + var token = TestToken; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var reads = 0; + rig.OnMembershipRead = () => + { + if (Interlocked.Increment(ref reads) == 1) + { + entered.TrySetResult(); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Stale membership read was not released."); + return oldMembership; + } + + return rig.Membership; + }; + var stale = Task.Run(() => rig.Publish(1), TestToken); + try + { + await Phase(entered.Task, "producer captured old membership"); + rig.SetMembership([rig.Members[0]]); + await Phase(Task.Run(() => Assert.False(rig.Batcher.Notify([rig.Notification(2)])), TestToken), + "new membership observed while older callback is blocked"); + } + finally + { + release.Set(); + } + + Assert.False((await Phase(stale, "stale membership cannot reverse retirement")).Accepted); + rig.Clock.Advance(Period); + Assert.True((await Phase(first, "remaining root contribution")).Accepted); + Assert.Equal(rig.Notification(0), Assert.Single(Assert.Single(rig.Attempts).Notifications)); + } + + [Fact] + public async Task MembershipWakeHandsAllPendingChunksToNewRootWithoutWaiting() + { + await using var rig = new TestRig(5); + rig.Options.MaxBatchItems = 2; + var first = await rig.StartPublicationAsync(0); + var second = rig.Publish(1); + var third = rig.Publish(2); + Assert.True(rig.Batcher.Notify([rig.Notification(3)])); + rig.Clock.Advance(Milliseconds(25)); + rig.SetMembership([rig.Members[1], rig.Members[0], rig.Members[2], rig.Members[3]]); + var routedToNewRoot = new ConcurrentQueue(); + rig.OnDispatch = _ => + { + routedToNewRoot.Enqueue(!rig.Membership.IsAggregationRoot); + return true; + }; + rig.Batcher.WakeForMembershipChange(); + Assert.All(await Phase(Task.WhenAll(first, second, third), "new root handoff receipts"), + receipt => Assert.Equal(new(true, Milliseconds(975)), receipt)); + Assert.Equal(new[] { 2, 2 }, rig.Attempts.Select(attempt => attempt.Notifications.Length)); + Assert.All(routedToNewRoot, routed => Assert.True(routed)); + Assert.False((await rig.Publish(0, 2)).Accepted); + await Phase(rig.Batcher.StopAsync(TestToken), "former root stopped"); + Assert.True(rig.Clock.TimerDisposed); + } + + [Fact] + public async Task MembershipChangeDuringDispatchSkipsRetiredChunkAndRejectsItsReceipt() + { + await using var rig = new TestRig(4); + rig.Options.MaxBatchItems = 1; + var first = await rig.StartPublicationAsync(0); + var second = rig.Publish(1); + var retired = rig.Publish(2); + rig.OnDispatch = attempt => + { + if (attempt.Number == 1) + { + rig.SetMembership([rig.Members[0], rig.Members[1]]); + } + + return true; + }; + + await Phase(rig.Batcher.FlushAsync(TestToken), "membership revalidated between chunks"); + var receipts = await Task.WhenAll(first, second, retired); + Assert.Equal(new[] { true, true, false }, receipts.Select(receipt => receipt.Accepted)); + Assert.Equal(new[] { rig.Notification(0), rig.Notification(1) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); } [Fact] - public async Task PrunePreservesLiveKeysPublishedAfterInventoryCapture() + public async Task RootLossBypassesOldRetryFloorOnceButFailedHandoffRemainsBounded() + { + await using var rig = new TestRig(2); + rig.OnDispatch = _ => false; + var first = await rig.StartPublicationAsync(0); + var scheduled = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Period); + Assert.False((await Phase(first, "first admission rejected")).Accepted); + Assert.Equal(Period, await Phase(scheduled, "old root retry floor")); + rig.Clock.Advance(Milliseconds(25)); + rig.SetMembership([rig.Members[1], rig.Members[0]]); + var handoffScheduled = rig.Clock.WhenScheduled(); + rig.Batcher.WakeForMembershipChange(); + Assert.Equal(Period, await Phase(handoffScheduled, "failed new-root handoff uses one period retry")); + Assert.Equal(2, rig.Attempts.Count); + rig.Clock.Advance(Milliseconds(999)); + Assert.Equal(2, rig.Attempts.Count); + rig.OnDispatch = _ => true; + rig.Clock.Advance(Milliseconds(1)); + await Phase(rig.Batcher.FlushAsync(TestToken), "handoff retry admitted"); + Assert.Equal(3, rig.Attempts.Count); + Assert.Equal(Milliseconds(25), rig.Clock.GetElapsedTime(rig.Attempts.First().Timestamp, rig.Attempts.ElementAt(1).Timestamp)); + } + + [Fact] + public async Task PruneRemovesRetiredKeysAndPreservesNewerOwnerInventory() { await using var rig = new TestRig(); - await rig.NotifyAsync(new("retired", 1, false), new("retained", 2, false)); + rig.Namespace.Options.MaxPendingItemCount = 3; + await rig.NotifyAsync(new("retired", 1, true), new("retained", 2, false)); var inventory = new HashSet { "retained" }; rig.Namespace.GetVersionHandler = key => key.Equals(new DisseminationKey("new")) ? 3 : 0; Assert.True(rig.Batcher.Notify([new("new", 3, false)])); var scheduled = rig.Clock.WhenScheduled(); - rig.Batcher.Prune(inventory); - - Assert.Equal(Milliseconds(25), await Phase(scheduled, "fresh owner versions checked before pruning")); - var attempt = await rig.AdvanceToDispatchAsync(Milliseconds(25)); - Assert.Equal( - new KeyNotification[] { new("retained", 2, false), new("new", 3, false) }, - attempt.Notifications); + Assert.Equal(Period, await Phase(scheduled, "inventory rechecked outside lock")); + Assert.True(rig.Batcher.Notify([new("extra", 4, false)])); + rig.Clock.Advance(Period); + await Phase(rig.Batcher.FlushAsync(TestToken), "pruned inventory dispatched"); + Assert.Equal(new KeyNotification[] { new("retained", 2, false), new("new", 3, false), new("extra", 4, false) }, + Assert.Single(rig.Attempts).Notifications); Assert.Equal(new DisseminationKey[] { "retired", "new" }, rig.Namespace.VersionReads); } @@ -410,202 +836,219 @@ public async Task PruneVersionReadRunsOutsideLockAndPreservesRacingNotifications await using var rig = new TestRig(); await rig.NotifyAsync(new KeyNotification("a", 1, false)); using var release = new ManualResetEventSlim(); - var cancellationToken = TestToken; var versionRead = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); rig.Namespace.GetVersionHandler = key => { Assert.Equal(new DisseminationKey("a"), key); versionRead.TrySetResult(); - Assert.True(release.Wait(TimeSpan.FromSeconds(10), cancellationToken), "Owner version read was not released."); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), TestToken), "Owner version read was not released."); return 0; }; var scheduled = rig.Clock.WhenScheduled(); - var prune = Task.Run(() => rig.Batcher.Prune(new HashSet()), cancellationToken); + var prune = Task.Run(() => rig.Batcher.Prune(new HashSet()), TestToken); try { - await Phase(versionRead.Task, "pruning captured the old admission generation"); - await Phase(Task.Run( - () => Assert.True(rig.Batcher.Notify([new("a", version, false)])), cancellationToken), - "notification admitted while the owner version read is blocked"); + await Phase(versionRead.Task, "pruning captured the old generation"); + await Phase(Task.Run(() => Assert.True(rig.Batcher.Notify([new("a", version, false)])), TestToken), + "notification admitted while namespace version read is blocked"); } finally { release.Set(); - await Phase(prune, "pruning reconciles the captured generation"); + await Phase(prune, "pruning reconciled admission generations"); } - Assert.Equal(Milliseconds(25), await Phase(scheduled, "racing notification remains scheduled")); - var attempt = await rig.AdvanceToDispatchAsync(Milliseconds(25)); - Assert.Equal(new KeyNotification("a", version, false), Assert.Single(attempt.Notifications)); + Assert.Equal(Period, await Phase(scheduled, "racing notification preserved")); + rig.Clock.Advance(Period); + await Phase(rig.Batcher.FlushAsync(TestToken), "racing notification dispatched"); + Assert.Equal(new KeyNotification("a", version, false), Assert.Single(Assert.Single(rig.Attempts).Notifications)); } [Fact] - public async Task PruneDuringDispatchDoesNotEraseReadmittedKey() + public async Task PruneDuringDispatchDoesNotEraseReadmittedIdentity() { await using var rig = new TestRig(); rig.Namespace.Options.MaxPendingItemCount = 1; using var release = new ManualResetEventSlim(); - var cancellationToken = TestToken; + var token = TestToken; rig.OnDispatch = attempt => { if (attempt.Number == 1) { - Assert.True(release.Wait(TimeSpan.FromSeconds(10), cancellationToken), "Pruned snapshot was not released."); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Pruned dispatch was not released."); } return true; }; - await rig.NotifyAsync(new KeyNotification("a", 1, true)); - var nextSchedule = rig.Clock.WhenScheduled(); - rig.Clock.Advance(Milliseconds(25)); + var first = await rig.StartPublicationAsync(0); + var scheduled = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Period); await rig.NextAttemptAsync(); + Task next; try { rig.Batcher.Prune(new HashSet()); - Assert.True(rig.Batcher.Notify([new("a", 7, true)])); + Assert.False((await Phase(first, "pruned in-flight receipt rejected")).Accepted); + next = rig.Publish(0, 7, force: true); } finally { release.Set(); } - Assert.Equal(Milliseconds(200), await Phase(nextSchedule, "readmitted identity scheduled")); - var next = await rig.AdvanceToDispatchAsync(Milliseconds(200)); - Assert.Equal(new KeyNotification("a", 7, true), Assert.Single(next.Notifications)); - } - - [Fact] - public async Task HighPriorityBypassesCollectionAndCadenceButRetriesRemainPaced() - { - await using var rig = new TestRig(); - rig.Namespace.Options.Priority = DisseminationPriority.High; - rig.OnDispatch = attempt => attempt.Number != 1; - var start = rig.Clock.GetTimestamp(); - var retryScheduled = rig.Clock.WhenScheduled(); - Assert.True(rig.Batcher.Notify([new("a", 1, true)])); - var first = await rig.NextAttemptAsync(); - Assert.Equal(start, first.Timestamp); - Assert.Equal(Milliseconds(200), await Phase(retryScheduled, "high-priority retry paced")); - await rig.AdvanceToDispatchAsync(Milliseconds(200)); - - Assert.True(rig.Batcher.Notify([new("b", 2, true)])); - var next = await rig.NextAttemptAsync(); - await Phase(rig.Batcher.FlushAsync(TestToken), "high-priority handoff completes"); - Assert.Equal(rig.Clock.GetTimestamp(), next.Timestamp); - Assert.Equal(new KeyNotification("b", 2, true), Assert.Single(next.Notifications)); + Assert.Equal(Period, await Phase(scheduled, "readmitted identity gets its own deadline")); + rig.Clock.Advance(Period); + Assert.True((await Phase(next, "readmitted identity accepted")).Accepted); + Assert.Equal(new[] { rig.Notification(0), rig.Notification(0, 7, true) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); } [Fact] - public async Task PruneReevaluatesCurrentRateWithoutGrantingCredits() + public async Task PublicationRetryProtectsAdmissionFromRacingPruneWithoutContributingTwice() { - await using var rig = new TestRig(); - await rig.NotifyAsync(new KeyNotification("a", 1, false)); - var first = await rig.AdvanceToDispatchAsync(Milliseconds(25)); - await rig.NotifyAsync(new KeyNotification("b", 1, false)); - rig.Clock.Advance(Milliseconds(20)); - rig.Options.Overlay.AggregationBroadcastsPerSecond = 10; + await using var rig = new TestRig(2); + var first = await rig.StartPublicationAsync(0); + using var release = new ManualResetEventSlim(); + var token = TestToken; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + rig.Namespace.GetVersionHandler = _ => + { + entered.TrySetResult(); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Prune version read was not released."); + return 0; + }; var scheduled = rig.Clock.WhenScheduled(); - rig.Batcher.Prune(new HashSet { "b" }); - Assert.Equal(Milliseconds(80), await Phase(scheduled, "updated rate applied to existing deadline")); - rig.Clock.Advance(Milliseconds(79)); - Assert.Single(rig.Attempts); - var second = await rig.AdvanceToDispatchAsync(Milliseconds(1)); - Assert.Equal(Milliseconds(100), rig.Clock.GetElapsedTime(first.Timestamp, second.Timestamp)); + var prune = Task.Run(() => rig.Batcher.Prune(new HashSet()), TestToken); + Task retry; + try + { + await Phase(entered.Task, "prune captured pre-retry admission"); + retry = rig.Publish(0); + Assert.False(first.IsCompleted); + Assert.False(retry.IsCompleted); + Assert.Empty(rig.Attempts); + } + finally + { + release.Set(); + await Phase(prune, "prune reconciled retry"); + } + + Assert.Equal(Period, await Phase(scheduled, "retry retained without changing deadline")); + var second = rig.Publish(1); + Assert.All(await Phase(Task.WhenAll(first, retry, second), "original contribution survives prune"), + receipt => Assert.Equal(new(true, Period), receipt)); + Assert.Equal(new[] { rig.Notification(0), rig.Notification(1) }, Assert.Single(rig.Attempts).Notifications); } [Theory] [InlineData(false)] [InlineData(true)] - public async Task MembershipWakeForceHandsOffPendingChunksWithoutCredits(bool previousDispatch) + public async Task NamespaceSettingsCallbacksRunOutsideBatcherLock(bool period) { await using var rig = new TestRig(); - var start = rig.Clock.GetTimestamp(); - rig.Options.MaxBatchItems = 2; - if (previousDispatch) + using var release = new ManualResetEventSlim(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var calls = 0; + Action callback = () => { - await rig.NotifyAsync(new KeyNotification("seed", 1, false)); - await rig.AdvanceToDispatchAsync(Milliseconds(25)); + if (Interlocked.Increment(ref calls) == 1) + { + entered.TrySetResult(); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), TestToken), "Namespace settings callback was not released."); + } + }; + if (period) + { + rig.Namespace.OnPeriodRead = callback; + } + else + { + rig.Namespace.OnOptionsRead = callback; } - await rig.NotifyAsync( - new("a", 2, true), new("b", 3, true), new("c", 4, true), new("d", 5, true), new("e", 6, true)); - var handoffTime = rig.Clock.GetTimestamp(); - - rig.Batcher.WakeForMembershipChange(); - - // Await every callback before calling Flush: otherwise Flush could conceal a missing - // membership wake or failure to bypass collection/pacing for later identity chunks. - var handoffs = new[] { await rig.NextAttemptAsync(), await rig.NextAttemptAsync(), await rig.NextAttemptAsync() }; - await Phase(rig.Batcher.FlushAsync(TestToken), "membership handoff reconciled"); - Assert.Equal(new[] { 2, 2, 1 }, handoffs.Select(static attempt => attempt.Notifications.Length)); - Assert.Equal(new long[] { 2, 3, 4, 5, 6 }, handoffs.SelectMany(static attempt => attempt.Notifications).Select(static n => n.Version)); - Assert.All(handoffs, attempt => Assert.Equal(handoffTime, attempt.Timestamp)); - Assert.Equal(previousDispatch ? Milliseconds(25) : TimeSpan.Zero, rig.Clock.GetElapsedTime(start, handoffTime)); + var admission = Task.Run(() => rig.Batcher.Notify([new("a", 1, true)]), TestToken); + try + { + await Phase(entered.Task, "namespace callback entered"); + await Phase(Task.Run(() => rig.Batcher.Prune(new HashSet()), TestToken), + "another thread enters batcher while namespace callback blocks"); + } + finally + { + release.Set(); + } - Assert.Equal(Milliseconds(200), await rig.NotifyAsync(new KeyNotification("later", 7, false))); - rig.Clock.Advance(Milliseconds(199)); - Assert.Equal(previousDispatch ? 4 : 3, rig.Attempts.Count); - var later = await rig.AdvanceToDispatchAsync(Milliseconds(1)); - Assert.Equal(new KeyNotification("later", 7, false), Assert.Single(later.Notifications)); + Assert.True(await Phase(admission, "namespace callback returned")); + await Phase(rig.Batcher.FlushAsync(TestToken), "namespace reentrancy work drained"); + Assert.Equal(new KeyNotification("a", 1, true), Assert.Single(Assert.Single(rig.Attempts).Notifications)); } [Fact] - public async Task MembershipWakeKeepsPartialHandoffRetriesPaced() + public async Task HighPriorityHintsBypassCollectionButFailedRetriesUsePublicationPeriod() { await using var rig = new TestRig(); + rig.Namespace.Options.Priority = DisseminationPriority.High; rig.OnDispatch = attempt => attempt.Number != 1; - await rig.NotifyAsync(new KeyNotification("a", 1, true)); - var first = await rig.AdvanceToDispatchAsync(Milliseconds(25), morePending: true); - rig.Clock.Advance(Milliseconds(20)); + var started = rig.Clock.GetTimestamp(); var scheduled = rig.Clock.WhenScheduled(); + Assert.True(rig.Batcher.Notify([new("a", 1, true)])); + Assert.Equal(started, (await rig.NextAttemptAsync()).Timestamp); + Assert.Equal(Period, await Phase(scheduled, "high priority retry bounded")); + rig.Clock.Advance(Period); + await Phase(rig.Batcher.FlushAsync(TestToken), "high priority retry accepted"); + Assert.Equal(2, rig.Attempts.Count); + } - rig.Batcher.WakeForMembershipChange(); - - Assert.Equal(Milliseconds(180), await Phase(scheduled, "membership handoff preserves the retry floor")); - rig.Clock.Advance(Milliseconds(179)); - Assert.Single(rig.Attempts); - var retry = await rig.AdvanceToDispatchAsync(Milliseconds(1)); - Assert.Equal(Milliseconds(200), rig.Clock.GetElapsedTime(first.Timestamp, retry.Timestamp)); - Assert.Equal(new KeyNotification("a", 1, true), Assert.Single(retry.Notifications)); + [Fact] + public async Task HighPriorityPublicationsStillRequireEveryProducerOrDeadline() + { + await using var rig = new TestRig(2); + rig.Namespace.Options.Priority = DisseminationPriority.High; + var first = await rig.StartPublicationAsync(0); + rig.Clock.Advance(Milliseconds(25)); + Assert.False(first.IsCompleted); + Assert.Empty(rig.Attempts); + var second = rig.Publish(1); + Assert.All(await Phase(Task.WhenAll(first, second), "priority cannot bypass the publication cohort"), + receipt => Assert.Equal(new(true, Milliseconds(975)), receipt)); + Assert.Equal(2, Assert.Single(rig.Attempts).Notifications.Length); } [Fact] - public async Task StopRejectsAdmissionAndForceDrainsBoundedChunksWithoutAdvancingTime() + public async Task StopSealsPartialCohortDrainsBoundedChunksAndRejectsFurtherAdmission() { - await using var rig = new TestRig(); + await using var rig = new TestRig(6); rig.Options.MaxBatchItems = 2; - await rig.NotifyAsync( - new("a", 1, true), new("b", 2, true), new("c", 3, true), new("d", 4, true), new("e", 5, true)); - var start = rig.Clock.GetTimestamp(); - - await Phase(rig.Batcher.StopAsync(TestToken), "accepted keys handed to peer queues"); + var receipts = new List> { await rig.StartPublicationAsync(0) }; + for (var i = 1; i < 5; i++) + { + receipts.Add(rig.Publish(i)); + } + await Phase(rig.Batcher.StopAsync(TestToken), "stop seals partial cohort"); + Assert.All(await Task.WhenAll(receipts), receipt => Assert.Equal(new(true, Period), receipt)); + Assert.Equal(new[] { 2, 2, 1 }, rig.Attempts.Select(attempt => attempt.Notifications.Length)); Assert.False(rig.Batcher.Notify([new("late", 6, true)])); - Assert.Equal(new[] { 2, 2, 1 }, rig.Attempts.Select(static attempt => attempt.Notifications.Length)); - Assert.Equal(new long[] { 1, 2, 3, 4, 5 }, rig.Attempts.SelectMany(static attempt => attempt.Notifications).Select(static n => n.Version)); - Assert.All(rig.Attempts, attempt => Assert.Equal(start, attempt.Timestamp)); + Assert.False((await rig.Publish(5)).Accepted); Assert.True(rig.Clock.TimerDisposed); await rig.Batcher.StopAsync(TestToken); } [Fact] - public async Task FlushForcesBoundedChunksWithoutStoppingAdmission() + public async Task FlushSealsPartialCohortWithoutStoppingLaterAdmission() { - await using var rig = new TestRig(); - rig.Options.MaxBatchItems = 2; - await rig.NotifyAsync(new("a", 1, true), new("b", 2, true), new("c", 3, true)); - var start = rig.Clock.GetTimestamp(); - - await Phase(rig.Batcher.FlushAsync(TestToken), "forced chunks handed to peer queues"); - - Assert.Equal(new[] { 2, 1 }, rig.Attempts.Select(static attempt => attempt.Notifications.Length)); - Assert.All(rig.Attempts, attempt => Assert.Equal(start, attempt.Timestamp)); - // Consume the two forced callbacks before awaiting the later, ordinarily paced callback. - await rig.NextAttemptAsync(); - await rig.NextAttemptAsync(); - Assert.Equal(Milliseconds(200), await rig.NotifyAsync(new KeyNotification("later", 4, true))); - var next = await rig.AdvanceToDispatchAsync(Milliseconds(200)); - Assert.Equal(new KeyNotification("later", 4, true), Assert.Single(next.Notifications)); + await using var rig = new TestRig(3); + var first = await rig.StartPublicationAsync(0); + rig.Clock.Advance(Milliseconds(25)); + await Phase(rig.Batcher.FlushAsync(TestToken), "flush seals partial cohort"); + Assert.Equal(new(true, Milliseconds(975)), await first); + Assert.False(rig.Clock.TimerDisposed); + rig.Clock.Advance(Milliseconds(975)); + var second = await rig.StartPublicationAsync(0, 2); + rig.Clock.Advance(Period); + Assert.Equal(new(true, TimeSpan.Zero), await Phase(second, "admission after flush")); + Assert.Equal(2, rig.Attempts.Count); } [Fact] @@ -614,61 +1057,91 @@ public async Task StopCancellationEndsRetryBudgetAndPreservesCallerToken() await using var rig = new TestRig(); using var cancellation = new CancellationTokenSource(); rig.OnDispatch = _ => false; - await rig.NotifyAsync(new KeyNotification("a", 1, true)); - var retryScheduled = rig.Clock.WhenScheduled(); + var publication = await rig.StartPublicationAsync(0); + var scheduled = rig.Clock.WhenScheduled(); var stop = rig.Batcher.StopAsync(cancellation.Token); - await rig.NextAttemptAsync(); - Assert.Equal(Milliseconds(200), await Phase(retryScheduled, "forced stop retry remains paced")); + Assert.False((await Phase(publication, "stop's failed attempt rejects receipt")).Accepted); + Assert.Equal(Period, await Phase(scheduled, "stop retry bounded")); Assert.False(stop.IsCompleted); - Assert.False(rig.Batcher.Notify([new("late", 2, true)])); - + Assert.False((await rig.Publish(1)).Accepted); cancellation.Cancel(); - - var failure = await Assert.ThrowsAnyAsync(() => Phase(stop, "stop caller budget observed")); - Assert.Equal(cancellation.Token, failure.CancellationToken); - var laterFailure = await Assert.ThrowsAnyAsync( - () => rig.Batcher.StopAsync(CancellationToken.None)); - Assert.Equal(cancellation.Token, laterFailure.CancellationToken); + var exception = await Assert.ThrowsAnyAsync(() => Phase(stop, "stop budget exhausted")); + Assert.Equal(cancellation.Token, exception.CancellationToken); + var later = await Assert.ThrowsAnyAsync(() => rig.Batcher.StopAsync(CancellationToken.None)); + Assert.Equal(cancellation.Token, later.CancellationToken); Assert.True(rig.Clock.TimerDisposed); rig.Clock.Advance(TimeSpan.FromDays(1)); Assert.Single(rig.Attempts); } [Fact] - public async Task PreCanceledStopRejectsAdmissionWithoutDispatch() + public async Task StopCancellationDuringDispatchRejectsWaitersAndDoesNotClaimNextChunk() { await using var rig = new TestRig(); using var cancellation = new CancellationTokenSource(); - await rig.NotifyAsync(new KeyNotification("a", 1, true)); - cancellation.Cancel(); + using var release = new ManualResetEventSlim(); + var token = TestToken; + rig.Options.MaxBatchItems = 1; + rig.OnDispatch = _ => + { + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Stop's in-flight callback was not released."); + return true; + }; + var first = await rig.StartPublicationAsync(0); + var second = rig.Publish(1); + var stop = rig.Batcher.StopAsync(cancellation.Token); + await rig.NextAttemptAsync(); + try + { + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync(() => Phase(stop, "stop callback budget")); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.All(await Phase(Task.WhenAll(first, second), "all in-flight and unclaimed receipts rejected"), + receipt => Assert.False(receipt.Accepted)); + } + finally + { + release.Set(); + } - var failure = await Assert.ThrowsAnyAsync( - () => rig.Batcher.StopAsync(cancellation.Token)); + await Assert.ThrowsAnyAsync(() => rig.Batcher.StopAsync(CancellationToken.None)); + Assert.Single(rig.Attempts); + Assert.True(rig.Clock.TimerDisposed); + } - Assert.Equal(cancellation.Token, failure.CancellationToken); - Assert.False(rig.Batcher.Notify([new("b", 2, true)])); + [Fact] + public async Task PreCanceledStopRejectsReceiptsWithoutDispatch() + { + await using var rig = new TestRig(); + using var cancellation = new CancellationTokenSource(); + var publication = await rig.StartPublicationAsync(0); + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync(() => rig.Batcher.StopAsync(cancellation.Token)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.False((await Phase(publication, "abandoned producer released")).Accepted); + Assert.False(rig.Batcher.Notify([new("late", 2, true)])); Assert.Empty(rig.Attempts); Assert.True(rig.Clock.TimerDisposed); } [Fact] - public async Task FlushCancellationLeavesAcceptedWorkAvailableForPacedRetry() + public async Task FlushCancellationLeavesAdmittedWorkForPeriodBoundedRetry() { await using var rig = new TestRig(); using var cancellation = new CancellationTokenSource(); rig.OnDispatch = attempt => attempt.Number != 1; - await rig.NotifyAsync(new KeyNotification("a", 1, true)); - var retryScheduled = rig.Clock.WhenScheduled(); + var publication = await rig.StartPublicationAsync(0, force: true); + var scheduled = rig.Clock.WhenScheduled(); var flush = rig.Batcher.FlushAsync(cancellation.Token); - await rig.NextAttemptAsync(); - Assert.Equal(Milliseconds(200), await Phase(retryScheduled, "force flush retry remains paced")); + Assert.False((await Phase(publication, "flush partial admission receipt")).Accepted); + Assert.Equal(Period, await Phase(scheduled, "flush retry bounded")); cancellation.Cancel(); - var failure = await Assert.ThrowsAnyAsync(() => flush); - Assert.Equal(cancellation.Token, failure.CancellationToken); - - var retry = await rig.AdvanceToDispatchAsync(Milliseconds(200)); - Assert.Equal(new KeyNotification("a", 1, true), Assert.Single(retry.Notifications)); - Assert.Equal(2, rig.Attempts.Count); + var exception = await Assert.ThrowsAnyAsync(() => flush); + Assert.Equal(cancellation.Token, exception.CancellationToken); + rig.Clock.Advance(Period); + await Phase(rig.Batcher.FlushAsync(TestToken), "admitted hint survives flush cancellation"); + Assert.Equal(new[] { rig.Notification(0, 1, true), rig.Notification(0, 1, true) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); } private static async Task Phase(Task task, string phase) @@ -696,46 +1169,52 @@ private sealed class TestRig : IAsyncDisposable private readonly Channel _attempts = Channel.CreateUnbounded(); private int _attemptNumber; - public TestRig() + public TestRig(int members = 3) { - Batcher = new(Clock, Namespace, new TestOptionsMonitor(Options), Dispatch, Logger); + Members = Enumerable.Range(0, members).Select(index => SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 18000 + index), 1)).ToImmutableArray(); + SetMembership(Members); + Batcher = new(Clock, Namespace, new TestOptionsMonitor(Options), () => OnMembershipRead?.Invoke() ?? Membership, Dispatch, Logger); } public TestClock Clock { get; } = new(); public TestNamespace Namespace { get; } = new(); public DisseminationOptions Options { get; } = new() { Enabled = true }; public TestLogger Logger { get; } = new(); + public ImmutableArray Members { get; } + public DisseminationMembershipSnapshot Membership { get; private set; } = null!; public DisseminationRootBatcher Batcher { get; } public ConcurrentQueue Attempts { get; } = new(); public Func? OnDispatch { get; set; } + public Func? OnMembershipRead { get; set; } - public async Task NotifyAsync(params KeyNotification[] notifications) + public void SetMembership(ImmutableArray members) => + Membership = new(new MembershipVersion((Membership?.MembershipVersion.Value ?? 0) + 1), + Members[0], members, Options.Overlay); + + public KeyNotification Notification(int producer, long version = 1, bool force = false) => + new(Members[producer], version, force); + + public Task Publish(int producer, long version = 1, bool force = false, CancellationToken? token = null) => + Batcher.PublishAsync(Notification(producer, version, force), token ?? TestToken).AsTask(); + + public async Task> StartPublicationAsync(int producer, long version = 1, bool force = false) + { + var scheduled = Clock.WhenScheduled(); + var publication = Publish(producer, version, force); + Assert.Equal(Period, await Phase(scheduled, "first contribution timer armed")); + return publication; + } + + public async Task NotifyAsync(params KeyNotification[] notifications) { var scheduled = Clock.WhenScheduled(); Assert.True(Batcher.Notify(notifications)); - return await Phase(scheduled, "notification timer armed"); + Assert.Equal(Period, await Phase(scheduled, "hint collection timer armed")); } public Task NextAttemptAsync() => Phase(_attempts.Reader.ReadAsync(TestToken).AsTask(), "dispatch callback entered"); - public async Task AdvanceToDispatchAsync(TimeSpan elapsed, bool morePending = false) - { - var scheduled = morePending ? Clock.WhenScheduled() : null; - Clock.Advance(elapsed); - var attempt = await NextAttemptAsync(); - if (scheduled is not null) - { - Assert.Equal(Milliseconds(200), await Phase(scheduled, $"next wave armed after attempt {attempt.Number}")); - } - else - { - await Phase(Batcher.FlushAsync(TestToken), $"attempt {attempt.Number} reconciled"); - } - - return attempt; - } - private bool Dispatch(ReadOnlySpan notifications) { var attempt = new Attempt(Interlocked.Increment(ref _attemptNumber), Clock.GetTimestamp(), notifications.ToArray()); @@ -762,27 +1241,66 @@ public async ValueTask DisposeAsync() private sealed class TestNamespace : IDisseminationNamespace { - public DisseminationNamespace Name => new("root-batcher-test"); - public ConcurrentQueue VersionReads { get; } = new(); - public Func? GetVersionHandler { get; set; } - public DisseminationNamespaceOptions Options { get; } = new() + private readonly DisseminationNamespaceOptions _options = new() { Enabled = true, MaxCoalescingDelay = TimeSpan.FromMilliseconds(25), MaxPendingItemCount = 8192, }; - public IEnumerable Digests => throw new InvalidOperationException("Root batching must not inspect namespace values."); + public DisseminationNamespace Name => new("root-batcher-test"); + public ConcurrentQueue VersionReads { get; } = new(); + public Func? GetVersionHandler { get; set; } + public Action? OnOptionsRead { get; set; } + public Action? OnPeriodRead { get; set; } + public bool MaterializeValues { get; set; } + public DisseminationNamespaceOptions Options + { + get + { + OnOptionsRead?.Invoke(); + return _options; + } + } + + public TimeSpan AggregationPeriod + { + get + { + OnPeriodRead?.Invoke(); + return Period; + } + } + + public IEnumerable Digests => throw new InvalidOperationException("Root cohorts must not inspect namespace values."); public long GetVersion(DisseminationKey key) { VersionReads.Enqueue(key); - return GetVersionHandler?.Invoke(key) ?? 0; + return GetVersionHandler?.Invoke(key) ?? (MaterializeValues ? 1 : 0); + } + + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) + { + if (!MaterializeValues) + { + throw new InvalidOperationException("Only peer queues may materialize payloads."); + } + + if (request.FromVersion >= 1) + { + return DisseminationRepairResult.Current(1); + } + + if (request.MaxBatchBytes < sizeof(long) || request.MaxPayloadBytes < sizeof(long) || request.MaxItemCount < 1) + { + return DisseminationRepairResult.InsufficientCapacity(1); + } + + return DisseminationRepairResult.Produced(1, [new(request.Key, 0, 1, BitConverter.GetBytes(1L))]); } - public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) => - throw new InvalidOperationException("Only peer queues may materialize payloads."); public ValueTask ApplyValueAsync(DisseminationValue value, CancellationToken cancellationToken) => - throw new InvalidOperationException("Root batching must not apply payloads."); + throw new InvalidOperationException("Root cohorts must not apply payloads."); } private sealed class TestOptionsMonitor(DisseminationOptions options) : IOptionsMonitor @@ -792,8 +1310,7 @@ private sealed class TestOptionsMonitor(DisseminationOptions options) : IOptions public IDisposable? OnChange(Action listener) => null; } - // The test is the sole fake-time driver. A schedule barrier completes only after the actual - // underlying timer was armed, so advancing time never races timer readiness. + // Exactly one fake-time driver advances time after a barrier proves the real timer was armed. private sealed class TestClock : TimeProvider { private readonly FakeTimeProvider _inner = new(); @@ -831,7 +1348,7 @@ public bool Change(TimeSpan dueTime, TimeSpan period) { if (dueTime > TimeSpan.Zero && Interlocked.Exchange(ref owner._failNextSchedule, 0) != 0) { - throw new InvalidOperationException("The root timer cannot schedule another wave."); + throw new InvalidOperationException("The root timer cannot schedule another cohort."); } var changed = inner.Change(dueTime, period); From 268ee7e15063a4b186eb9be958f81114fee6856b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 16 Sep 2026 20:33:03 -0700 Subject: [PATCH 41/50] fix(dissemination): preserve native publication completion during cancellation --- .../Placement/DeploymentLoadPublisher.cs | 43 +++++++-- ...seminationProtocolTests.CohortPublisher.cs | 88 ++++++++++++++++++- 2 files changed, 119 insertions(+), 12 deletions(-) diff --git a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs index 37a5dd4a3ca..9b65a7bbe5b 100644 --- a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs +++ b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs @@ -125,10 +125,14 @@ private async Task PublishStatisticsAndGetReceipt(Cancell cancellationToken.ThrowIfCancellationRequested(); // Startup and the interleaving timer can overlap. Join the pending sample instead of // submitting another source publication while its cohort is still open. - var publication = _publicationTask is { IsCompleted: false } pending - ? pending - : _publicationTask = PublishStatisticsCore(cancellationToken); - return await publication.WaitAsync(cancellationToken); + if (_publicationTask is { IsCompleted: false } pending) + { + return await AwaitSharedPublication(pending, cancellationToken); + } + + // Native operations own this caller's cancellation and may complete successfully + // after observing it. Only joining callers need an independently cancellable wait. + return await (_publicationTask = PublishStatisticsCore(cancellationToken)); } private async Task PublishStatisticsCore(CancellationToken cancellationToken) @@ -240,10 +244,32 @@ private async Task PublishStatisticsViaDissemination( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var publication = _disseminationPublicationTask is { IsCompleted: false } pending - ? pending - : _disseminationPublicationTask = PublishStatisticsViaDisseminationCore(myStats, cancellationToken); - return await publication.WaitAsync(cancellationToken); + if (_disseminationPublicationTask is { IsCompleted: false } pending) + { + return await AwaitSharedPublication(pending, cancellationToken); + } + + return await (_disseminationPublicationTask = PublishStatisticsViaDisseminationCore(myStats, cancellationToken)); + } + + private static async Task AwaitSharedPublication( + Task publication, + CancellationToken cancellationToken) + { + try + { + return await publication.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + if (publication.IsCompletedSuccessfully) + { + return await publication; + } + + cancellationToken.ThrowIfCancellationRequested(); + throw; + } } private async Task PublishStatisticsViaDisseminationCore( @@ -279,7 +305,6 @@ private async Task PublishStatisticsViaDisseminationCore( myStats.DateTime.Ticks, cancellation.Token); var receiptTimestamp = timeProvider.GetTimestamp(); - cancellationToken.ThrowIfCancellationRequested(); return new(receipt, timeProvider, receiptTimestamp); } catch (OperationCanceledException) when ( diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs index 58e29faa146..1db66153030 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs @@ -184,6 +184,86 @@ public async Task CohortPublisherCallerCancellationPreservesIdentityWithoutFallb Assert.Equal(0, harness.Dissemination.QueryCount); } + [Fact] + public async Task CohortPublisherOwningCancellationPreservesNativeDirectCompletion() + { + await using var harness = new CohortPublisherHarness(refreshTime: TimeSpan.Zero); + using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var directStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cancellationObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.DirectHandler = async token => + { + using var registration = token.Register(() => cancellationObserved.TrySetResult()); + directStarted.TrySetResult(token); + await cancellationObserved.Task; + }; + var owning = harness.Publisher.RunOrQueueTask( + async token => + { + await harness.Publisher.PublishStatistics(token); + return true; + }, + caller.Token); + try + { + var nativeToken = await directStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var joining = harness.Publisher.RunOrQueueTask( + async token => + { + await harness.Publisher.PublishStatistics(token); + return true; + }, + TestContext.Current.CancellationToken); + await harness.OwnerBarrierAsync(); + var nativeSample = harness.Publisher.LocalRuntimeStatistics; + Assert.False(owning.IsCompleted); + Assert.False(joining.IsCompleted); + Assert.Single(harness.DirectUpdates); + Assert.Equal(caller.Token, nativeToken); + + caller.Cancel(); + await Task.WhenAll(owning, joining).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(owning.IsCompletedSuccessfully); + Assert.True(joining.IsCompletedSuccessfully); + Assert.Same(nativeSample, harness.Publisher.LocalRuntimeStatistics); + Assert.Same(nativeSample, harness.Publisher.PeriodicStatistics[harness.Publisher.Silo]); + Assert.Single(harness.DirectUpdates); + Assert.Empty(harness.Dissemination.Publications); + } + finally + { + caller.Cancel(); + } + } + + [Fact] + public async Task CohortPublisherOwningCancellationPreservesNativeReceiptCompletion() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + harness.Dissemination.PreserveNativeCompletion = true; + using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var owning = harness.Publisher.RunOrQueueTask( + async token => + { + await harness.Publisher.PublishStatistics(token); + return true; + }, + caller.Token); + var publication = await harness.Dissemination.NextPublicationAsync(); + caller.Cancel(); + publication.Receipt.SetResult(new(true, harness.Period)); + await owning.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(owning.IsCompletedSuccessfully); + Assert.True(publication.Token.IsCancellationRequested); + Assert.Single(harness.Dissemination.Publications); + Assert.Equal(1, harness.Dissemination.QueryCount); + Assert.Empty(harness.DirectUpdates); + } + [Fact] public async Task CohortPublisherStartupReceiptWaitKeepsInboundOwnerApplicationLive() { @@ -317,7 +397,7 @@ private sealed class CohortPublisherHarness : IAsyncDisposable { private readonly ServiceProvider _serializerServices; - public CohortPublisherHarness() + public CohortPublisherHarness(TimeSpan? refreshTime = null) { var local = CreateSilo(40501); Peer = CreateSilo(40502); @@ -325,7 +405,7 @@ public CohortPublisherHarness() statusOracle.SetStatus(local, SiloStatus.Active); statusOracle.SetStatus(Peer, SiloStatus.Active); var details = new FakeLocalSiloDetails(local); - Options = new() { DeploymentLoadPublisherRefreshTime = Period }; + Options = new() { DeploymentLoadPublisherRefreshTime = refreshTime ?? Period }; Options.Dissemination.Enabled = true; _serializerServices = new ServiceCollection().AddSerializer().BuildServiceProvider(); TimerClock = new(Clock); @@ -453,6 +533,7 @@ private sealed class CohortPublisherDisseminationService : IDisseminationService public ConcurrentQueue Publications { get; } = new(); public DisseminationPublicationReceipt? AutomaticReceipt { get; set; } = new(true, TimeSpan.FromSeconds(1)); + public bool PreserveNativeCompletion { get; set; } public Exception? Failure { get; set; } public IReadOnlyList UnconfirmedPeers { get; set; } = []; public int QueryCount { get; set; } @@ -475,7 +556,8 @@ public ValueTask PublishAggregated( publication.Receipt.SetResult(receipt); } - var result = new ValueTask(publication.Receipt.Task.WaitAsync(cancellationToken)); + var result = new ValueTask( + PreserveNativeCompletion ? publication.Receipt.Task : publication.Receipt.Task.WaitAsync(cancellationToken)); _publications.Writer.TryWrite(publication); return result; } From 6024ad190132213058935a16d40d8e666bbc3337 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 17 Sep 2026 00:43:23 -0700 Subject: [PATCH 42/50] refactor(dissemination): specialize repair and scheduling for runtime state --- .../docs/host/monitoring/metrics-catalog.md | 2 +- .../host/monitoring/runtime-dissemination.md | 4 +- .../implementation/runtime-dissemination.md | 38 +- .../IDs/SiloAddress.cs | 9 - .../Options/ClusterMembershipOptions.cs | 9 +- .../Options/DisseminationOptions.cs | 41 - .../IDisseminationSystemTarget.cs | 2 +- .../Options/DisseminationOptionsValidator.cs | 9 - ...entLoadStatisticsDisseminationNamespace.cs | 12 +- .../DisseminationBroadcastQueue.cs | 323 +-- .../Dissemination/DisseminationEvents.cs | 8 +- .../Dissemination/DisseminationInstruments.cs | 58 +- .../DisseminationProtocol.Cohorts.cs | 6 +- .../Dissemination/DisseminationProtocol.cs | 322 +-- .../Dissemination/DisseminationRootBatcher.cs | 49 +- .../Dissemination/IDisseminationNamespace.cs | 42 +- .../MembershipDisseminationNamespace.cs | 353 +-- .../Dissemination/WakeTimer.cs | 19 +- .../Placement/DeploymentLoadPublisher.cs | 41 +- .../Orleans.Core.Abstractions.cs | 4 +- src/api/Orleans.Core/Orleans.Core.cs | 10 - .../README.md | 2 +- .../Silo/NewRuntime.cs | 30 +- .../Tests/VersionSkewTests.cs | 16 +- .../Runtime/DeploymentLoadPublisherTests.cs | 57 +- .../DisseminationAggregationOptionsTests.cs | 2 - .../DisseminationMembershipSnapshotTests.cs | 64 +- .../DisseminationProtocolTests.Aggregation.cs | 79 +- ...seminationProtocolTests.CohortPublisher.cs | 73 +- .../DisseminationProtocolTests.Cohorts.cs | 66 +- ...isseminationProtocolTests.FinalComments.cs | 16 +- ...sseminationProtocolTests.FollowupReview.cs | 28 +- ...seminationProtocolTests.MembershipReset.cs | 19 +- ...eminationProtocolTests.MembershipReview.cs | 175 +- ...eminationProtocolTests.MetricBoundaries.cs | 4 +- ...minationProtocolTests.PerformanceReview.cs | 4 +- ...sseminationProtocolTests.ProtocolReview.cs | 114 +- ...seminationProtocolTests.PumpDiagnostics.cs | 8 +- .../DisseminationProtocolTests.QueueReview.cs | 32 +- .../DisseminationProtocolTests.cs | 2274 +++++------------ .../DisseminationRootBatcherTests.cs | 81 +- .../Dissemination/WakeTimerTests.cs | 55 +- 42 files changed, 1199 insertions(+), 3361 deletions(-) diff --git a/docs/site/src/content/docs/host/monitoring/metrics-catalog.md b/docs/site/src/content/docs/host/monitoring/metrics-catalog.md index e729b784ee7..f745ae32608 100644 --- a/docs/site/src/content/docs/host/monitoring/metrics-catalog.md +++ b/docs/site/src/content/docs/host/monitoring/metrics-catalog.md @@ -74,7 +74,7 @@ Request latency covers the interval until the caller's callback completes, inclu | `orleans-dissemination-anti-entropy-failures` | C | `operations` | `reason` | Outbound anti-entropy peer operations which ended in `timeout` or `error`. | | `orleans-dissemination-anti-entropy-values` | C | `values` | `direction` | Repair values returned by successful anti-entropy exchanges. | | `orleans-dissemination-broadcast-received` | C | `messages` | `namespace`, `kind` | Broadcast batches received, counted once for each namespace represented in the batch. | -| `orleans-dissemination-broadcast-scheduled` | C | `schedules` | `reason` | Per-peer pump schedules split into `immediate`, `coalesce`, `retry`, and `priority`. | +| `orleans-dissemination-broadcast-scheduled` | C | `schedules` | `reason` | Per-peer pump schedules split into `immediate` and `retry`. | | `orleans-dissemination-broadcast-send-failures` | C | `attempts` | `reason` | Broadcast send attempts which ended in `timeout` or `error`. | | `orleans-dissemination-broadcast-sent` | C | `messages` | `namespace`, `kind` | Successfully completed broadcast sends, counted once for each namespace represented in the batch. | | `orleans-dissemination-bytes-sent` | C | `bytes` | `namespace`, `kind` | Serialized dissemination payload bytes in successful broadcasts. | diff --git a/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md b/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md index e5abfa76150..2afae48fb15 100644 --- a/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md +++ b/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md @@ -67,7 +67,7 @@ Retry scheduling metrics describe timer decisions. The send-gate diagnostic iden Identify the `namespace` and compare update cardinality with . Increase the limit only after confirming the process has memory headroom and the destination can drain work. Broadcast item and byte limits still bound each transmission. -Capacity planning must include all retained peer/namespace pairs: pending-key storage scales with their summed per-peer limits, and acknowledgment ledgers scale with known keys. Active local attempts materialize at most one batch per broadcast slot, bounded by . Namespace history, metadata, and messaging buffers retained by pending RPCs remain additional costs. +Capacity planning must include all retained peer/namespace pairs: pending-key storage scales with their summed per-peer limits, and acknowledgment ledgers scale with known keys. Active local attempts materialize at most one batch per broadcast slot, bounded by . Cached namespace payloads, metadata, and messaging buffers retained by pending RPCs remain additional costs. ### Anti-entropy remains truncated @@ -85,7 +85,7 @@ The `Microsoft.Orleans.Dissemination` diagnostic listener exposes detailed, opt- |---|---| | `Dissemination.ValueApply` | Inspect one value's namespace, key, version transition, peer, result, and payload size. | | `Dissemination.PayloadDrop` | Inspect a value rejected by a payload guard. | -| `Dissemination.BroadcastScheduled` | Observe immediate, coalesced, priority, and retry scheduling with due time and attempt. | +| `Dissemination.BroadcastScheduled` | Observe immediate and retry scheduling with due time and attempt. | | `Dissemination.SendGate` | Observe broadcast FIFO entry (`Kind=broadcast`, `Stage=queued`) or a local repair attempt releasing its admission (`Kind=repair`, `Stage=released`), including timeout and cancellation. | | `Dissemination.QueueAdmissionRejected` | Observe the peer, namespace, configured limit, and bounded rejection reason. | diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index 0ccb78325b0..c153d8b8eba 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -12,7 +12,7 @@ Runtime dissemination is an internal silo-to-silo protocol which accelerates con The dissemination protocol currently carries: - deployment-load statistics among Active silos; -- membership snapshots and diffs among Joining, Active, ShuttingDown, and Stopping silos. +- membership snapshots among Joining, Active, ShuttingDown, and Stopping silos. The deployment-load publisher and membership manager retain their direct delivery paths and validation rules. Tree broadcast and anti-entropy form one convergence feature: broadcast accelerates new updates, and repair discovers and delivers state missed by the tree. @@ -34,7 +34,7 @@ flowchart LR Repair --> Authority ``` -`DisseminationProtocol` coordinates routing, peer capability evidence, anti-entropy, and application isolation. `DisseminationRootBatcher` owns the aggregation root's bounded latest-key cohorts, producer contributions, and seal receipts. `DisseminationBroadcastQueue` owns per-peer scheduling, retry, drain, and acknowledged-version ledgers. `DisseminationMembership` projects one membership snapshot into topology-specific member sets. Each `IDisseminationNamespace` owns serialization, current versions, retained history, payload limits, repair construction, and application semantics. +`DisseminationProtocol` coordinates routing, peer capability evidence, anti-entropy, and application isolation. `DisseminationRootBatcher` owns the aggregation root's bounded latest-key cohorts, producer contributions, and seal receipts. `DisseminationBroadcastQueue` owns per-peer scheduling, retry, drain, and acknowledged-version ledgers. `DisseminationMembership` projects one membership snapshot into topology-specific member sets. Each `IDisseminationNamespace` owns serialization, current versions, payload limits, repair construction, and application semantics. Queue entries contain `(namespace, key)` identities. A pump acquires both destination ownership and a global broadcast slot before materializing a repair. Coalesced notifications therefore use the latest namespace state and acknowledged baseline after admission, without retaining serialized payloads for waiting peers. @@ -44,25 +44,24 @@ Applied state wakes each forwarding child, including same-version membership liv ## Version and application invariants -A `DisseminationValue` names one key and a `[FromVersion, ToVersion]` transition: +A `DisseminationValue` carries one key's full current state and version: -- a full value starts at version zero and can establish state without a retained baseline; -- a delta starts at the receiver baseline and must form a contiguous version chain; +- its zero `FromVersion` identifies a full value which can establish state at any receiver baseline; - acknowledged peer versions advance monotonically from explicit receiver evidence; - duplicate and obsolete values leave authoritative state unchanged; and - one rejected value does not prevent later values in the batch from being considered. -The namespace reports whether a repair is current, produced, unavailable, or unable to fit the supplied item and byte budgets. Produced repairs can be complete or a valid prefix. Prefix acknowledgments establish the receiver's new baseline and let the next batch continue the chain. +The namespace reports whether the peer is current, a full value was produced, the key is unavailable, or the value exceeds the supplied byte budgets. Batch assembly reserves an item slot before requesting a value and retains unsent keys for subsequent bounded batches. -Broadcast senders advertise support for compact acknowledgments. Receive selection explicitly reports whether the whole batch passed validation. When it does and each key's current version equals the maximum version transmitted for that key, one response flag certifies those versions. Namespace entries retain capability evidence with empty per-key lists. The sender advances each ledger to its captured transmitted version, including when that batch is only a prefix of a larger repair. Receivers report explicit versions for older callers, truncated batches, and differing receiver versions, preserving precise repair baselines across version skew. +Broadcast senders advertise support for compact acknowledgments. Receive selection explicitly reports whether the whole batch passed validation. When it does and each key's current version equals the maximum version transmitted for that key, one response flag certifies those versions. Namespace entries retain capability evidence with empty per-key lists. The sender advances each ledger to its captured transmitted version. Receivers report explicit versions for older callers, truncated batches, and differing receiver versions. -Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. It retains 32 snapshots and prefers a smaller diff when a lower peer baseline is present. Other baselines, including numerically greater peer versions, receive full snapshots. Current diffs identify their entries as the complete target inventory, so table cleanup is reflected even when a receiver retained a different same-version baseline. Older partial-diff payloads retain their incremental semantics. Same-version peer snapshots can advance liveness or remove inactive entries while preserving every Active entry and every retained versioned field, including status, suspect votes, endpoint metadata, and start time. Authoritative same-version table reads preserve previously accepted versioned fields while merging heartbeat progress, accommodating provider timestamp rounding. Dissemination reports whether the membership manager actually published changed state. +Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. Repair sends the current full snapshot and caches its serialized payload until the owner changes state. The complete inventory reconciles table cleanup across peers with different same-version baselines. Same-version peer snapshots can advance liveness or remove inactive entries while preserving every Active entry and every retained versioned field, including status, suspect votes, endpoint metadata, and start time. Authoritative same-version table reads preserve previously accepted versioned fields while merging heartbeat progress, accommodating provider timestamp rounding. Dissemination reports whether the membership manager actually published changed state. -Development clustering recreates its volatile membership table when the primary restarts. Its manager treats received membership snapshots as notifications to perform a fresh authoritative read, so retained peer state from the prior table cannot replace the recreated table's view. Durable membership providers retain snapshot-based application. Dissemination records current owner state rather than delayed publication arguments, and clears retained history and serialized payloads when the owner replaces its view with a lower version. +Development clustering recreates its volatile membership table when the primary restarts. Its manager treats received membership snapshots as notifications to perform a fresh authoritative read, so retained peer state from the prior table cannot replace the recreated table's view. Durable membership providers retain snapshot-based application. Repair re-reads the current owner under the payload-cache lock, including after a delayed publication or an owner-authorized version replacement. If an existing development silo observes a table counter below the version it knew when it started the read, it requests local shutdown and reports that it must restart against the recreated table. This preserves the monotonic membership-version contract shared with directory and other runtime consumers. A read which began before a newer local observation can finish normally; its older result is ignored. Restart the affected silos after recreating development membership. -The membership namespace requests authoritative validation for older full values. Broadcast and anti-entropy let those values reach the owner, and repair selection consults the namespace for numerically greater peer baselines. The owner decides the resulting state; delta chains and monotonic namespaces retain their numeric version guards. +The membership namespace requests authoritative validation for older full values. Broadcast and anti-entropy let those values reach the owner, and repair selection consults the namespace for numerically greater peer baselines. The owner decides the resulting state. Deployment-load samples retain their timestamp ordering. Deployment-load versions are sample timestamps. Each repair is a full latest value. Application runs on the deployment-load publisher's scheduler and accepts samples only for Active silo generations in the membership oracle. Terminal membership transitions remove placement statistics, and the oracle's monotonic membership state rejects later samples for departed generations. @@ -116,14 +115,13 @@ Dissemination is opt-in. Enable | 3 | Maximum peers selected in one repair round and independent per-silo active local repair attempt limit. | | / | 8,192 / 1 MiB | Independent repair limits, capped by the global batch limits and negotiated with the peer. | | | 1,024; load uses 8,192 | Distinct retained keys per namespace and peer, and in a root's pending set. | -| | 100 ms | Normal-priority batching window. | | | 30 seconds | Independent local transport and application budgets for each hop. | | | 10 seconds | Quiet period before a digest is offered for repair. | | | 1 MiB | Maximum serialized value size for the namespace. | Each integration has its own . Operators can enable and tune membership and deployment-load dissemination independently while retaining the local concurrency and per-message bounds. -Deployment load samples use the configured publication period, with receipt feedback aligning subsequent samples to cohort boundaries. The expected repair-update cadence remains five seconds. Producer ingress and relay forwarding send immediately; root receipts follow cohort sealing. The publication receipt budget is twice the publication period, capped by the platform timer limit: one period for collection and one for transport and scheduling. Ordinary RPC timeouts also apply. Caller cancellation stops the publication; an unavailable or rejecting root, receipt expiry, or a shorter RPC timeout selects the direct delivery path. Membership retains high priority and bypasses coalescing. Configure the load pending-key bound to cover the intended active inventory and stalls; each newer notification replaces the retained value for that key rather than adding another sample. +Deployment load samples use the configured publication period, with receipt feedback aligning subsequent samples to cohort boundaries. The expected repair-update cadence remains five seconds. Producer ingress and relay forwarding send immediately; root receipts follow cohort sealing. The publication receipt budget is twice the publication period, capped by the platform timer limit: one period for collection and one for transport and scheduling. Ordinary RPC timeouts also apply. Caller cancellation stops the publication; an unavailable or rejecting root, receipt expiry, or a shorter RPC timeout selects the direct delivery path. Membership has fixed high priority. Configure the load pending-key bound to cover the intended active inventory and stalls; each newer notification replaces the retained value for that key rather than adding another sample. Inventory and peer-ledger maintenance runs on membership changes and bounded one-second maintenance opportunities instead of every received update. Failed passes remain eligible for retry. This keeps full inventory scans out of the normal per-message root path while promptly retiring peers when the routing view changes. @@ -137,21 +135,21 @@ The `Dissemination compatibility` workflow builds the pre-feature runtime and cu Four- and eight-silo partition tests cover all three runtime modes. Healing drains affected connections at both endpoints, including unfinished handshakes, then confirms acknowledged bidirectional control calls before offering one publication round. Convergence requires exact load inventory and values. Recovery diagnostics include connection draining and readiness in elapsed time. -Additional tests isolate anti-entropy from tree delivery, direct gossip, and membership-table reads to establish history-miss and same-version heartbeat repair. Cancellation and partitioned shutdown have explicit completion bounds. Artifacts retain test results, binary manifests, process logs, and failure snapshots for diagnosis. Performance comparisons use separately maintained, explicitly invoked tooling. +Additional tests isolate anti-entropy from tree delivery, direct gossip, and membership-table reads to establish snapshot and same-version heartbeat repair. Cancellation and partitioned shutdown have explicit completion bounds. Artifacts retain test results, binary manifests, process logs, and failure snapshots for diagnosis. Performance comparisons use separately maintained, explicitly invoked tooling. ## Broadcast pumps and backpressure -Each destination has one independent pump. Repeated notifications coalesce by namespace and key, high-priority namespaces bypass the coalescing window, and batches obey global item and byte bounds plus namespace payload bounds. An owner-wide managed admission gate combines the local broadcast limit with one active local broadcast attempt per destination. +Each destination has one independent pump. New notifications schedule it immediately and replace pending notifications for the same namespace and key. Membership values are ordered first within a batch. Batches obey global item and byte bounds plus namespace payload bounds. An owner-wide managed admission gate combines the local broadcast limit with one active local broadcast attempt per destination. -Admission uses an explicit FIFO linked list. Releasing a batch admits the oldest ready destinations, skipping peers with an active local attempt. A peer with a backlog or acknowledged repair prefix rejoins admission for every batch, behind other ready peers. Priority changes when a pump becomes ready, rather than allowing it to overtake already-ready peers. Destination ownership uses the full `SiloAddress`, including generation, and ends with the local attempt. +Admission uses an explicit FIFO linked list. Releasing a batch admits the oldest ready destinations, skipping peers with an active local attempt. Backlogged peers rejoin admission for every batch, behind other ready peers. New notifications schedule their destination immediately while retaining its FIFO position. Destination ownership uses the full `SiloAddress`, including generation, and ends with the local attempt. `MaxPendingItemCount` bounds distinct retained keys per namespace and peer. A notification for an admitted key refreshes its generation at the limit. A new key is rejected until acknowledgment-driven completion, an oversized repair, or membership pruning releases capacity. Oversized repairs retain acknowledged peer versions separately so a later publication can resume from the established baseline. The runtime emits a rejection metric and a diagnostic event without including the key. A rejected tree target makes publication return `false`, allowing the producer's direct path to deliver the update. The aggregate pending-key bound is topology-dependent: sum `MaxPendingItemCount` over the retained peer/namespace pairs, with identity-only snapshots for active flushes. Acknowledged-version ledgers additionally scale with known namespace keys and retained peers and are pruned against membership and the authoritative key inventory. -With fixed configuration, active local broadcast attempts materialize at most `MaxConcurrentSends` batches, each bounded by `MaxBatchBytes`. Waiting peers retain identities until admission. Namespace-owned current values and repair history, identity and acknowledgment metadata, and messaging-layer buffers for pending RPCs are separate memory costs. Broadcast and aggregate repair concurrency limits are captured when the protocol is constructed; tune them before silo startup. +With fixed configuration, active local broadcast attempts materialize at most `MaxConcurrentSends` batches, each bounded by `MaxBatchBytes`. Waiting peers retain identities until admission. Namespace-owned current values, identity and acknowledgment metadata, and messaging-layer buffers for pending RPCs are separate memory costs. Broadcast and aggregate repair concurrency limits are captured when the protocol is constructed; tune them before silo startup. -A successful RPC is transport completion. The receiver response is the evidence which advances the peer ledger. Missing or prefix acknowledgments retain work for repair. Transport timeout and failure requeue the generation with exponential backoff capped by the anti-entropy interval. A newer notification can replace that schedule with its own urgency. +A successful RPC is transport completion. The receiver response is the evidence which advances the peer ledger. Missing or insufficient acknowledgments retain work for repair. Transport timeout and failure requeue the generation with exponential backoff capped by the anti-entropy interval. A newer notification schedules an immediate attempt. The transport hop lifetime starts after admission. Local completion, timeout, or pump cancellation releases both the broadcast slot and destination ownership. Timeout signals the RPC's cancellation token and requeues the generation with backoff, allowing subsequent attempts and other ready destinations to proceed while the earlier RPC completes under the messaging runtime's lifetime rules. Pump cancellation removes pending admission requests; a concurrent grant is either used by the caller or released exactly once. @@ -167,15 +165,15 @@ Repair has its own admission gate, independent of broadcast slots. Across overla Digests identify namespace-owned keys, including missing state at version zero, so repair can discover work rejected during queue admission. Recently advanced streams suppress redundant probes until `ExpectedUpdateCadence` elapses. Responses obey item, batch-byte, payload-byte, and hop-lifetime bounds. Persistent per-requester cursors rotate truncated responses so hot early keys cannot starve later candidates. Cursor state for non-members is least-recently-used and bounded to 64 entries. -Requests advertise the receiver's item and payload-byte limits. Responders use the smaller local and advertised limits, allowing their response cursor to provide fair service even when peers use different batch sizes. Older requests use the responder's own limits. Receivers independently cap incoming broadcasts and repair responses before grouping or applying values. Broadcast acknowledgments cover only the admitted keys at their actual local versions. Receive cursors advance through bounded ordered intervals for oversized deliveries while preserving the order of each repair chain. Cursor state is scoped to peer and direction, with at most 64 retained non-member cursors. Request digest inventory remains separate from these response admission budgets. +Requests advertise the receiver's item and payload-byte limits. Responders use the smaller local and advertised limits, allowing their response cursor to provide fair service even when peers use different batch sizes. Older requests use the responder's own limits. Receivers independently cap incoming broadcasts and repair responses before applying values. Broadcast acknowledgments cover only the admitted keys at their actual local versions. Receive cursors advance through bounded ordered intervals for oversized deliveries. Cursor state is scoped to peer and direction, with at most 64 retained non-member cursors. Request digest inventory remains separate from these response admission budgets. -Repairs from each sender remain ordered. Competing sender chains are ranked before application, and the namespace's monotonic apply contract rejects an obsolete or incompatible chain. Responses which completed successfully within their local attempt are still applied when another peer exceeds the round deadline. Late RPC faults are observed, and subsequent rounds repair responses which missed the application window. During a partition, local attempts remain bounded and retries follow the configured cadence; after connectivity and membership views recover, rotating peer selection and cursors continue convergence. +Each selected full value is offered to its namespace. This preserves complementary same-version membership heartbeat advances from different peers; the owner merges them and classifies duplicates or obsolete state. Values with a nonzero baseline or nonpositive destination version are rejected before application. Responses which completed successfully within their local attempt are still applied when another peer exceeds the round deadline. Late RPC faults are observed, and subsequent rounds repair responses which missed the application window. During a partition, local attempts remain bounded and retries follow the configured cadence; after connectivity and membership views recover, rotating peer selection and cursors continue convergence. ## Mixed-version behavior Peer namespace support is evidence-driven. Inbound traffic, broadcast acknowledgments, and authoritative anti-entropy responses confirm support for one silo generation. An explicit unsupported response revokes it and retires the publication generations covered by that flush. Publications admitted during the send remain queued with a fresh peer baseline. Membership pruning removes evidence for departed generations. -Deployment-load publication uses the aggregation tree once all captured Active peers confirm namespace support. During bootstrap or mixed-version operation, direct fanout covers all Active peers so that an unsupported intermediate node cannot separate otherwise capable participants. Direct fanout also applies if dissemination is disabled, unavailable, rejected, or unable to accept the update within one refresh interval. Membership direct gossip begins before optional dissemination and keeps the caller's cancellation and shutdown deadline. +Deployment-load publication uses the aggregation tree once all captured Active peers confirm namespace support. During bootstrap or mixed-version operation, direct fanout covers all Active peers so that an unsupported intermediate node cannot separate otherwise capable participants. Direct fanout also applies if dissemination is disabled, unavailable, rejected, or exceeds the publication receipt budget. Membership direct gossip begins before optional dissemination and keeps the caller's cancellation and shutdown deadline. ## Concurrency and lifetime diff --git a/src/Orleans.Core.Abstractions/IDs/SiloAddress.cs b/src/Orleans.Core.Abstractions/IDs/SiloAddress.cs index 48f8e2f9ac0..23b13db9a68 100644 --- a/src/Orleans.Core.Abstractions/IDs/SiloAddress.cs +++ b/src/Orleans.Core.Abstractions/IDs/SiloAddress.cs @@ -24,7 +24,6 @@ namespace Orleans.Runtime public sealed class SiloAddress : IEquatable, IComparable, - IComparable, ISpanFormattable, IParsable, IUtf8SpanParsable @@ -567,14 +566,6 @@ public int CompareTo(SiloAddress? other) return IPAddressComparer.Instance.Compare(Endpoint.Address, other.Endpoint.Address); } - - int IComparable.CompareTo(object? obj) - { - if (obj is null) return 1; - return obj is SiloAddress other - ? CompareTo(other) - : throw new ArgumentException($"Object must be of type {nameof(SiloAddress)}.", nameof(obj)); - } } /// diff --git a/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs b/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs index 9a53a1c6a44..bce7fab91f8 100644 --- a/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs +++ b/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs @@ -101,14 +101,7 @@ public TimeSpan MaxProbeTimeout /// best-effort accelerator. Direct membership gossip delivers shutdown-critical updates and reaches peers /// during mixed-version operation. /// - public DisseminationNamespaceOptions Dissemination { get; set; } = new() - { - ExpectedUpdateCadence = TimeSpan.FromSeconds(10), - - // Membership changes must propagate as quickly as possible, ahead of and without waiting to coalesce - // with other gossip. - Priority = DisseminationPriority.High, - }; + public DisseminationNamespaceOptions Dissemination { get; set; } = new(); /// /// Gets or sets the number of silos each silo probes for liveness. diff --git a/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs index 13cadec9e6f..13ba4aa991b 100644 --- a/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs +++ b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs @@ -159,16 +159,6 @@ public sealed class DisseminationNamespaceOptions /// . Enable each participating namespace explicitly. public bool Enabled { get; set; } - /// - /// Gets or sets the dissemination priority for this namespace. - /// - /// - /// namespaces bypass the coalescing window and are placed ahead of - /// lower-priority namespaces within each per-peer batch, so their updates are disseminated as quickly as - /// possible. is not applied to high-priority namespaces. - /// - public DisseminationPriority Priority { get; set; } = DisseminationPriority.Normal; - /// /// Gets or sets the hard maximum number of distinct pending keys retained for this namespace by each peer. /// @@ -180,19 +170,6 @@ public sealed class DisseminationNamespaceOptions /// public int MaxPendingItemCount { get; set; } = 1024; - /// - /// Gets or sets the maximum delay for namespace coalescing. - /// - /// - /// Per-peer batches can contain values from multiple namespaces and use the shortest configured delay among - /// enabled namespaces. High-priority namespaces (see ) do not coalesce and are excluded - /// from this calculation. - /// Deployment-load aggregation uses its publisher's refresh interval as the cohort deadline. - /// Producer ingress and distribution relays send immediately. Batch limits can split a cohort into multiple wire messages. - /// - /// The delay is 100 milliseconds by default and must be between 1 millisecond and approximately 49.7 days. - public TimeSpan MaxCoalescingDelay { get; set; } = TimeSpan.FromMilliseconds(100); - /// /// Gets or sets the local transport and application budgets for a namespace value. /// @@ -220,21 +197,3 @@ public sealed class DisseminationNamespaceOptions /// public int MaxPayloadBytes { get; set; } = 1024 * 1024; } - -/// -/// Describes how urgently a dissemination namespace's updates are broadcast relative to other namespaces. -/// -public enum DisseminationPriority -{ - /// - /// Updates are coalesced within the namespace's - /// and are sent after any higher-priority namespaces. - /// - Normal, - - /// - /// Updates bypass the coalescing window, are sent immediately, and are placed ahead of normal-priority - /// namespaces within each per-peer batch. - /// - High, -} diff --git a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs index cd0544e3485..a4ff1aba8a0 100644 --- a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs +++ b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs @@ -215,7 +215,7 @@ public DigestEntry(DisseminationKey key, long version, long fingerprint = 0) public long Fingerprint { get; } } -// FromVersion zero denotes a full value which can replace any baseline; nonzero ranges must form a chain. +// Values carry full state. FromVersion remains zero on the wire. [GenerateSerializer, Immutable] internal readonly struct DisseminationValue { diff --git a/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs b/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs index 9600121f3d5..771427c2517 100644 --- a/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs +++ b/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs @@ -73,8 +73,6 @@ public ValidateOptionsResult Validate(string? name, DisseminationOptions options internal sealed class DisseminationNamespaceOptionsValidator { - private static readonly TimeSpan MaxTimerPeriod = TimeSpan.FromMilliseconds(uint.MaxValue - 1); - public static ValidateOptionsResult Validate(string owner, DisseminationNamespaceOptions options) { if (options.MaxPendingItemCount <= 0) @@ -82,13 +80,6 @@ public static ValidateOptionsResult Validate(string owner, DisseminationNamespac return ValidateOptionsResult.Fail($"{owner}.{nameof(DisseminationNamespaceOptions.MaxPendingItemCount)} must be greater than 0."); } - if (options.MaxCoalescingDelay < TimeSpan.FromMilliseconds(1) - || options.MaxCoalescingDelay > MaxTimerPeriod) - { - return ValidateOptionsResult.Fail( - $"{owner}.{nameof(DisseminationNamespaceOptions.MaxCoalescingDelay)} must be between 1 millisecond and {MaxTimerPeriod}."); - } - if (options.StaleItemTtl <= TimeSpan.Zero) { return ValidateOptionsResult.Fail($"{owner}.{nameof(DisseminationNamespaceOptions.StaleItemTtl)} must be greater than 0."); diff --git a/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs index 9d7da5cca32..a9dbb6d6c21 100644 --- a/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs @@ -85,26 +85,16 @@ public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest requ } var version = statistics.DateTime.Ticks; - if (request.ToVersion is { } targetVersion && targetVersion != version) - { - return DisseminationRepairResult.Unavailable(version); - } - if (request.FromVersion is { } peerVersion && peerVersion >= version) { return DisseminationRepairResult.Current(version); } - if (request.MaxItemCount <= 0) - { - return DisseminationRepairResult.InsufficientCapacity(version); - } - // Every repair is a full value from zero, making the peer's exact baseline irrelevant. var value = CreateValue(siloAddress, statistics); return value.Payload.Length <= request.MaxPayloadBytes && value.Payload.Length <= request.MaxBatchBytes - ? DisseminationRepairResult.Produced(version, [value]) + ? DisseminationRepairResult.Produced(value) : DisseminationRepairResult.InsufficientCapacity(version); } diff --git a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs index 3f260dc19b6..6f5f66d0c23 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs @@ -12,13 +12,12 @@ namespace Orleans.Runtime.Dissemination; // Payloads are materialized from namespace state only when a peer is ready to send. internal sealed partial class DisseminationBroadcastQueue { - private static readonly TimeSpan DefaultSchedulingDelay = TimeSpan.FromMilliseconds(100); + private static readonly TimeSpan InitialRetryDelay = TimeSpan.FromMilliseconds(100); private static readonly TimeSpan MaxTransportLifetime = TimeSpan.FromMilliseconds(uint.MaxValue - 1); private readonly TimeProvider _timeProvider; private readonly SiloAddress _localSilo; private readonly IInternalGrainFactory _grainFactory; private readonly IOptionsMonitor _options; - private readonly IDisseminationNamespace[] _disseminationNamespaces; private readonly FrozenDictionary _namespaces; private readonly ILogger _logger; private readonly Action? _responseObserver; @@ -41,21 +40,19 @@ public DisseminationBroadcastQueue( _localSilo = localSilo; _grainFactory = grainFactory; _options = options; - _disseminationNamespaces = [.. disseminationNamespaces]; - _namespaces = _disseminationNamespaces.ToFrozenDictionary(static ns => ns.Name); + _namespaces = disseminationNamespaces.ToFrozenDictionary(static ns => ns.Name); _logger = logger; _responseObserver = responseObserver; _sendGate = new(Math.Max(1, options.CurrentValue.MaxConcurrentSends)); _inventoryPool = new DefaultObjectPool>>( - new InventoryPoolPolicy(_disseminationNamespaces), maximumRetained: 1); + new InventoryPoolPolicy(_namespaces), maximumRetained: 1); } public bool Notify( SiloAddress peer, IDisseminationNamespace disseminationNamespace, DisseminationKey key, - bool force = true, - bool immediate = false) + bool force = true) { PeerQueuePump pump; lock (_lock) @@ -70,14 +67,13 @@ public bool Notify( // Notification diagnostics run outside the queue lock and can reenter the queue. var version = disseminationNamespace.GetVersion(key); - return pump.Notify(disseminationNamespace, [new(key, version, force)], immediate); + return pump.Notify(disseminationNamespace, [new(key, version, force)]); } public bool NotifyBatch( SiloAddress peer, IDisseminationNamespace disseminationNamespace, - ReadOnlySpan notifications, - bool immediate) + ReadOnlySpan notifications) { PeerQueuePump pump; lock (_lock) @@ -95,7 +91,7 @@ public bool NotifyBatch( pump = GetOrCreatePeerUnsafe(peer); } - return pump.Notify(disseminationNamespace, notifications, immediate); + return pump.Notify(disseminationNamespace, notifications); } internal readonly record struct KeyNotification(DisseminationKey Key, long Version, bool Force); @@ -176,7 +172,7 @@ public async Task Prune( var activeKeys = _inventoryPool.Get(); try { - foreach (var disseminationNamespace in _disseminationNamespaces) + foreach (var disseminationNamespace in _namespaces.Values) { if (disseminationNamespace.Options.Enabled) { @@ -226,13 +222,13 @@ public async Task Prune( } } - private sealed class InventoryPoolPolicy(IDisseminationNamespace[] namespaces) + private sealed class InventoryPoolPolicy(FrozenDictionary namespaces) : PooledObjectPolicy>> { public override Dictionary> Create() { - var result = new Dictionary>(namespaces.Length); - foreach (var ns in namespaces) + var result = new Dictionary>(namespaces.Count); + foreach (var ns in namespaces.Values) { result.Add(ns.Name, []); } @@ -262,30 +258,10 @@ private PeerQueuePump GetOrCreatePeerUnsafe(SiloAddress peer) return result; } - private TimeSpan GetCoalescingDelay(TimeSpan namespaceDelay) - { - // A peer pump can mix namespaces, so its next wake honors the most urgent coalescing namespace. - // High-priority namespaces never coalesce, so they do not lower other namespaces' windows. - var result = namespaceDelay; - foreach (var disseminationNamespace in _disseminationNamespaces) - { - var namespaceOptions = disseminationNamespace.Options; - if (namespaceOptions.Enabled - && namespaceOptions.Priority != DisseminationPriority.High - && namespaceOptions.MaxCoalescingDelay < result) - { - result = namespaceOptions.MaxCoalescingDelay; - } - } - - return result == TimeSpan.MaxValue ? GetBoundedFallbackDelay() : result; - } - private TimeSpan GetRetryDelay(int attempt) { - // Retries begin at the normal coalescing cadence and back off no further than anti-entropy. - // High-priority namespaces do not coalesce, so they do not define the retry floor. - var floor = GetCoalescingDelay(TimeSpan.MaxValue); + // Failed local attempts back off independently of the root's load collection period. + var floor = InitialRetryDelay; var cap = _options.CurrentValue.Overlay.AntiEntropyInterval; if (floor > cap) { @@ -296,12 +272,6 @@ private TimeSpan GetRetryDelay(int attempt) return TimeSpan.FromTicks((long)Math.Min(cap.Ticks, floor.Ticks * multiplier)); } - private TimeSpan GetBoundedFallbackDelay() - { - var antiEntropyInterval = _options.CurrentValue.Overlay.AntiEntropyInterval; - return antiEntropyInterval < DefaultSchedulingDelay ? antiEntropyInterval : DefaultSchedulingDelay; - } - private sealed class PeerQueuePump { private readonly DisseminationBroadcastQueue _owner; @@ -318,7 +288,6 @@ private sealed class PeerQueuePump // Notifications arriving during a send advance the epoch and remain dirty for the next pass. private long _notificationEpoch; private int _retryAttempt; - private bool _wakeScheduled; private bool _stopping; private bool _draining; @@ -338,16 +307,13 @@ public PeerQueuePump(SiloAddress peer, DisseminationBroadcastQueue owner) public bool Notify( IDisseminationNamespace disseminationNamespace, - ReadOnlySpan notifications, - bool immediate) + ReadOnlySpan notifications) { ScheduledFlush? scheduled = null; var rejections = 0; var accepted = true; lock (_lock) { - var wasEmpty = DirtyCount == 0; - var wasRetrying = _retryAttempt > 0; var anyChanged = false; foreach (var notification in notifications) { @@ -360,8 +326,9 @@ public bool Notify( if (anyChanged) { - // Publish the complete key set before waking a relay's send loop. - scheduled = ScheduleNotificationUnsafe(disseminationNamespace, immediate, wasEmpty, wasRetrying); + // Only the aggregation root collects load; peer pumps send each accepted batch immediately. + _flushTimer.Wake(); + scheduled = new(DisseminationBroadcastScheduleReason.Immediate, TimeSpan.Zero, _retryAttempt, _notificationEpoch); } } @@ -390,7 +357,7 @@ private bool NotifyKeyUnsafe( } var namespaceState = GetOrCreateNamespaceStateUnsafe(disseminationNamespace); - namespaceState.TryGetKey(key, out var keyState); + namespaceState.Keys.TryGetValue(key, out var keyState); if (!force) { // Duplicate deliveries seed unknown peers without perpetuating cycles between skewed views. @@ -421,48 +388,11 @@ private bool NotifyKeyUnsafe( keyState.NotificationVersion = version; keyState.NotificationGeneration = ++_notificationEpoch; _retryAttempt = 0; - MarkDirtyUnsafe(namespaceState, keyState); + MarkDirtyUnsafe(keyState); changed = true; return true; } - private ScheduledFlush? ScheduleNotificationUnsafe( - IDisseminationNamespace disseminationNamespace, bool immediate, bool wasEmpty, bool wasRetrying) - { - var currentOptions = _owner._options.CurrentValue; - if (immediate) - { - _flushTimer.Wake(); - _wakeScheduled = true; - return new(DisseminationBroadcastScheduleReason.Immediate, TimeSpan.Zero, _retryAttempt, _notificationEpoch); - } - - if (disseminationNamespace.Options.Priority == DisseminationPriority.High) - { - _flushTimer.Change(TimeSpan.Zero); - _wakeScheduled = true; - return new(DisseminationBroadcastScheduleReason.Priority, TimeSpan.Zero, _retryAttempt, _notificationEpoch); - } - - if (DirtyCount >= currentOptions.MaxBatchItems - || _statesByNamespace[disseminationNamespace.Name].DirtyCount >= disseminationNamespace.Options.MaxPendingItemCount) - { - _flushTimer.Change(TimeSpan.Zero); - _wakeScheduled = true; - return new(DisseminationBroadcastScheduleReason.Immediate, TimeSpan.Zero, _retryAttempt, _notificationEpoch); - } - - if (wasEmpty || wasRetrying || !_wakeScheduled) - { - var delay = _owner.GetCoalescingDelay(disseminationNamespace.Options.MaxCoalescingDelay); - _flushTimer.Change(delay); - _wakeScheduled = true; - return new(DisseminationBroadcastScheduleReason.Coalesce, delay, _retryAttempt, _notificationEpoch); - } - - return null; - } - private void EmitNotification(IDisseminationNamespace disseminationNamespace, int rejections, ScheduledFlush? scheduled) { for (var index = 0; index < rejections; index++) @@ -550,7 +480,7 @@ public void ObservePeerVersion( // Peer knowledge is evidence-driven and monotonic across acknowledgments, pushes, and anti-entropy. var namespaceState = GetOrCreateNamespaceStateUnsafe(disseminationNamespace); - if (namespaceState.TryGetKey(key, out var keyState) && keyState is not null) + if (namespaceState.Keys.TryGetValue(key, out var keyState)) { if (keyState.KnownVersion is not { } knownVersion || version > knownVersion) { @@ -576,8 +506,9 @@ public void PruneKeys( { if (!membershipSnapshots.GetSnapshot(namespaceState.Namespace.MembershipScope).ContainsMember(Peer)) { - droppedDirtyCount += namespaceState.DirtyCount; - DirtyCount -= namespaceState.DirtyCount; + var removedDirtyCount = namespaceState.Keys.Values.Count(static key => key.Dirty); + droppedDirtyCount += removedDirtyCount; + DirtyCount -= removedDirtyCount; _statesByNamespace.Remove(namespaceName); continue; } @@ -626,7 +557,6 @@ public async ValueTask FlushAsync(CancellationToken cancellationToken) { flushCompletion = _nextFlushCompletion.Task; wake = true; - _wakeScheduled = true; } else { @@ -675,7 +605,6 @@ public async ValueTask StopAsync(bool drain, CancellationToken cancellationToken { flushCompletion = _nextFlushCompletion.Task; wake = true; - _wakeScheduled = true; } else { @@ -778,7 +707,6 @@ private async Task RunScheduledFlushIteration(CancellationToken cancellationToke // Move one dirty generation to in-flight atomically; a concurrent notification can mark it dirty again. lock (_lock) { - _wakeScheduled = false; if (DirtyCount == 0) { return; @@ -791,7 +719,7 @@ private async Task RunScheduledFlushIteration(CancellationToken cancellationToke work = DrainDirtyUnsafe(); } - var result = SendWorkResult.None; + var result = default(SendWorkResult); try { result = await SendValues(work, cancellationToken); @@ -818,48 +746,20 @@ private async Task RunScheduledFlushIteration(CancellationToken cancellationToke _retryAttempt = 0; } - if (DirtyCount > 0 && (!_stopping || _draining)) + // A newer notification already signaled the next pass. + if (DirtyCount > 0 && (!_stopping || _draining) && notificationEpoch == _notificationEpoch) { - // Do not overwrite a timer which a newer notification already chose. - if (notificationEpoch == _notificationEpoch) + if (result.RequiresBackoff) { - if (result.RequiresBackoff) - { - _retryAttempt++; - var delay = _owner.GetRetryDelay(_retryAttempt); - _flushTimer.Change(delay); - scheduled = new(DisseminationBroadcastScheduleReason.Retry, delay, _retryAttempt, _notificationEpoch); - } - else if (HasHighPriorityDirtyUnsafe()) - { - // Leftover high-priority work must not wait for another coalescing window. - _flushTimer.Change(TimeSpan.Zero); - scheduled = new(DisseminationBroadcastScheduleReason.Priority, TimeSpan.Zero, _retryAttempt, _notificationEpoch); - } - else - { - var delay = _owner.GetCoalescingDelay(TimeSpan.MaxValue); - _flushTimer.Change(delay); - scheduled = new(DisseminationBroadcastScheduleReason.Coalesce, delay, _retryAttempt, _notificationEpoch); - } - - _wakeScheduled = true; + _retryAttempt++; + var delay = _owner.GetRetryDelay(_retryAttempt); + _flushTimer.Change(delay); + scheduled = new(DisseminationBroadcastScheduleReason.Retry, delay, _retryAttempt, _notificationEpoch); } - else if (!_wakeScheduled) + else { - if (HasHighPriorityDirtyUnsafe()) - { - _flushTimer.Change(TimeSpan.Zero); - scheduled = new(DisseminationBroadcastScheduleReason.Priority, TimeSpan.Zero, _retryAttempt, _notificationEpoch); - } - else - { - var delay = _owner.GetCoalescingDelay(TimeSpan.MaxValue); - _flushTimer.Change(delay); - scheduled = new(DisseminationBroadcastScheduleReason.Coalesce, delay, _retryAttempt, _notificationEpoch); - } - - _wakeScheduled = true; + _flushTimer.Wake(); + scheduled = new(DisseminationBroadcastScheduleReason.Immediate, TimeSpan.Zero, _retryAttempt, _notificationEpoch); } } @@ -887,7 +787,6 @@ private bool TryRecoverFromUnexpectedIterationFailure(Exception exception) _retryAttempt++; var delay = _owner.GetRetryDelay(_retryAttempt); _flushTimer.Change(delay); - _wakeScheduled = true; scheduled = new(DisseminationBroadcastScheduleReason.Retry, delay, _retryAttempt, _notificationEpoch); } @@ -1013,8 +912,6 @@ private async ValueTask SendValues( var request = new DisseminationRepairRequest( work.Key, knownVersion, - toVersion: null, - currentOptions.MaxBatchItems - itemCount, currentOptions.MaxBatchBytes - byteCount, work.Namespace.Options.MaxPayloadBytes); var repair = work.Namespace.CreateRepair(request); @@ -1065,20 +962,16 @@ private async ValueTask SendValues( work.Namespace.Name, out _); namespaceValues ??= []; - long sentVersion = 0; - foreach (var value in repair.Values) + var value = repair.Value; + namespaceValues.Add(new DisseminationBroadcastValue { - namespaceValues.Add(new DisseminationBroadcastValue - { - Value = value, - TimeToLive = work.Namespace.Options.StaleItemTtl, - }); - itemCount++; - byteCount += value.Payload.Length; - sentVersion = Math.Max(sentVersion, value.ToVersion); - } + Value = value, + TimeToLive = work.Namespace.Options.StaleItemTtl, + }); + itemCount++; + byteCount += value.Payload.Length; - sentKeys.Add(new(work, knownVersion, repair.Version, sentVersion)); + sentKeys.Add(new(work, knownVersion, value.ToVersion)); if (itemCount >= currentOptions.MaxBatchItems || byteCount >= currentOptions.MaxBatchBytes) { break; @@ -1131,11 +1024,6 @@ private async ValueTask SendValues( var completion = CompleteAcknowledged(sent, acknowledgedVersion); madeProgress |= completion.MadeProgress; requiresBackoff |= completion.RequiresBackoff; - if (completion.ImmediateRetry is { } retry) - { - // An acknowledged prefix rejoins admission before continuing from the receiver's new version. - pending.Enqueue(retry); - } } } finally @@ -1255,10 +1143,10 @@ private IDisseminationSystemTarget GetTarget() => private List DrainDirtyUnsafe() { // Snapshot dirty work into an in-flight generation; any later notification will set Dirty again. - // Drain higher-priority namespaces first so their values fill the earliest batches. + // Membership precedes load when a peer has both kinds of work. var result = new List(DirtyCount); foreach (var namespaceState in _statesByNamespace.Values - .OrderByDescending(static state => (int)state.Namespace.Options.Priority)) + .OrderBy(static state => state.Namespace.RoutingMode == DisseminationRoutingMode.AggregationTree)) { foreach (var (key, keyState) in namespaceState.Keys) { @@ -1269,7 +1157,6 @@ private List DrainDirtyUnsafe() keyState.Dirty = false; keyState.InFlight = true; - namespaceState.DirtyCount--; DirtyCount--; result.Add(new( namespaceState.Namespace, @@ -1323,7 +1210,7 @@ private void CompleteCurrent(PendingKeyWork work, long version) } } - private AcknowledgmentCompletion CompleteAcknowledged(SentKey sent, long acknowledgedVersion) + private SendWorkResult CompleteAcknowledged(SentKey sent, long acknowledgedVersion) { lock (_lock) { @@ -1341,33 +1228,22 @@ private AcknowledgmentCompletion CompleteAcknowledged(SentKey sent, long acknowl var madeProgress = sent.FromVersion is null ? acknowledgedVersion > 0 : acknowledgedVersion > sent.FromVersion; + keyState.InFlight = false; // Retire only the generation we sent; a newer notification remains queued even if this repair reached its target. if (keyState.NotificationGeneration != sent.Work.NotificationGeneration || keyState.Dirty) { - keyState.InFlight = false; - return new(null, RequiresBackoff: false, madeProgress); + return new(RequiresBackoff: false, madeProgress); } - if (keyState.KnownVersion >= sent.ResolvedVersion) + if (keyState.KnownVersion >= sent.SentVersion) { - keyState.InFlight = false; namespaceState.RetireKey(sent.Work.Key, keyState); - return new(null, RequiresBackoff: false, madeProgress); + return new(RequiresBackoff: false, madeProgress); } - if (madeProgress) - { - // Partial repairs stay in flight and are re-materialized immediately from the acknowledged prefix. - return new( - sent.Work with { NotificationGeneration = keyState.NotificationGeneration, KnownVersion = keyState.KnownVersion }, - RequiresBackoff: false, - MadeProgress: true); - } - - keyState.InFlight = false; - MarkDirtyUnsafe(namespaceState, keyState); - return new(null, RequiresBackoff: true, MadeProgress: false); + MarkDirtyUnsafe(keyState); + return new(RequiresBackoff: true, madeProgress); } } @@ -1381,7 +1257,7 @@ private bool CompleteFromExistingEvidence(SentKey sent) return true; } - if (keyState.KnownVersion >= sent.ResolvedVersion) + if (keyState.KnownVersion >= sent.SentVersion) { keyState.InFlight = false; if (keyState.NotificationGeneration == sent.Work.NotificationGeneration && !keyState.Dirty) @@ -1393,7 +1269,7 @@ private bool CompleteFromExistingEvidence(SentKey sent) } keyState.InFlight = false; - MarkDirtyUnsafe(namespaceState, keyState); + MarkDirtyUnsafe(keyState); return false; } } @@ -1418,8 +1294,7 @@ private void CompleteUnsupported(DisseminationNamespace namespaceName, List work) foreach (var item in work) { - if (TryGetKeyStateUnsafe(item, out var namespaceState, out var keyState)) + if (TryGetKeyStateUnsafe(item, out _, out var keyState)) { keyState.InFlight = false; - MarkDirtyUnsafe(namespaceState, keyState); + MarkDirtyUnsafe(keyState); } } } @@ -1538,7 +1412,6 @@ private void ClearPendingUnsafe() _statesByNamespace.Clear(); DirtyCount = 0; _retryAttempt = 0; - _wakeScheduled = false; } private PeerNamespaceState GetOrCreateNamespaceStateUnsafe(IDisseminationNamespace disseminationNamespace) @@ -1569,7 +1442,7 @@ private bool TryGetKeyStateUnsafe( return false; } - private void MarkDirtyUnsafe(PeerNamespaceState namespaceState, PeerKeyState keyState) + private void MarkDirtyUnsafe(PeerKeyState keyState) { if (keyState.Dirty) { @@ -1577,24 +1450,9 @@ private void MarkDirtyUnsafe(PeerNamespaceState namespaceState, PeerKeyState key } keyState.Dirty = true; - namespaceState.DirtyCount++; DirtyCount++; } - private bool HasHighPriorityDirtyUnsafe() - { - foreach (var namespaceState in _statesByNamespace.Values) - { - if (namespaceState.DirtyCount > 0 - && namespaceState.Namespace.Options.Priority == DisseminationPriority.High) - { - return true; - } - } - - return false; - } - private static Dictionary CreateAcknowledgmentLookup( Dictionary> acknowledgments) { @@ -1623,47 +1481,14 @@ private static bool ValidateRepair( in DisseminationRepairRequest request, in DisseminationRepairResult repair) { - // Defensively enforce a bounded, contiguous chain toward the namespace's resolved target. - if (repair.Status is not DisseminationRepairStatus.Produced - || repair.Values.IsDefaultOrEmpty - || repair.Version <= 0 - || request.ToVersion is { } requestedToVersion && repair.Version > requestedToVersion - || repair.Values.Length > request.MaxItemCount) - { - return false; - } - - var byteCount = 0; - var expectedFromVersion = request.FromVersion; - foreach (var value in repair.Values) - { - if (value.Key != request.Key - || value.FromVersion < 0 - || value.ToVersion <= value.FromVersion - || value.ToVersion > repair.Version - || value.Payload.Length > request.MaxPayloadBytes) - { - return false; - } - - if (expectedFromVersion is null && value.FromVersion != 0 - || expectedFromVersion is { } fromVersion - && value.FromVersion != 0 - && value.FromVersion != fromVersion) - { - return false; - } - - expectedFromVersion = value.ToVersion; - byteCount += value.Payload.Length; - if (byteCount > request.MaxBatchBytes) - { - return false; - } - } - - var lastVersion = repair.Values[^1].ToVersion; - return repair.IsComplete ? lastVersion == repair.Version : lastVersion < repair.Version; + var value = repair.Value; + return repair.Status is DisseminationRepairStatus.Produced + && repair.Version > 0 + && value.Key == request.Key + && value.FromVersion == 0 + && value.ToVersion == repair.Version + && value.Payload.Length <= request.MaxPayloadBytes + && value.Payload.Length <= request.MaxBatchBytes; } private sealed class PeerNamespaceState(IDisseminationNamespace disseminationNamespace) @@ -1674,11 +1499,6 @@ private sealed class PeerNamespaceState(IDisseminationNamespace disseminationNam public Dictionary KnownVersions { get; } = []; - public int DirtyCount { get; set; } - - public bool TryGetKey(DisseminationKey key, out PeerKeyState? keyState) => - Keys.TryGetValue(key, out keyState); - public PeerKeyState AddKey(DisseminationKey key) { var result = new PeerKeyState(); @@ -1745,22 +1565,13 @@ private readonly record struct PendingKeyWork( private readonly record struct SentKey( PendingKeyWork Work, long? FromVersion, - long ResolvedVersion, long SentVersion); private readonly record struct DigestKey( DisseminationNamespace Namespace, DisseminationKey Key); - private readonly record struct AcknowledgmentCompletion( - PendingKeyWork? ImmediateRetry, - bool RequiresBackoff, - bool MadeProgress); - - private readonly record struct SendWorkResult(bool RequiresBackoff, bool MadeProgress) - { - public static SendWorkResult None => default; - } + private readonly record struct SendWorkResult(bool RequiresBackoff, bool MadeProgress); } [LoggerMessage( diff --git a/src/Orleans.Runtime/Dissemination/DisseminationEvents.cs b/src/Orleans.Runtime/Dissemination/DisseminationEvents.cs index c17150aff95..7907a6674a5 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationEvents.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationEvents.cs @@ -112,17 +112,11 @@ public static void EmitQueueAdmissionRejected( // Why a peer pump (re)armed its flush timer, exposed for deterministic tests and diagnostics. internal enum DisseminationBroadcastScheduleReason { - // A batch filled, so the pump flushes without further coalescing. + // New work is ready for admission. Immediate, - // The pump is coalescing and will flush after the namespace delay. - Coalesce, - // A prior send failed and the pump re-armed after backoff. Retry, - - // A high-priority namespace bypasses the coalescing window and flushes immediately. - Priority, } internal sealed class DisseminationBroadcastScheduledEvent diff --git a/src/Orleans.Runtime/Dissemination/DisseminationInstruments.cs b/src/Orleans.Runtime/Dissemination/DisseminationInstruments.cs index 824d0faf41b..5d3beb640da 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationInstruments.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationInstruments.cs @@ -36,30 +36,16 @@ internal static class DisseminationInstruments public static void OnBroadcastSent(DisseminationNamespace namespaceName, string kind, int itemCount, int byteCount) { - var broadcastSentEnabled = BroadcastSent.Enabled; - var valuesSentEnabled = ValuesSent.Enabled; - var bytesSentEnabled = BytesSent.Enabled; - if (!broadcastSentEnabled && !valuesSentEnabled && !bytesSentEnabled) + if (!BroadcastSent.Enabled && !ValuesSent.Enabled && !BytesSent.Enabled) { return; } var namespaceTag = Tag("namespace", namespaceName); var kindTag = Tag("kind", kind); - if (broadcastSentEnabled) - { - BroadcastSent.Add(1, namespaceTag, kindTag); - } - - if (valuesSentEnabled) - { - ValuesSent.Add(itemCount, namespaceTag, kindTag); - } - - if (bytesSentEnabled) - { - BytesSent.Add(byteCount, namespaceTag, kindTag); - } + BroadcastSent.Add(1, namespaceTag, kindTag); + ValuesSent.Add(itemCount, namespaceTag, kindTag); + BytesSent.Add(byteCount, namespaceTag, kindTag); } public static void OnBroadcastSent(Dictionary> valuesByNamespace, string kind) @@ -77,23 +63,15 @@ public static void OnBroadcastSent(Dictionary private static string GetScheduleReason(DisseminationBroadcastScheduleReason reason) => reason switch { DisseminationBroadcastScheduleReason.Immediate => "immediate", - DisseminationBroadcastScheduleReason.Coalesce => "coalesce", DisseminationBroadcastScheduleReason.Retry => "retry", - DisseminationBroadcastScheduleReason.Priority => "priority", _ => "unknown", }; diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs index bc1adc1d877..990bcbd4bd5 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs @@ -47,19 +47,17 @@ public async ValueTask PublishAggregated( } var repairRequest = new DisseminationRepairRequest( - key, fromVersion: null, toVersion: null, maxItemCount: 1, + key, fromVersion: null, options.MaxBatchBytes, disseminationNamespace.Options.MaxPayloadBytes); var repair = disseminationNamespace.CreateRepair(repairRequest); if (repair.Status != DisseminationRepairStatus.Produced - || !repair.IsComplete || repair.Values.Length != 1 || repair.Version < version - || repair.Values[0].FromVersion != 0 || !ValidateRepair(disseminationNamespace, repairRequest, repair, options)) { return Reject("invalid-repair"); } - var value = repair.Values[0]; + var value = repair.Value; RecordValueUpdate(disseminationNamespace.Name, key, value.ToVersion); DisseminationPublicationReceipt receipt; if (membership.IsAggregationRoot) diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index 711b884aa32..f9b81eed418 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -143,9 +143,7 @@ public async ValueTask Publish( { foreach (var peer in membership.GetOriginatorTargets(disseminationNamespace.RoutingMode)) { - accepted &= _broadcastQueue.Notify( - peer, disseminationNamespace, key, - immediate: disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree); + accepted &= _broadcastQueue.Notify(peer, disseminationNamespace, key); } } @@ -211,23 +209,13 @@ public async Task ReceiveBroadcast( namespaceName, item.Value.Key, item.Value.ToVersion); - DisseminationApplyResult result; - try - { - result = await ApplyReceivedValue( - disseminationNamespace, - item, - batch.Sender, - options, - receivedTimestamp, - cancellationToken); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - cancellationToken.ThrowIfCancellationRequested(); - throw; - } - + var result = await ApplyReceivedValue( + disseminationNamespace, + item, + batch.Sender, + options, + receivedTimestamp, + cancellationToken); namespaceKeys[item.Value.Key] = new(sentVersion, keyState.Applied || result is DisseminationApplyResult.Applied); } } @@ -240,48 +228,28 @@ public async Task ReceiveBroadcast( foreach (var (disseminationNamespace, keys) in receivedKeys) { var membership = membershipSnapshots.GetSnapshot(disseminationNamespace.MembershipScope); - if (disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree) + var notifications = new DisseminationBroadcastQueue.KeyNotification[keys.Count]; + var notificationCount = 0; + foreach (var (key, state) in keys) { - var notifications = new DisseminationBroadcastQueue.KeyNotification[keys.Count]; - var notificationCount = 0; - foreach (var (key, state) in keys) + var version = disseminationNamespace.GetVersion(key); + if (version > 0) { - var version = disseminationNamespace.GetVersion(key); - if (version > 0) - { - notifications[notificationCount++] = new(key, version, state.Applied); - } - } - - if (membership.IsAggregationRoot) - { - NotifyRoot(disseminationNamespace, notifications.AsSpan(0, notificationCount)); - } - else - { - foreach (var peer in membership.GetForwardingTargets(disseminationNamespace.RoutingMode, batch.Sender)) - { - _broadcastQueue.NotifyBatch( - peer, disseminationNamespace, notifications.AsSpan(0, notificationCount), immediate: true); - } + notifications[notificationCount++] = new(key, version, state.Applied); } + } + if (disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree && membership.IsAggregationRoot) + { + NotifyRoot(disseminationNamespace, notifications.AsSpan(0, notificationCount)); continue; } - foreach (var (key, state) in keys) + foreach (var peer in membership.GetForwardingTargets(disseminationNamespace.RoutingMode, batch.Sender)) { - if (disseminationNamespace.GetVersion(key) <= 0) + if (!Equals(peer, batch.Sender)) { - continue; - } - - foreach (var peer in membership.GetForwardingTargets(disseminationNamespace.RoutingMode)) - { - if (!Equals(peer, batch.Sender)) - { - _broadcastQueue.Notify(peer, disseminationNamespace, key, force: state.Applied); - } + _broadcastQueue.NotifyBatch(peer, disseminationNamespace, notifications.AsSpan(0, notificationCount)); } } } @@ -451,42 +419,22 @@ private Dictionary CreateAntiEntro DisseminationOptions options) { var result = new Dictionary(); - var namespacesByScope = new Dictionary(); - foreach (var scope in Enum.GetValues()) - { - var scopedNamespaces = _namespaces.Values - .Where(disseminationNamespace => disseminationNamespace.Options.Enabled - && disseminationNamespace.MembershipScope == scope) - .ToArray(); - if (scopedNamespaces.Length > 0) - { - namespacesByScope.Add(scope, scopedNamespaces); - } - } - - if (namespacesByScope.Count == 0) - { - return result; - } - + var enabledNamespaces = _namespaces.Values.Where(static ns => ns.Options.Enabled).ToArray(); foreach (var peer in peers) { var peerDigests = new Dictionary>(); var supportedNamespaces = new List(); - foreach (var (scope, scopedNamespaces) in namespacesByScope) + foreach (var disseminationNamespace in enabledNamespaces) { - if (!membershipSnapshots.GetSnapshot(scope).ContainsMember(peer)) + if (!membershipSnapshots.GetSnapshot(disseminationNamespace.MembershipScope).ContainsMember(peer)) { continue; } - foreach (var disseminationNamespace in scopedNamespaces) + supportedNamespaces.Add(disseminationNamespace.Name); + if (requestDigests.TryGetValue(disseminationNamespace.Name, out var digest)) { - supportedNamespaces.Add(disseminationNamespace.Name); - if (requestDigests.TryGetValue(disseminationNamespace.Name, out var digest)) - { - peerDigests.Add(disseminationNamespace.Name, digest); - } + peerDigests.Add(disseminationNamespace.Name, digest); } } @@ -647,8 +595,6 @@ private async Task ApplyAntiEntropyResponses( // Completed exchanges get a separate local application window, even if another peer used its whole // transport budget. No remote clock or transmission delay is inferred from a relative wire lifetime. var receivedTimestamp = _timeProvider.GetTimestamp(); - // Keep each sender's chain intact and rank all repairs which completed within the round's hop lifetime. - Dictionary>? repairs = null; foreach (var response in responses) { if (response is null) @@ -666,46 +612,19 @@ private async Task ApplyAntiEntropyResponses( cancellationToken.ThrowIfCancellationRequested(); var disseminationNamespace = _namespaces[namespaceName]; - foreach (var stream in values.GroupBy(static item => item.Value.Key)) + foreach (var item in values) { - repairs ??= []; - var items = stream.ToList(); - var key = new DigestKey(namespaceName, stream.Key); - if (!repairs.TryGetValue(key, out var candidates)) - { - candidates = []; - repairs.Add(key, candidates); - } - - var terminalVersion = items.Max(static item => item.Value.ToVersion); _broadcastQueue.ObservePeerVersion( response.Sender, namespaceName, - stream.Key, - terminalVersion); - candidates.Add(new(disseminationNamespace, items, response.Sender)); - } - } - } - - if (repairs is null) - { - return; - } - - foreach (var candidates in repairs.Values) - { - // Try the furthest-reaching repair first, preferring a full value when candidates tie. - candidates.Sort(CompareAntiEntropyRepairs); + item.Value.Key, + item.Value.ToVersion); - foreach (var candidate in candidates) - { - foreach (var item in candidate.Items) - { + // Equal-version membership snapshots can contain complementary heartbeat advances. await ApplyReceivedValue( - candidate.Namespace, + disseminationNamespace, item, - candidate.Sender, + response.Sender, options, receivedTimestamp, cancellationToken); @@ -791,7 +710,7 @@ private Dictionary> Se if (payloadBytes > maxBytes - byteCount) { - // Preserve this candidate for a fresh budget instead of discarding a repair-chain suffix. + // Preserve this candidate for a fresh budget instead of discarding it. goto Complete; } @@ -814,8 +733,8 @@ private Dictionary> Se var members = _membership.CurrentSnapshots.AllMembers; lock (_receivedBatchCursorLock) { - // Do not wrap within a batch: doing so would reorder a key's delta chain. Subsequent deliveries - // resume an ordered, bounded interval, allowing later keys past a repeatedly rejected/hot prefix. + // Inspect each item at most once per delivery. Repeated oversized batches resume later, + // allowing cold keys past a repeatedly rejected or hot prefix. if (position < totalCount) { _receivedBatchCursors[cursorKey] = new(position, totalCount, ++_receivedBatchCursorAccess); @@ -1002,6 +921,12 @@ private DisseminationAntiEntropyResponse CreateAntiEntropyResponse( for (var i = 0; i < candidates.Count; i++) { cancellationToken.ThrowIfCancellationRequested(); + if (valueCount >= maxResponseItems) + { + truncated = true; + break; + } + var candidate = candidates[(start + i) % candidates.Count]; examined++; var requestedNamespace = candidate.Namespace; @@ -1010,46 +935,22 @@ private DisseminationAntiEntropyResponse CreateAntiEntropyResponse( var repairRequest = new DisseminationRepairRequest( localDigest.Key, peerDigest.Version, - toVersion: null, - maxResponseItems - valueCount, - maxResponseBytes - byteCount, + maxResponseBytes, requestedNamespace.Options.MaxPayloadBytes); var repair = requestedNamespace.CreateRepair(repairRequest); - if (repair.Status is DisseminationRepairStatus.InsufficientCapacity) + if (repair.Status is not DisseminationRepairStatus.Produced + || !ValidateRepair(requestedNamespace, repairRequest, repair, options)) { - if (valueCount > 0) - { - // Probe with a fresh batch budget to distinguish truncation from a permanently oversized key. - var emptyBatchRequest = new DisseminationRepairRequest( - localDigest.Key, - peerDigest.Version, - toVersion: null, - maxResponseItems, - maxResponseBytes, - requestedNamespace.Options.MaxPayloadBytes); - var emptyBatchRepair = requestedNamespace.CreateRepair(emptyBatchRequest); - if (emptyBatchRepair.Status is DisseminationRepairStatus.Produced - && ValidateRepair( - requestedNamespace, - emptyBatchRequest, - emptyBatchRepair, - options)) - { - // Resume with this candidate next round because the current response budget, not the - // candidate itself, prevented it from being included. - examined--; - truncated = true; - break; - } - } - continue; } - if (repair.Status is not DisseminationRepairStatus.Produced - || !ValidateRepair(requestedNamespace, repairRequest, repair, options)) + var value = repair.Value; + if (value.Payload.Length > maxResponseBytes - byteCount) { - continue; + // This value fits a fresh response, so resume here instead of discarding it. + examined--; + truncated = true; + break; } if (!valuesByNamespace.TryGetValue(requestedNamespace.Name, out var namespaceValues)) @@ -1058,19 +959,9 @@ private DisseminationAntiEntropyResponse CreateAntiEntropyResponse( valuesByNamespace.Add(requestedNamespace.Name, namespaceValues); } - foreach (var value in repair.Values) - { - namespaceValues.Add(CreateBroadcastValue(requestedNamespace, value)); - ++valueCount; - byteCount += value.Payload.Length; - } - - if (!repair.IsComplete) - { - // A valid prefix consumes this response; the caller can continue in its next round. - truncated = true; - break; - } + namespaceValues.Add(CreateBroadcastValue(requestedNamespace, value)); + ++valueCount; + byteCount += value.Payload.Length; } if (truncated) @@ -1198,6 +1089,7 @@ private async ValueTask ApplyReceivedValue( } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { + cancellationToken.ThrowIfCancellationRequested(); throw; } catch (Exception exception) @@ -1239,7 +1131,7 @@ private async ValueTask ApplyReceivedValueCore( return DisseminationApplyResult.Obsolete; } - // Reject gaps before deserializing; full values from version zero can replace any baseline. + // Only full values are supported; obsolete values need no deserialization unless the owner validates authority. if (TryGetTerminalApplyResult(disseminationNamespace, item.Value, out var terminalResult)) { EmitApplyResult(namespaceName, item, sender, terminalResult); @@ -1508,14 +1400,14 @@ private bool DispatchRootBatch( { // A departing root hands accepted state to the new root, even after leaving Active membership. return membership.Members.IsEmpty - || _broadcastQueue.NotifyBatch(membership.Members[0], disseminationNamespace, notifications, immediate: true); + || _broadcastQueue.NotifyBatch(membership.Members[0], disseminationNamespace, notifications); } var accepted = true; foreach (var child in membership.AggregationChildren) { // Include a child producer's own value so it reaches that child's descendants. - accepted &= _broadcastQueue.NotifyBatch(child, disseminationNamespace, notifications, immediate: true); + accepted &= _broadcastQueue.NotifyBatch(child, disseminationNamespace, notifications); } return accepted; @@ -1648,14 +1540,11 @@ private bool TryValidatePublish( return false; } - // Validate the complete repair chain for an unknown peer. Individual values must fit a batch, - // while the chain can be delivered across multiple acknowledged batches. + // Validate a complete current value for an unknown peer before accepting the publication. var request = new DisseminationRepairRequest( key, fromVersion: null, - toVersion: null, - maxItemCount: int.MaxValue, - maxBatchBytes: int.MaxValue, + options.MaxBatchBytes, disseminationNamespace.Options.MaxPayloadBytes); var repair = disseminationNamespace.CreateRepair(request); if (repair.Status is DisseminationRepairStatus.InsufficientCapacity) @@ -1666,7 +1555,6 @@ private bool TryValidatePublish( } if (repair.Status is not DisseminationRepairStatus.Produced - || !repair.IsComplete || repair.Version < version || !ValidateRepair(disseminationNamespace, request, repair, options)) { @@ -1686,77 +1574,37 @@ private bool ValidateRepair( in DisseminationRepairResult repair, DisseminationOptions options) { - // Keep namespace-specific serialization behind one common range and budget contract. - if (repair.Status is not DisseminationRepairStatus.Produced - || repair.Values.IsDefaultOrEmpty - || repair.Version <= 0 - || request.ToVersion is { } requestedToVersion && repair.Version > requestedToVersion - || repair.Values.Length > request.MaxItemCount) - { - return false; - } - - var byteCount = 0; - var expectedFromVersion = request.FromVersion; - foreach (var value in repair.Values) - { - if (value.Key != request.Key - || !IsValidVersionRange(value) - || value.ToVersion > repair.Version - || !ValidatePayloadSize(disseminationNamespace, value, options.MaxBatchBytes)) - { - return false; - } - - if (expectedFromVersion is null && value.FromVersion != 0 - || expectedFromVersion is { } fromVersion - && value.FromVersion != 0 - && value.FromVersion != fromVersion) - { - return false; - } - - expectedFromVersion = value.ToVersion; - byteCount += value.Payload.Length; - if (byteCount > request.MaxBatchBytes) - { - return false; - } - } - - var lastVersion = repair.Values[^1].ToVersion; - return repair.IsComplete ? lastVersion == repair.Version : lastVersion < repair.Version; + var value = repair.Value; + return repair.Status is DisseminationRepairStatus.Produced + && repair.Version > 0 + && value.Key == request.Key + && value.FromVersion == 0 + && value.ToVersion == repair.Version + && ValidatePayloadSize(disseminationNamespace, value, options.MaxBatchBytes) + && value.Payload.Length <= request.MaxPayloadBytes + && value.Payload.Length <= request.MaxBatchBytes; } - private static bool IsValidVersionRange(DisseminationValue value) => - value is { FromVersion: >= 0, ToVersion: > 0 } && value.ToVersion > value.FromVersion; - private static bool TryGetTerminalApplyResult( IDisseminationNamespace disseminationNamespace, DisseminationValue value, out DisseminationApplyResult result) { - if (!IsValidVersionRange(value)) + if (value.FromVersion != 0 || value.ToVersion <= 0) { result = DisseminationApplyResult.Rejected; return true; } var localVersion = disseminationNamespace.GetVersion(value.Key); - if (value.ToVersion < localVersion && !(value.FromVersion == 0 && disseminationNamespace.ValidateOlderFullValues)) + if (value.ToVersion < localVersion && !disseminationNamespace.ValidateOlderFullValues) { result = DisseminationApplyResult.Obsolete; return true; } - if (value.ToVersion == localVersion || value.FromVersion == 0 || value.FromVersion == localVersion) - { - result = default; - return false; - } - - result = DisseminationApplyResult.Rejected; - return true; + result = default; + return false; } private DisseminationBroadcastValue CreateBroadcastValue( @@ -1840,25 +1688,6 @@ private static Dictionary CreateDigestLookup(List private static int GetValueCount(Dictionary> valuesByNamespace) => valuesByNamespace.Values.Sum(values => values.Count); - // Prefer the highest terminal version, then a universal full value, then stable peer order. - private static int CompareAntiEntropyRepairs(AntiEntropyRepair left, AntiEntropyRepair right) - { - var result = right.Items[^1].Value.ToVersion.CompareTo(left.Items[^1].Value.ToVersion); - if (result != 0) - { - return result; - } - - var leftIsFullValue = left.Items[0].Value.FromVersion == 0; - var rightIsFullValue = right.Items[0].Value.FromVersion == 0; - if (leftIsFullValue != rightIsFullValue) - { - return leftIsFullValue ? -1 : 1; - } - - return left.Sender.CompareTo(right.Sender); - } - private readonly record struct DigestKey(DisseminationNamespace Namespace, DisseminationKey Key); private readonly record struct ValueUpdate(long Version, long Timestamp); @@ -1867,11 +1696,6 @@ private static int CompareAntiEntropyRepairs(AntiEntropyRepair left, AntiEntropy private readonly record struct ReceivedBatchCursor(long Position, long TotalCount, long LastAccess); - private readonly record struct AntiEntropyRepair( - IDisseminationNamespace Namespace, - List Items, - SiloAddress Sender); - private readonly record struct AntiEntropyResponseCandidate( IDisseminationNamespace Namespace, DigestEntry LocalDigest, diff --git a/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs b/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs index bb59a9ddc43..5ab61a7faae 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs @@ -29,7 +29,6 @@ internal sealed partial class DisseminationRootBatcher private Cohort? _cohort; private TaskCompletionSource? _drainCompletion; private Exception? _failure; - private long _generation; private long? _retryTimestamp; private TimeSpan _retryPeriod; private bool _dispatching; @@ -93,7 +92,7 @@ public bool Notify(ReadOnlySpan notifications) } } - wake = _ready.Count > 0 && (wasEmpty || settings.HighPriority); + wake = _ready.Count > 0 && wasEmpty; } } @@ -135,16 +134,16 @@ public ValueTask PublishAsync( // Force on a retry is not a new invalidation; independent invalidations use Notify. if (_pending.TryGetValue(notification.Key, out var retained)) { - retained.AdmissionGeneration = ++_generation; + retained.AdmissionGeneration++; } - if (notification.Version <= producer.CompletedVersion && producer.Outcome is { } outcome) + if (notification.Version <= producer.CompletedVersion) { - return new(CreateReceipt(outcome)); + return new(CreateReceipt(producer.Outcome)); } completion = notification.Version <= producer.InFlightVersion && producer.InFlightReceipt is { } inFlight - ? inFlight.Task : producer.Receipt!.Task; + ? inFlight.Task : retained!.Receipt!.Task; } else { @@ -159,10 +158,7 @@ public ValueTask PublishAsync( producer.Version = notification.Version; // All newer versions before sealing share one signal and occupy one producer slot. pending.Receipt ??= new(TaskCreationOptions.RunContinuationsAsynchronously); - producer.Receipt = pending.Receipt; pending.Producer = producer; - pending.ProducerVersion = notification.Version; - _cohort!.HasPublications = true; if (_cohort!.Membership.ContainsMember(silo)) { _cohort.Contributors.Add(silo); @@ -316,7 +312,7 @@ private bool TryAddUnsafe( { if (_pending.TryGetValue(notification.Key, out pending!)) { - pending.AdmissionGeneration = ++_generation; + pending.AdmissionGeneration++; if (!notification.Force && notification.Version <= pending.Notification.Version && (!contribution || pending.Node.List is not null)) { @@ -335,7 +331,7 @@ private bool TryAddUnsafe( return false; } - pending = new(notification, ++_generation); + pending = new(notification); _pending.Add(notification.Key, pending); } @@ -387,7 +383,7 @@ private async Task RunAsync() break; } - var delay = GetDelayUnsafe(settings.HighPriority); + var delay = GetDelayUnsafe(); if (delay > TimeSpan.Zero) { _timer.Change(delay); @@ -402,10 +398,10 @@ private async Task RunAsync() var pending = _ready.First!.Value; _ready.RemoveFirst(); work[i] = new(pending, pending.Generation, pending.Notification, - pending.Receipt, pending.Producer, pending.ProducerVersion); + pending.Receipt, pending.Producer, pending.Producer?.Version ?? 0); if (pending.Producer is { } producer) { - producer.InFlightVersion = pending.ProducerVersion; + producer.InFlightVersion = producer.Version; producer.InFlightReceipt = pending.Receipt; } @@ -440,7 +436,7 @@ private async Task RunAsync() } } - private TimeSpan GetDelayUnsafe(bool highPriority) + private TimeSpan GetDelayUnsafe() { var cohort = _cohort!; if (_handoff) @@ -452,8 +448,7 @@ private TimeSpan GetDelayUnsafe(bool highPriority) ? _retryPeriod - _timeProvider.GetElapsedTime(retry) : TimeSpan.Zero; var complete = cohort.Membership.Members.Length > 0 && cohort.Contributors.Count == cohort.Membership.Members.Length; - var collectionDelay = _stopping || _drainCompletion is not null - || highPriority && !cohort.HasPublications || complete + var collectionDelay = _stopping || _drainCompletion is not null || complete ? TimeSpan.Zero : cohort.Period - _timeProvider.GetElapsedTime(cohort.Started); return collectionDelay > retryDelay ? collectionDelay : retryDelay; } @@ -612,7 +607,7 @@ private void RemoveUnsafe(PendingKey pending) { var outcome = new ReceiptOutcome(false, _cohort?.Started ?? _timeProvider.GetTimestamp(), _cohort?.Period ?? TimeSpan.Zero); CompleteReceiptUnsafe(new(pending, pending.Generation, pending.Notification, - pending.Receipt, pending.Producer, pending.ProducerVersion), outcome); + pending.Receipt, pending.Producer, pending.Producer?.Version ?? 0), outcome); pending.InFlight?.TrySetResult(outcome); if (pending.Receipt is not null && pending.Notification.Key.Value is SiloAddress silo) { @@ -638,7 +633,6 @@ private Settings GetSettings() var period = _namespace.AggregationPeriod; return new( options.Enabled && namespaceOptions.Enabled, - namespaceOptions.Priority == DisseminationPriority.High, Math.Max(1, namespaceOptions.MaxPendingItemCount), Math.Max(1, options.MaxBatchItems), TimeSpan.FromMilliseconds(Math.Clamp(period.TotalMilliseconds, 1, uint.MaxValue - 1))); @@ -767,19 +761,16 @@ private void Fail(Exception exception) private sealed class PendingKey { public KeyNotification Notification; - public long Generation; - public long AdmissionGeneration; - public long ProducerVersion; + public long Generation = 1; + public long AdmissionGeneration = 1; public Producer? Producer; public TaskCompletionSource? Receipt; public TaskCompletionSource? InFlight; public LinkedListNode Node { get; } - public PendingKey(KeyNotification notification, long generation) + public PendingKey(KeyNotification notification) { Notification = notification; - Generation = generation; - AdmissionGeneration = generation; Node = new(this); } } @@ -787,10 +778,9 @@ public PendingKey(KeyNotification notification, long generation) private sealed class Producer { public long Version; - public long CompletedVersion = long.MinValue; + public long CompletedVersion; public long InFlightVersion; - public ReceiptOutcome? Outcome; - public TaskCompletionSource? Receipt; + public ReceiptOutcome Outcome; public TaskCompletionSource? InFlightReceipt; } @@ -800,14 +790,13 @@ private sealed class Cohort(DisseminationMembershipSnapshot membership, long sta public long Started { get; } = started; public TimeSpan Period { get; } = period; public HashSet Contributors { get; } = []; - public bool HasPublications; } private readonly record struct ReceiptOutcome(bool Accepted, long Started, TimeSpan Period); private readonly record struct PendingDispatch( PendingKey Pending, long Generation, KeyNotification Notification, TaskCompletionSource? Receipt, Producer? Producer, long ProducerVersion); - private readonly record struct Settings(bool Enabled, bool HighPriority, int MaxPendingItemCount, int MaxBatchItems, TimeSpan Period); + private readonly record struct Settings(bool Enabled, int MaxPendingItemCount, int MaxBatchItems, TimeSpan Period); [LoggerMessage(Level = LogLevel.Warning, Message = "Dissemination root dispatch for {Namespace} failed. Receipts are rejected and retained hints will retry after the publication period.")] private static partial void LogDispatchFailed(ILogger logger, Exception exception, DisseminationNamespace @namespace); diff --git a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs index 7041db3714e..72ef3b3072c 100644 --- a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs @@ -1,9 +1,8 @@ -using System.Collections.Immutable; using Orleans.Configuration; namespace Orleans.Runtime.Dissemination; -// A namespace owns current state plus any history and caching needed to repair a peer from an acknowledged version. +// A namespace owns current state, serialized payload caching, and repair construction. internal interface IDisseminationNamespace { DisseminationNamespace Name { get; } @@ -12,7 +11,7 @@ internal interface IDisseminationNamespace DisseminationRoutingMode RoutingMode => DisseminationRoutingMode.BroadcastTree; - TimeSpan AggregationPeriod => Options.MaxCoalescingDelay; + TimeSpan AggregationPeriod => TimeSpan.FromSeconds(1); // Full values can be authority-refresh hints even when their numeric version is older. bool ValidateOlderFullValues => false; @@ -54,12 +53,10 @@ internal enum DisseminationRoutingMode AggregationTree, } -// A null FromVersion means no known peer baseline; a null ToVersion asks for the highest repairable version. +// A null FromVersion means no known peer baseline. Repairs materialize the current full value. internal readonly struct DisseminationRepairRequest( DisseminationKey key, long? fromVersion, - long? toVersion, - int maxItemCount, int maxBatchBytes, int maxPayloadBytes) { @@ -67,55 +64,44 @@ internal readonly struct DisseminationRepairRequest( public long? FromVersion { get; } = fromVersion; - public long? ToVersion { get; } = toVersion; - - public int MaxItemCount { get; } = maxItemCount; - public int MaxBatchBytes { get; } = maxBatchBytes; public int MaxPayloadBytes { get; } = maxPayloadBytes; } -// Version reports the namespace's resolved or current version. -// For Produced results, IsComplete says whether Values reaches that version or only forms a prefix. +// Version reports the current namespace version; a Produced result carries its full value. internal readonly struct DisseminationRepairResult( DisseminationRepairStatus status, long version, - ImmutableArray values, - bool isComplete) + DisseminationValue value) { public DisseminationRepairStatus Status { get; } = status; public long Version { get; } = version; - public ImmutableArray Values { get; } = values.IsDefault ? [] : values; - - public bool IsComplete { get; } = isComplete; + public DisseminationValue Value { get; } = value; public static DisseminationRepairResult Current(long version) => - new(DisseminationRepairStatus.Current, version, [], isComplete: true); + new(DisseminationRepairStatus.Current, version, default); - public static DisseminationRepairResult Produced( - long version, - ImmutableArray values, - bool isComplete = true) => - new(DisseminationRepairStatus.Produced, version, values, isComplete); + public static DisseminationRepairResult Produced(DisseminationValue value) => + new(DisseminationRepairStatus.Produced, value.ToVersion, value); public static DisseminationRepairResult Unavailable(long version) => - new(DisseminationRepairStatus.Unavailable, version, [], isComplete: false); + new(DisseminationRepairStatus.Unavailable, version, default); public static DisseminationRepairResult InsufficientCapacity(long version) => - new(DisseminationRepairStatus.InsufficientCapacity, version, [], isComplete: false); + new(DisseminationRepairStatus.InsufficientCapacity, version, default); } internal enum DisseminationRepairStatus { // The peer is already at or beyond the resolved version. Current, - // Values contains a valid repair, possibly a prefix when IsComplete is false. + // Value contains the current full value. Produced, - // The key or requested target cannot currently be reconstructed. + // The key has no current value. Unavailable, - // No valid repair fits within the supplied item or byte budget. + // The current value exceeds the supplied byte budget. InsufficientCapacity, } diff --git a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs index 11af67fa82a..7ba4cf935fa 100644 --- a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs @@ -1,4 +1,3 @@ -using System.Collections.Immutable; using Microsoft.Extensions.Options; using Orleans.Configuration; using Orleans.Runtime.MembershipService; @@ -6,20 +5,16 @@ namespace Orleans.Runtime.Dissemination; -// Membership retains a bounded snapshot history so each peer can receive either a compact diff -// or a universal full snapshot. +// Full snapshots preserve the complete inventory and same-version heartbeat advances. internal sealed class MembershipDisseminationNamespace( IMembershipManager membershipManager, IOptionsMonitor options, Serializer serializer) : IDisseminationNamespace { - private const int MaxSnapshotHistory = 32; private static readonly DisseminationKey[] MembershipKeys = [DisseminationKey.Default]; - private readonly object _historyLock = new(); - private readonly SortedDictionary _snapshotHistory = new(); - private readonly Dictionary> _snapshotPayloads = []; - private readonly Dictionary<(long FromVersion, long ToVersion), ReadOnlyMemory> _diffPayloads = []; - private MembershipTableSnapshot? _currentSnapshot; + private readonly object _cacheLock = new(); + private MembershipTableSnapshot? _cachedSnapshot; + private byte[]? _cachedPayload; public DisseminationNamespace Name => DisseminationNamespaceNames.Membership; @@ -31,15 +26,13 @@ internal sealed class MembershipDisseminationNamespace( public IEnumerable Keys => MembershipKeys; - public async ValueTask PublishAsync( + public ValueTask PublishAsync( IDisseminationService disseminationService, MembershipTableSnapshot snapshot, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - // Capture authoritative state: a delayed publication can outlive the snapshot which caused it. - RememberCurrentSnapshot(); - return await disseminationService.Publish( + return disseminationService.Publish( this, DisseminationKey.Default, snapshot.Version.Value, @@ -50,7 +43,7 @@ public IEnumerable Digests { get { - var snapshot = RememberCurrentSnapshot(); + var snapshot = membershipManager.CurrentSnapshot; // Version alone misses same-version liveness advances, so the digest fingerprints heartbeat state too. yield return new DigestEntry( DisseminationKey.Default, @@ -71,61 +64,21 @@ public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest requ return DisseminationRepairResult.Unavailable(version: 0); } - // Select history and cached bytes atomically because membership can change without advancing its version. - lock (_historyLock) + lock (_cacheLock) { - var currentSnapshot = RememberCurrentSnapshotUnsafe(); - var targetVersion = request.ToVersion ?? currentSnapshot.Version.Value; - if (targetVersion > currentSnapshot.Version.Value - || !_snapshotHistory.TryGetValue(targetVersion, out var targetSnapshot)) + // Re-read the owner under the cache lock: publication arguments can outlive the state they describe. + var snapshot = membershipManager.CurrentSnapshot; + var version = snapshot.Version.Value; + if (_cachedSnapshot is null || !MembershipSnapshotsEqual(_cachedSnapshot, snapshot)) { - return DisseminationRepairResult.Unavailable(currentSnapshot.Version.Value); + _cachedPayload = null; } - // A numerically greater peer version can belong to an older table incarnation. - // Only a retained lower baseline is useful for a diff; all other peers receive a full snapshot. - MembershipTableSnapshot? baseSnapshot = null; - if (request.FromVersion is { } fromVersion - && fromVersion > 0 - && fromVersion < targetVersion) - { - _snapshotHistory.TryGetValue(fromVersion, out baseSnapshot); - } - - var resolvedVersion = targetSnapshot.Version.Value; - if (request.MaxItemCount <= 0) - { - return DisseminationRepairResult.InsufficientCapacity(resolvedVersion); - } - - var snapshotValue = CreateSnapshotValue(targetSnapshot); - var selectedValue = snapshotValue; - if (baseSnapshot is not null) - { - // Use the smaller representation, retaining the full snapshot as the capacity fallback. - var diffValue = CreateDiffValue(baseSnapshot, targetSnapshot); - if (diffValue.Payload.Length < snapshotValue.Payload.Length) - { - selectedValue = diffValue; - } - } - - if (selectedValue.Payload.Length > request.MaxPayloadBytes - || selectedValue.Payload.Length > request.MaxBatchBytes) - { - if (selectedValue.FromVersion != 0 - && snapshotValue.Payload.Length <= request.MaxPayloadBytes - && snapshotValue.Payload.Length <= request.MaxBatchBytes) - { - selectedValue = snapshotValue; - } - else - { - return DisseminationRepairResult.InsufficientCapacity(resolvedVersion); - } - } - - return DisseminationRepairResult.Produced(resolvedVersion, [selectedValue]); + _cachedSnapshot = snapshot; + _cachedPayload ??= serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = snapshot }); + return _cachedPayload.Length <= request.MaxPayloadBytes && _cachedPayload.Length <= request.MaxBatchBytes + ? DisseminationRepairResult.Produced(new DisseminationValue(DisseminationKey.Default, 0, version, _cachedPayload)) + : DisseminationRepairResult.InsufficientCapacity(version); } } @@ -139,26 +92,9 @@ public async ValueTask ApplyValueAsync( return DisseminationApplyResult.Rejected; } - if (serializer.Deserialize(value.Payload) is not { } update) - { - return DisseminationApplyResult.Rejected; - } - - if (update.Diff is { } diff) - { - return update.Snapshot is null - && value.FromVersion == diff.BaseVersion.Value - && value.ToVersion == diff.Version.Value - ? await ApplyDiff(diff, cancellationToken) - : DisseminationApplyResult.Rejected; - } - - if (update.Snapshot is not { } snapshot) - { - return DisseminationApplyResult.Rejected; - } - - if (value.FromVersion != 0 || value.ToVersion != snapshot.Version.Value) + if (value.FromVersion != 0 + || serializer.Deserialize(value.Payload)?.Snapshot is not { } snapshot + || value.ToVersion != snapshot.Version.Value) { return DisseminationApplyResult.Rejected; } @@ -174,187 +110,13 @@ public async ValueTask ApplyValueAsync( } } - var result = await ApplySnapshot(currentSnapshot, snapshot, cancellationToken); - return snapshot.Version < currentSnapshot.Version && result == DisseminationApplyResult.Duplicate - ? DisseminationApplyResult.Obsolete - : result; - } - - private DisseminationValue CreateSnapshotValue(MembershipTableSnapshot snapshot) => new( - DisseminationKey.Default, - fromVersion: 0, - toVersion: snapshot.Version.Value, - GetSnapshotPayload(snapshot)); - - private DisseminationValue CreateDiffValue( - MembershipTableSnapshot baseSnapshot, - MembershipTableSnapshot snapshot) => new( - DisseminationKey.Default, - fromVersion: baseSnapshot.Version.Value, - toVersion: snapshot.Version.Value, - GetDiffPayload(baseSnapshot, snapshot)); - - private async ValueTask ApplyDiff(MembershipTableSnapshotDiff diff, CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - var current = membershipManager.CurrentSnapshot; - if (current.Version.Value > diff.Version.Value) - { - return DisseminationApplyResult.Obsolete; - } - - if (current.Version.Value != diff.Version.Value - && current.Version.Value != diff.BaseVersion.Value) - { - return DisseminationApplyResult.Rejected; - } - - // Table cleanup can change the inventory without changing its version, so a receiver's - // baseline can contain entries which the sender never saw and therefore could not remove. - var entries = diff.IncludesAllEntries - ? ImmutableDictionary.CreateBuilder() - : current.Entries.ToBuilder(); - foreach (var silo in diff.RemovedSilos) - { - entries.Remove(silo); - } - - foreach (var entry in diff.UpdatedEntries) - { - entries[entry.SiloAddress] = PreserveIAmAliveTime(current, entry); - } - - var snapshot = new MembershipTableSnapshot(diff.Version, entries.ToImmutable()); - if (current.Version.Value == diff.Version.Value && !snapshot.IsSuccessorTo(current)) - { - return DisseminationApplyResult.Duplicate; - } - - return await ApplySnapshot(current, snapshot, cancellationToken); - } - - private async ValueTask ApplySnapshot( - MembershipTableSnapshot previous, - MembershipTableSnapshot snapshot, - CancellationToken cancellationToken) - { await membershipManager.ProcessGossipSnapshot(snapshot, cancellationToken); - var current = RememberCurrentSnapshot(); - return MembershipSnapshotsEqual(previous, current) + var result = MembershipSnapshotsEqual(currentSnapshot, membershipManager.CurrentSnapshot) ? DisseminationApplyResult.Duplicate : DisseminationApplyResult.Applied; - } - - private MembershipTableSnapshot RememberCurrentSnapshot() - { - lock (_historyLock) - { - return RememberCurrentSnapshotUnsafe(); - } - } - - private MembershipTableSnapshot RememberCurrentSnapshotUnsafe() - { - var snapshot = membershipManager.CurrentSnapshot; - if (_currentSnapshot is { } previousCurrent && snapshot.Version < previousCurrent.Version) - { - _snapshotHistory.Clear(); - _snapshotPayloads.Clear(); - _diffPayloads.Clear(); - } - - _currentSnapshot = snapshot; - if (_snapshotHistory.TryGetValue(snapshot.Version.Value, out var previous) - && !MembershipSnapshotsEqual(previous, snapshot)) - { - // Replacing a same-version snapshot invalidates bytes derived from the older liveness state. - InvalidatePayloads(snapshot.Version.Value); - } - - _snapshotHistory[snapshot.Version.Value] = snapshot; - while (_snapshotHistory.Count > MaxSnapshotHistory) - { - using var enumerator = _snapshotHistory.Keys.GetEnumerator(); - if (enumerator.MoveNext()) - { - var removedVersion = enumerator.Current; - _snapshotHistory.Remove(removedVersion); - InvalidatePayloads(removedVersion); - } - } - - return snapshot; - } - - private void InvalidatePayloads(long version) - { - _snapshotPayloads.Remove(version); - foreach (var key in _diffPayloads.Keys - .Where(key => key.FromVersion == version || key.ToVersion == version) - .ToArray()) - { - _diffPayloads.Remove(key); - } - } - - private ReadOnlyMemory GetSnapshotPayload(MembershipTableSnapshot snapshot) - { - lock (_historyLock) - { - if (!_snapshotPayloads.TryGetValue(snapshot.Version.Value, out var payload)) - { - payload = serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = snapshot }); - _snapshotPayloads.Add(snapshot.Version.Value, payload); - } - - return payload; - } - } - - private ReadOnlyMemory GetDiffPayload( - MembershipTableSnapshot baseSnapshot, - MembershipTableSnapshot snapshot) - { - var key = (baseSnapshot.Version.Value, snapshot.Version.Value); - lock (_historyLock) - { - if (!_diffPayloads.TryGetValue(key, out var payload)) - { - payload = serializer.SerializeToArray(new MembershipTableSnapshotUpdate - { - Diff = CreateDiff(baseSnapshot, snapshot), - }); - _diffPayloads.Add(key, payload); - } - - return payload; - } - } - - private static MembershipTableSnapshotDiff CreateDiff(MembershipTableSnapshot baseSnapshot, MembershipTableSnapshot snapshot) - { - // Include every current entry: peers at the same table version can still have different liveness baselines. - var updated = ImmutableArray.CreateBuilder(); - foreach (var entry in snapshot.Entries) - { - updated.Add(entry.Value); - } - - var removed = ImmutableArray.CreateBuilder(); - foreach (var entry in baseSnapshot.Entries) - { - if (!snapshot.Entries.ContainsKey(entry.Key)) - { - removed.Add(entry.Key); - } - } - - return new MembershipTableSnapshotDiff( - baseSnapshot.Version, - snapshot.Version, - updated.ToImmutable(), - removed.ToImmutable(), - includesAllEntries: true); + return snapshot.Version < currentSnapshot.Version && result == DisseminationApplyResult.Duplicate + ? DisseminationApplyResult.Obsolete + : result; } private static long GetFingerprint(MembershipTableSnapshot snapshot) @@ -372,33 +134,6 @@ private static long GetFingerprint(MembershipTableSnapshot snapshot) return unchecked((long)hash); } - private static MembershipEntry PreserveIAmAliveTime(MembershipTableSnapshot previousSnapshot, MembershipEntry entry) - { - // A repair must not move locally observed liveness backward. - if (previousSnapshot.Entries.TryGetValue(entry.SiloAddress, out var previousEntry) - && previousEntry.IAmAliveTime > entry.IAmAliveTime) - { - return CopyWithIAmAliveTime(entry, previousEntry.IAmAliveTime); - } - - return entry; - } - - private static MembershipEntry CopyWithIAmAliveTime(MembershipEntry entry, DateTime iAmAliveTime) => new() - { - SiloAddress = entry.SiloAddress, - Status = entry.Status, - SuspectTimes = entry.SuspectTimes is null ? null : new(entry.SuspectTimes), - ProxyPort = entry.ProxyPort, - HostName = entry.HostName, - SiloName = entry.SiloName, - RoleName = entry.RoleName, - UpdateZone = entry.UpdateZone, - FaultZone = entry.FaultZone, - StartTime = entry.StartTime, - IAmAliveTime = iAmAliveTime, - }; - private static bool MembershipEntriesEqual(MembershipEntry left, MembershipEntry right) => MembershipTableSnapshot.AreVersionedFieldsEqual(left, right) && left.IAmAliveTime == right.IAmAliveTime; @@ -428,7 +163,6 @@ private static bool MembershipSnapshotsEqual( return true; } - } [GenerateSerializer, Immutable] @@ -436,41 +170,4 @@ internal sealed class MembershipTableSnapshotUpdate { [Id(0)] public MembershipTableSnapshot? Snapshot { get; init; } - - [Id(1)] - public MembershipTableSnapshotDiff? Diff { get; init; } -} - -[GenerateSerializer, Immutable] -internal sealed class MembershipTableSnapshotDiff -{ - public MembershipTableSnapshotDiff( - MembershipVersion baseVersion, - MembershipVersion version, - ImmutableArray updatedEntries, - ImmutableArray removedSilos, - bool includesAllEntries = false) - { - BaseVersion = baseVersion; - Version = version; - UpdatedEntries = updatedEntries; - RemovedSilos = removedSilos; - IncludesAllEntries = includesAllEntries; - } - - [Id(0)] - public MembershipVersion BaseVersion { get; } - - [Id(1)] - public MembershipVersion Version { get; } - - [Id(2)] - public ImmutableArray UpdatedEntries { get; } - - [Id(3)] - public ImmutableArray RemovedSilos { get; } - - // Absent on older payloads, where UpdatedEntries can be only a partial update. - [Id(4)] - public bool IncludesAllEntries { get; } } diff --git a/src/Orleans.Runtime/Dissemination/WakeTimer.cs b/src/Orleans.Runtime/Dissemination/WakeTimer.cs index cb3bf0078e8..3f6b89989ed 100644 --- a/src/Orleans.Runtime/Dissemination/WakeTimer.cs +++ b/src/Orleans.Runtime/Dissemination/WakeTimer.cs @@ -75,10 +75,6 @@ public void Wake() } waiter = CompleteWaitUnsafe(); - if (waiter is null) - { - _signaled = true; - } } waiter?.TrySetResult(true); @@ -153,14 +149,8 @@ public void Dispose() _armed = false; } - try - { - waiter?.TrySetResult(false); - } - finally - { - _timer.Dispose(); - } + waiter?.TrySetResult(false); + _timer.Dispose(); } private TaskCompletionSource? CompleteWaitUnsafe() @@ -169,6 +159,7 @@ public void Dispose() _waiter = null; _armed = false; _timer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + _signaled = waiter is null; return waiter; } @@ -190,10 +181,6 @@ private void OnTimer() } waiter = CompleteWaitUnsafe(); - if (waiter is null) - { - _signaled = true; - } } waiter?.TrySetResult(true); diff --git a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs index 9b65a7bbe5b..b49f969b139 100644 --- a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs +++ b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs @@ -37,7 +37,6 @@ internal sealed partial class DeploymentLoadPublisher : SystemTarget, IDeploymen private long _lastUpdateDateTimeTicks; private IGrainTimer? _publishTimer; private Task? _publicationTask; - private Task? _disseminationPublicationTask; public ConcurrentDictionary PeriodicStatistics => _periodicStats; @@ -127,7 +126,7 @@ private async Task PublishStatisticsAndGetReceipt(Cancell // submitting another source publication while its cohort is still open. if (_publicationTask is { IsCompleted: false } pending) { - return await AwaitSharedPublication(pending, cancellationToken); + return await pending.WaitAsync(cancellationToken); } // Native operations own this caller's cancellation and may complete successfully @@ -234,47 +233,9 @@ internal bool IsRuntimeStatisticsObsolete(SiloAddress siloAddress, long timestam internal Dictionary GetActiveSiloStatusesForStatisticsDigest() => _siloStatusOracle.GetApproximateSiloStatuses(onlyActive: true); - internal async Task TryPublishStatisticsViaDissemination( - SiloRuntimeStatistics myStats, - CancellationToken cancellationToken) => - (await PublishStatisticsViaDissemination(myStats, cancellationToken)).Receipt.Accepted; - private async Task PublishStatisticsViaDissemination( SiloRuntimeStatistics myStats, CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - if (_disseminationPublicationTask is { IsCompleted: false } pending) - { - return await AwaitSharedPublication(pending, cancellationToken); - } - - return await (_disseminationPublicationTask = PublishStatisticsViaDisseminationCore(myStats, cancellationToken)); - } - - private static async Task AwaitSharedPublication( - Task publication, - CancellationToken cancellationToken) - { - try - { - return await publication.WaitAsync(cancellationToken); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - if (publication.IsCompletedSuccessfully) - { - return await publication; - } - - cancellationToken.ThrowIfCancellationRequested(); - throw; - } - } - - private async Task PublishStatisticsViaDisseminationCore( - SiloRuntimeStatistics myStats, - CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); if (_statisticsRefreshTime <= TimeSpan.Zero) diff --git a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs index 4673ca09681..72f59d1cc80 100644 --- a/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs +++ b/src/api/Orleans.Core.Abstractions/Orleans.Core.Abstractions.cs @@ -3003,7 +3003,7 @@ public sealed partial class ResourceOptimizedPlacement : PlacementStrategy [System.Text.Json.Serialization.JsonConverter(typeof(SiloAddressConverter))] [System.Diagnostics.DebuggerDisplay("SiloAddress {ToString()}")] [SuppressReferenceTracking] - public sealed partial class SiloAddress : System.IEquatable, System.IComparable, System.IComparable, System.ISpanFormattable, System.IFormattable, System.IParsable, System.IUtf8SpanParsable + public sealed partial class SiloAddress : System.IEquatable, System.IComparable, System.ISpanFormattable, System.IFormattable, System.IParsable, System.IUtf8SpanParsable { internal SiloAddress() { } @@ -3051,8 +3051,6 @@ internal SiloAddress() { } public static SiloAddress Parse(string value) { throw null; } - int System.IComparable.CompareTo(object? obj) { throw null; } - string System.IFormattable.ToString(string? format, System.IFormatProvider? formatProvider) { throw null; } bool System.ISpanFormattable.TryFormat(System.Span destination, out int charsWritten, System.ReadOnlySpan format, System.IFormatProvider? provider) { throw null; } diff --git a/src/api/Orleans.Core/Orleans.Core.cs b/src/api/Orleans.Core/Orleans.Core.cs index 62eec4aa7be..d2906789dfa 100644 --- a/src/api/Orleans.Core/Orleans.Core.cs +++ b/src/api/Orleans.Core/Orleans.Core.cs @@ -540,14 +540,10 @@ public sealed partial class DisseminationNamespaceOptions public System.TimeSpan ExpectedUpdateCadence { get { throw null; } set { } } - public System.TimeSpan MaxCoalescingDelay { get { throw null; } set { } } - public int MaxPayloadBytes { get { throw null; } set { } } public int MaxPendingItemCount { get { throw null; } set { } } - public DisseminationPriority Priority { get { throw null; } set { } } - public System.TimeSpan StaleItemTtl { get { throw null; } set { } } } @@ -585,12 +581,6 @@ public sealed partial class DisseminationOverlayOptions public int TargetHopCount { get { throw null; } set { } } } - public enum DisseminationPriority - { - Normal = 0, - High = 1 - } - public partial class GatewayOptions { public const int DEFAULT_PREFERED_GATEWAY_INDEX = -1; diff --git a/test/Dissemination.IntegrationHarness/README.md b/test/Dissemination.IntegrationHarness/README.md index 06dba28a4df..0a32851a3bf 100644 --- a/test/Dissemination.IntegrationHarness/README.md +++ b/test/Dissemination.IntegrationHarness/README.md @@ -15,7 +15,7 @@ that checkout; Orleans runtime sources remain unchanged. - Rolling upgrades, mixed enablement, legacy fallback, and rollback. - Exact load-state convergence after three partition/heal cycles, at four and eight silos, using original, current-disabled, and current-enabled runtimes. -- History-miss and same-version heartbeat repair with tree delivery, direct +- Full-snapshot and same-version heartbeat repair with tree delivery, direct gossip, and membership-table reads independently isolated. - Cancellation of an entered remote RPC and bounded partitioned shutdown. - Harness guards for connection draining, exact inventory/value comparison, diff --git a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs index 13ef9d5e62a..572cfef58ab 100644 --- a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs +++ b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs @@ -47,7 +47,6 @@ private static void ConfigureRecovery(DisseminationNamespaceOptions options, boo if (fastRecovery) { options.ExpectedUpdateCadence = TimeSpan.FromMilliseconds(100); - options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(25); options.StaleItemTtl = TimeSpan.FromSeconds(1); } } @@ -142,39 +141,34 @@ public static async Task Execute(IServiceProvider services, Comman case "publish-membership": await ns.PublishAsync(dissemination, manager.CurrentSnapshot, cancellationToken); break; - case "membership-history": + case "membership-update": { if (!services.GetRequiredService().Suppressed) { - throw new InvalidOperationException("History injection requires legacy gossip isolation."); + throw new InvalidOperationException("Membership injection requires legacy gossip isolation."); } var table = services.GetRequiredService(); await table.FreezeReads(false, cancellationToken); - for (var index = 0; index < command.Count; index++) + var current = await table.ReadAllAsync(cancellationToken); + var row = current.Members.Single(row => row.Item1.SiloAddress.Equals(local)); + row.Item1.HostName = $"membership-update-{current.Version.Version + 1}"; + if (!await table.UpdateRowAsync(row.Item1, row.Item2, current.Version.Next(), cancellationToken)) { - cancellationToken.ThrowIfCancellationRequested(); - var current = await table.ReadAllAsync(cancellationToken); - var row = current.Members.Single(row => row.Item1.SiloAddress.Equals(local)); - row.Item1.HostName = $"retained-history-{current.Version.Version + 1}"; - if (!await table.UpdateRowAsync(row.Item1, row.Item2, current.Version.Next(), cancellationToken)) - { - throw new InvalidOperationException("Unexpected concurrent membership writer during isolated history test."); - } - - await manager.Refresh(null, cancellationToken); - await ns.PublishAsync(dissemination, manager.CurrentSnapshot, cancellationToken); + throw new InvalidOperationException("Unexpected concurrent membership writer during isolated repair test."); } + await manager.Refresh(null, cancellationToken); + await ns.PublishAsync(dissemination, manager.CurrentSnapshot, cancellationToken); await table.FreezeReads(true, cancellationToken); var repair = ns.CreateRepair(new( - DisseminationKey.Default, command.Version, null, 100, 1024 * 1024, 1024 * 1024)); - if (repair.Status != DisseminationRepairStatus.Produced || repair.Values.Length != 1) + DisseminationKey.Default, command.Version, 1024 * 1024, 1024 * 1024)); + if (repair.Status != DisseminationRepairStatus.Produced) { throw new InvalidOperationException($"Expected a production membership repair, got {repair.Status}."); } - return Program.Snapshot(services) with { RepairFromVersion = repair.Values[0].FromVersion }; + return Program.Snapshot(services) with { RepairFromVersion = repair.Value.FromVersion }; } case "membership-heartbeat": { diff --git a/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs index 645e9a92e4b..152755d62af 100644 --- a/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs +++ b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs @@ -157,9 +157,9 @@ await cluster.Eventually("enabled peers confirm support, explicitly disabled pee [Fact] [Trait("Category", "DisseminationProcess")] - public async Task PartitionHistoryAndHeartbeat_RequireDisseminationProvenance() + public async Task PartitionSnapshotAndHeartbeat_RequireDisseminationProvenance() { - await using var cluster = new ProcessCluster("partition-history-heartbeat"); + await using var cluster = new ProcessCluster("partition-snapshot-heartbeat"); var origin = await cluster.Start("New", enabled: true); var relay = await cluster.Start("New", enabled: true); var receiver = await cluster.Start("New", enabled: true); @@ -196,18 +196,16 @@ await cluster.Eventually("identical membership before isolation", async () => var oldVersion = receiver.Last.MembershipVersion; await receiver.Send("partition", value: true); await relay.Send("partition", value: true); - var source = await origin.Send("membership-history", count: 40, version: oldVersion); - Assert.Equal(oldVersion + 40, source.MembershipVersion); - Assert.Equal(0, source.RepairFromVersion); // The actual production namespace evicted the old baseline. + var source = await origin.Send("membership-update", version: oldVersion); + Assert.Equal(oldVersion + 1, source.MembershipVersion); + Assert.Equal(0, source.RepairFromVersion); var blocked = await receiver.Send("snapshot"); Assert.Equal(oldVersion, blocked.MembershipVersion); Assert.Equal(oldVersion, (await relay.Send("snapshot")).MembershipVersion); Assert.False(SameMembership(source, blocked)); await receiver.Send("partition", value: false); - // Only the origin can repair this receiver until its full history-miss repair is observed. - // A relay which first learned the target could otherwise serve a valid retained-baseline delta. - await cluster.Eventually("anti-entropy-only full repair after a real transport partition and retained-history miss", async () => + await cluster.Eventually("anti-entropy-only full snapshot repair after a real transport partition", async () => { await receiver.Send("snapshot"); return SameMembership(source, receiver.Last) @@ -225,7 +223,7 @@ await cluster.Eventually("relay convergence after the receiver's full repair", a await AssertNoTreeAdmissions(cluster, before); Assert.True(receiver.Last.MembershipReadsFrozen); Assert.True(receiver.Last.LegacyGossipSuppressed); - await cluster.Save("history-recovery.json", new { Before = before, After = cluster.Active.Select(node => node.Last) }); + await cluster.Save("snapshot-recovery.json", new { Before = before, After = cluster.Active.Select(node => node.Last) }); var priorHeartbeatApplications = receiver.Last.Applies.Count(evidence => evidence.Namespace == "membership" && evidence.Result == "Applied" && evidence.ToVersion == source.MembershipVersion); diff --git a/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs b/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs index 77cac0e21dd..8419c0c9ea5 100644 --- a/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs +++ b/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs @@ -334,9 +334,10 @@ public async Task PublishStatistics_CancellationPreservesNativeCompletion() } [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task RefreshClusterStatistics_Cancellation_CancelsNativeRequests(bool useLinkedReceiverToken) + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + public async Task RefreshClusterStatistics_Cancellation_PreservesNativeOutcome(bool completeSuccessfully, bool useLinkedReceiverToken) { using var rig = CreateTestRig(TimeSpan.Zero); using var cancellation = new CancellationTokenSource(); @@ -347,11 +348,21 @@ public async Task RefreshClusterStatistics_Cancellation_CancelsNativeRequests(bo var requestToken = await requested.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); cancellation.Cancel(); - await Assert.ThrowsAnyAsync( - () => refresh.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); Assert.Equal(cancellation.Token, requestToken); Assert.True(requestToken.IsCancellationRequested); - Assert.Empty(rig.Publisher.PeriodicStatistics); + if (completeSuccessfully) + { + await refresh.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.True(refresh.IsCompletedSuccessfully); + Assert.Equal(2, rig.Publisher.PeriodicStatistics.Count); + Assert.All(rig.Publisher.PeriodicStatistics.Values, statistics => Assert.Same(rig.InitialStatistics, statistics)); + } + else + { + await Assert.ThrowsAnyAsync( + () => refresh.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Empty(rig.Publisher.PeriodicStatistics); + } async Task ReadStatistics(CancellationToken token) { @@ -361,37 +372,17 @@ async Task ReadStatistics(CancellationToken token) requested.TrySetResult(token); } - await Task.Delay(Timeout.InfiniteTimeSpan, useLinkedReceiverToken ? receiverCancellation.Token : token); - return rig.InitialStatistics; - } - } - - [Fact] - public async Task RefreshClusterStatistics_CompletedRequests_PreserveNativeResults() - { - using var rig = CreateTestRig(TimeSpan.Zero); - using var cancellation = new CancellationTokenSource(); - var requested = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var startedRequests = 0; - rig.Control.GetRuntimeStatistics(Arg.Any()).Returns(async call => - { - if (Interlocked.Increment(ref startedRequests) == 2) + if (completeSuccessfully) + { + await token.WhenCancelled(); + } + else { - requested.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, useLinkedReceiverToken ? receiverCancellation.Token : token); } - await call.ArgAt(0).WhenCancelled(); return rig.InitialStatistics; - }); - var refresh = rig.Publisher.RefreshClusterStatistics(cancellation.Token); - await requested.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - - cancellation.Cancel(); - await refresh.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - - Assert.True(refresh.IsCompletedSuccessfully); - Assert.Equal(2, rig.Publisher.PeriodicStatistics.Count); - Assert.All(rig.Publisher.PeriodicStatistics.Values, statistics => Assert.Same(rig.InitialStatistics, statistics)); + } } [Fact] diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs index f07fe64af26..905226e19d4 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs @@ -31,7 +31,6 @@ public void DefaultsAreBoundedAndOptIn() Assert.Equal(1048576, options.Overlay.MaxAntiEntropyBatchBytes); Assert.Equal(options.MaxBatchItems, options.Overlay.MaxAntiEntropyBatchItems); Assert.Equal(options.MaxBatchBytes, options.Overlay.MaxAntiEntropyBatchBytes); - Assert.Equal(new DisseminationNamespaceOptions().MaxCoalescingDelay, load.Dissemination.MaxCoalescingDelay); Assert.Equal(8192, load.Dissemination.MaxPendingItemCount); Assert.Equal(TimeSpan.FromSeconds(5), load.Dissemination.ExpectedUpdateCadence); Assert.Equal(TimeSpan.FromSeconds(1), load.DeploymentLoadPublisherRefreshTime); @@ -39,7 +38,6 @@ public void DefaultsAreBoundedAndOptIn() Assert.False(new DisseminationNamespaceOptions().Enabled); Assert.False(load.Dissemination.Enabled); Assert.False(membership.Dissemination.Enabled); - Assert.Equal(DisseminationPriority.High, membership.Dissemination.Priority); Assert.Equal(ValidateOptionsResult.Success, Validate(options)); Assert.Equal(ValidateOptionsResult.Success, new DeploymentLoadPublisherOptionsValidator().Validate(Options.DefaultName, load)); } diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs index ab1fa689c38..223b5818c18 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs @@ -272,9 +272,9 @@ public void ScopeProjectionOrderingDoesNotDependOnStartTimePrecision(bool active static DisseminationMembershipSnapshots CreateProjections(MembershipEntry[] entries, SiloAddress local) => new DisseminationMembership( - new CountingMembershipManager(new( + new MutableMembershipManager(new( new MembershipVersion(42), - entries.ToImmutableDictionary(static entry => entry.SiloAddress))), + entries.ToImmutableDictionary(static entry => entry.SiloAddress)), TestContext.Current.CancellationToken), new ScopeLocalSiloDetails(local), Microsoft.Extensions.Options.Options.Create(new DisseminationOptions { @@ -551,7 +551,7 @@ public async Task StaleConcurrentReadCannotRestoreSameVersionRemovedParticipant( private static ( DisseminationMembershipSnapshots Snapshots, - CountingMembershipManager Manager, + MutableMembershipManager Manager, ScopeSilos Silos) CreateScopeProjections(bool reverseSourceEntries = false) { var silos = new ScopeSilos( @@ -580,7 +580,7 @@ private static ( var source = new MembershipTableSnapshot( new MembershipVersion(42), entries.ToImmutableDictionary(static entry => entry.SiloAddress)); - var manager = new CountingMembershipManager(source); + var manager = new MutableMembershipManager(source, TestContext.Current.CancellationToken); var membership = new DisseminationMembership( manager, new ScopeLocalSiloDetails(silos.Local), @@ -644,62 +644,6 @@ private sealed class ScopeLocalSiloDetails(SiloAddress siloAddress) : ILocalSilo public SiloAddress GatewayAddress => siloAddress; } - private sealed class CountingMembershipManager( - MembershipTableSnapshot snapshot) - : Orleans.Runtime.MembershipService.IMembershipManager - { - public int SnapshotReadCount { get; private set; } - - public MembershipTableSnapshot CurrentSnapshot - { - get - { - SnapshotReadCount++; - return snapshot; - } - } - - public IAsyncEnumerable MembershipUpdates => - EmptyUpdates(TestContext.Current.CancellationToken); - - public SiloStatus LocalSiloStatus => SiloStatus.Active; - - public Task UpdateLocalStatus(SiloStatus status, CancellationToken cancellationToken) => Task.CompletedTask; - - public Task TryKillSilo(SiloAddress silo, CancellationToken cancellationToken) => Task.FromResult(false); - - public Task TrySuspectSilo( - SiloAddress silo, - SiloAddress? indirectProbingSilo, - CancellationToken cancellationToken) => Task.FromResult(false); - - public Task Refresh(MembershipVersion? targetVersion, CancellationToken cancellationToken, bool requireFresh = false) => Task.CompletedTask; - - public Task ProcessGossipSnapshot( - MembershipTableSnapshot value, - CancellationToken cancellationToken) => Task.CompletedTask; - - public Task UpdateIAmAlive(CancellationToken cancellationToken) => Task.CompletedTask; - - public bool CheckHealth(DateTime lastCheckTime, out string reason) - { - reason = string.Empty; - return true; - } - - public void Participate(ISiloLifecycle lifecycle) - { - } - - private static async IAsyncEnumerable EmptyUpdates( - [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - await Task.CompletedTask; - yield break; - } - } - private sealed class MutableMembershipManager( MembershipTableSnapshot snapshot, CancellationToken cancellationToken) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs index c509ed66e07..71c63c97625 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs @@ -82,7 +82,7 @@ public async Task AggregationTreeDeliversWholeRoundsWithLinearBatchCount(int cou RoutingMode = DisseminationRoutingMode.AggregationTree, MembershipScope = DisseminationMembershipScope.ActiveMembers, }; - ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); + ns.AggregationPeriod = TimeSpan.FromHours(1); var transport = new FakeTransport(member, members.Where(peer => !peer.Equals(member)).ToArray()); var protocol = CreateProtocol(transport, ns, options => { @@ -168,7 +168,7 @@ public async Task RootAggregationBatchesOnceAndCrossesMultipleLevelsWithoutAnoth MembershipScope = DisseminationMembershipScope.ActiveMembers, ApplyObserved = new(TaskCreationOptions.RunContinuationsAsynchronously), }; - ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(500); + ns.AggregationPeriod = TimeSpan.FromMilliseconds(500); var transport = new FakeTransport(member, members.Where(peer => !peer.Equals(member)).ToArray()); var protocol = CreateProtocol(transport, ns, options => { @@ -225,6 +225,43 @@ await Task.WhenAll(members.Where(silo => !silo.Equals(origin)) } } + [Fact] + public async Task MembershipBroadcastsDrainAheadOfAggregatedValues() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40821); + var peer = CreateSilo(40822); + var transport = new FakeTransport(local, peer); + var membership = new FakeNamespace(local, "membership"); + var load = new FakeNamespace(local, "load", DisseminationMembershipScope.ActiveMembers) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + }; + membership.SetValue("value", 1); + load.SetValue("value", 2); + var queue = CreateBroadcastQueue(transport, [load, membership], options => options.MaxBatchItems = 1); + try + { + var accepted = BeforeBroadcastPumpsRun(() => ( + Load: queue.Notify(peer, load, "value"), + Membership: queue.Notify(peer, membership, "value"))); + Assert.True(accepted.Load); + Assert.True(accepted.Membership); + await queue.FlushPendingBroadcast(cancellationToken); + + Assert.Equal( + new[] { (membership.Name, 1L), (load.Name, 2L) }, + transport.BroadcastBatches.Select(batch => ( + Assert.Single(batch.Batch.Values.Keys), + Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion))); + Assert.All(transport.BroadcastBatches, batch => Assert.Equal(peer, batch.Peer)); + } + finally + { + await queue.StopAsync(cancellationToken); + } + } + [Fact] public async Task AggregationDistributionRequiresBroadcastAcknowledgmentBeforeSuppressingKnownValue() { @@ -235,14 +272,19 @@ public async Task AggregationDistributionRequiresBroadcastAcknowledgmentBeforeSu RoutingMode = DisseminationRoutingMode.AggregationTree, MembershipScope = DisseminationMembershipScope.ActiveMembers, }; - ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(500); + ns.AggregationPeriod = TimeSpan.FromMilliseconds(500); var transport = new FakeTransport(members[0], members[1]); var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); try { ns.SetValue("value", 1); - Assert.True(queue.Notify(members[1], ns, "value", force: false)); - queue.ObservePeerVersion(members[1], ns.Name, "value", 1); + Assert.True(BeforeBroadcastPumpsRun(() => + { + var accepted = queue.Notify(members[1], ns, "value", force: false); + queue.ObservePeerVersion(members[1], ns.Name, "value", 1); + Assert.Empty(transport.BroadcastBatches); + return accepted; + })); await queue.FlushPendingBroadcast(cancellationToken); var batch = Assert.Single(transport.BroadcastBatches).Batch; @@ -272,7 +314,7 @@ public async Task FormerAggregationRootRedirectsIngressToCurrentRootImmediately( RoutingMode = DisseminationRoutingMode.AggregationTree, MembershipScope = DisseminationMembershipScope.ActiveMembers, }; - ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(500); + ns.AggregationPeriod = TimeSpan.FromMilliseconds(500); var transport = new FakeTransport(local, members.Where(peer => !peer.Equals(local)).ToArray()); var forwarded = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); transport.SendBroadcastResponseHandler = (peer, batch, _) => @@ -317,7 +359,7 @@ public async Task AggregationRelayForwardsWholeBatchAtomicallyWithoutCoalescing( RoutingMode = DisseminationRoutingMode.AggregationTree, MembershipScope = DisseminationMembershipScope.ActiveMembers, }; - ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(500); + ns.AggregationPeriod = TimeSpan.FromMilliseconds(500); var transport = new FakeTransport(local, members.Where(peer => !peer.Equals(local)).ToArray()); var forwarded = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var count = 0; @@ -374,7 +416,7 @@ public void RootAggregationRequiresAddressOrderedActiveMembership() } [Fact] - public async Task QueueCoalescingWindowBatchesStaggeredKeysAndPreservesNextWindow() + public async Task QueueBatchesAcceptedUpdatesAcrossInFlightSend() { var cancellationToken = TestContext.Current.CancellationToken; var local = CreateSilo(40801); @@ -385,7 +427,6 @@ public async Task QueueCoalescingWindowBatchesStaggeredKeysAndPreservesNextWindo RoutingMode = DisseminationRoutingMode.AggregationTree, MembershipScope = DisseminationMembershipScope.ActiveMembers, }; - ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(500); var transport = new FakeTransport(local, peer); var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var secondStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -416,31 +457,27 @@ public async Task QueueCoalescingWindowBatchesStaggeredKeysAndPreservesNextWindo for (var key = 0; key < 100; key++) { ns.SetValue($"key-{key}", 1); - Assert.True(queue.Notify(peer, ns, $"key-{key}")); - clock.Advance(TimeSpan.FromMilliseconds(1)); } - clock.Advance(TimeSpan.FromMilliseconds(399)); - Assert.Empty(transport.BroadcastBatches); - clock.Advance(TimeSpan.FromMilliseconds(1)); + Assert.True(queue.NotifyBatch(peer, ns, + [.. Enumerable.Range(0, 100).Select(key => new DisseminationBroadcastQueue.KeyNotification($"key-{key}", 1, true))])); await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); var firstFlush = queue.FlushPendingBroadcast(cancellationToken); Assert.Equal(100, GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch).Count()); - Assert.Equal(TimeSpan.FromMilliseconds(500), clock.GetElapsedTime(start, timestamps[0])); + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(start, timestamps[0])); + Assert.All(GetBroadcastValues(transport.BroadcastBatches[0].Batch), value => Assert.Equal(1, value.Value.ToVersion)); for (var key = 0; key < 100; key++) { ns.SetValue($"key-{key}", 2); - Assert.True(queue.Notify(peer, ns, $"key-{key}")); - clock.Advance(TimeSpan.FromMilliseconds(1)); } + Assert.True(queue.NotifyBatch(peer, ns, + [.. Enumerable.Range(0, 100).Select(key => new DisseminationBroadcastQueue.KeyNotification($"key-{key}", 2, true))])); + Assert.Single(transport.BroadcastBatches); releaseFirst.SetResult(); await firstFlush.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - clock.Advance(TimeSpan.FromMilliseconds(399)); - Assert.Single(transport.BroadcastBatches); - clock.Advance(TimeSpan.FromMilliseconds(1)); await secondStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - Assert.Equal(TimeSpan.FromMilliseconds(1000), clock.GetElapsedTime(start, timestamps[1])); + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(start, timestamps[1])); Assert.Equal(2, transport.BroadcastBatches.Count); Assert.Equal(100, GetBroadcastValues(transport.BroadcastBatches[1].Batch).Count()); Assert.All(GetBroadcastValues(transport.BroadcastBatches[1].Batch), value => Assert.Equal(2, value.Value.ToVersion)); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs index 1db66153030..0ec69d226fb 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs @@ -165,13 +165,7 @@ public async Task CohortPublisherCallerCancellationPreservesIdentityWithoutFallb await harness.StartAsync(); harness.Dissemination.AutomaticReceipt = null; using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); - var publish = harness.Publisher.RunOrQueueTask( - async token => - { - await harness.Publisher.PublishStatistics(token); - return true; - }, - caller.Token); + var publish = harness.PublishAsync(caller.Token); var publication = await harness.Dissemination.NextPublicationAsync(); caller.Cancel(); var exception = await Assert.ThrowsAnyAsync( @@ -197,23 +191,11 @@ public async Task CohortPublisherOwningCancellationPreservesNativeDirectCompleti directStarted.TrySetResult(token); await cancellationObserved.Task; }; - var owning = harness.Publisher.RunOrQueueTask( - async token => - { - await harness.Publisher.PublishStatistics(token); - return true; - }, - caller.Token); + var owning = harness.PublishAsync(caller.Token); try { var nativeToken = await directStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - var joining = harness.Publisher.RunOrQueueTask( - async token => - { - await harness.Publisher.PublishStatistics(token); - return true; - }, - TestContext.Current.CancellationToken); + var joining = harness.PublishAsync(TestContext.Current.CancellationToken); await harness.OwnerBarrierAsync(); var nativeSample = harness.Publisher.LocalRuntimeStatistics; Assert.False(owning.IsCompleted); @@ -245,13 +227,7 @@ public async Task CohortPublisherOwningCancellationPreservesNativeReceiptComplet harness.Dissemination.AutomaticReceipt = null; harness.Dissemination.PreserveNativeCompletion = true; using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); - var owning = harness.Publisher.RunOrQueueTask( - async token => - { - await harness.Publisher.PublishStatistics(token); - return true; - }, - caller.Token); + var owning = harness.PublishAsync(caller.Token); var publication = await harness.Dissemination.NextPublicationAsync(); caller.Cancel(); publication.Receipt.SetResult(new(true, harness.Period)); @@ -303,13 +279,7 @@ public async Task CohortPublisherOverlappingCallerCancellationDoesNotCancelPendi var tick = await harness.StartTickAsync(); var publication = await harness.Dissemination.NextPublicationAsync(); using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); - var overlapping = harness.Publisher.RunOrQueueTask( - async token => - { - await harness.Publisher.PublishStatistics(token); - return true; - }, - caller.Token); + var overlapping = harness.PublishAsync(caller.Token); await harness.OwnerBarrierAsync(); caller.Cancel(); var exception = await Assert.ThrowsAnyAsync( @@ -357,13 +327,7 @@ public async Task CohortPublisherExplicitPublicationPreservesDisposedTimerPause( var firstVersion = harness.Publisher.LocalRuntimeStatistics.DateTime.Ticks; await harness.Lifecycle.OnStop(TestContext.Current.CancellationToken); - await harness.Publisher.RunOrQueueTask( - async token => - { - await harness.Publisher.PublishStatistics(token); - return true; - }, - TestContext.Current.CancellationToken); + await harness.PublishAsync(TestContext.Current.CancellationToken); harness.Clock.Advance(harness.Period * 3); Assert.True(harness.Publisher.LocalRuntimeStatistics.DateTime.Ticks > firstVersion); @@ -376,23 +340,6 @@ await harness.Publisher.RunOrQueueTask( Assert.Empty(harness.DirectUpdates); } - [Fact] - public async Task CohortPublisherCompatibilityWrapperPublishesOneReceipt() - { - await using var harness = new CohortPublisherHarness(); - await harness.StartAsync(); - var result = await harness.Publisher.RunOrQueueTask( - token => harness.Publisher.TryPublishStatisticsViaDissemination( - harness.Publisher.LocalRuntimeStatistics, token), - TestContext.Current.CancellationToken); - - Assert.True(result); - Assert.Equal( - harness.Publisher.LocalRuntimeStatistics.DateTime.Ticks, - Assert.Single(harness.Dissemination.Publications).Version); - Assert.Empty(harness.DirectUpdates); - } - private sealed class CohortPublisherHarness : IAsyncDisposable { private readonly ServiceProvider _serializerServices; @@ -508,6 +455,14 @@ await Publisher.RunOrQueueTask(() => public Task OwnerBarrierAsync() => Publisher.QueueTask(static () => Task.CompletedTask); + public Task PublishAsync(CancellationToken cancellationToken) => Publisher.RunOrQueueTask( + async token => + { + await Publisher.PublishStatistics(token); + return true; + }, + cancellationToken); + public async ValueTask DisposeAsync() { await Lifecycle.OnStop(CancellationToken.None); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Cohorts.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Cohorts.cs index 10c107d59e6..a3ec52426bb 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Cohorts.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Cohorts.cs @@ -18,7 +18,6 @@ public async Task HeldCohortReceiptKeepsMembershipPeerAdmissionAvailable() var local = CreateSilo(41302); var load = CreateCohortNamespace(local); var membership = new FakeNamespace(local, new DisseminationNamespace("urgent-membership")); - membership.Options.Priority = DisseminationPriority.High; var transport = new FakeTransport(local, peer); var ingressStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var heldReceipt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -148,8 +147,10 @@ public async Task StoppingRootSealsHeldIngressBeforeDrainingProtocolAdmissions() } } - [Fact] - public async Task CanceledPublicationReleasesLocalAttemptBeforeNonCooperativeRpcReturns() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CanceledPublicationReleasesLocalAttemptBeforeNonCooperativeRpcReturns(bool stopProtocol) { var token = TestContext.Current.CancellationToken; var root = CreateSilo(41411); @@ -170,12 +171,21 @@ public async Task CanceledPublicationReleasesLocalAttemptBeforeNonCooperativeRpc ns.SetValue(local, 1); var first = protocol.PublishAggregated(ns, local, 1, cancellation.Token).AsTask(); await started.Task.WaitAsync(TimeSpan.FromSeconds(5), token); - cancellation.Cancel(); - var exception = await Assert.ThrowsAnyAsync(() => first); - Assert.Equal(cancellation.Token, exception.CancellationToken); - ns.SetValue(local, 2); - Assert.True((await protocol.PublishAggregated(ns, local, 2, token)).Accepted); - Assert.Equal(2, transport.PublicationRequests.Count); + if (stopProtocol) + { + await protocol.StopAsync(token).WaitAsync(TimeSpan.FromSeconds(5), token); + await Assert.ThrowsAnyAsync(() => first); + Assert.False((await protocol.PublishAggregated(ns, local, 1, token)).Accepted); + } + else + { + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync(() => first); + Assert.Equal(cancellation.Token, exception.CancellationToken); + ns.SetValue(local, 2); + Assert.True((await protocol.PublishAggregated(ns, local, 2, token)).Accepted); + Assert.Equal(2, transport.PublicationRequests.Count); + } Assert.False(late.Task.IsCompleted); } finally @@ -214,39 +224,6 @@ public async Task PublicationToFormerRootRejectsBeforeOwnerMutation() } } - [Fact] - public async Task ShutdownCancelsHeldOutgoingReceiptAndDrainsItsAdmission() - { - var token = TestContext.Current.CancellationToken; - var root = CreateSilo(41441); - var local = CreateSilo(41442); - var ns = CreateCohortNamespace(local); - var transport = new FakeTransport(local, root); - var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var late = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - transport.PublishAggregatedHandler = (_, _, _) => - { - started.TrySetResult(); - return late.Task; - }; - var protocol = CreateProtocol(transport, ns); - try - { - ns.SetValue(local, 1); - var publication = protocol.PublishAggregated(ns, local, 1, token).AsTask(); - await started.Task.WaitAsync(TimeSpan.FromSeconds(5), token); - await protocol.StopAsync(token).WaitAsync(TimeSpan.FromSeconds(5), token); - await Assert.ThrowsAnyAsync(() => publication); - Assert.False(late.Task.IsCompleted); - Assert.False((await protocol.PublishAggregated(ns, local, 1, token)).Accepted); - } - finally - { - late.TrySetResult(default); - await protocol.StopAsync(token); - } - } - [Fact] public void CohortPublicationWireContractPreservesPayloadAndSchedulingReceipt() { @@ -273,7 +250,6 @@ private static FakeNamespace CreateCohortNamespace(SiloAddress local) RoutingMode = DisseminationRoutingMode.AggregationTree, MembershipScope = DisseminationMembershipScope.ActiveMembers, }; - result.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); return result; } @@ -290,7 +266,7 @@ public async Task RootPendingStateHandsOffImmediatelyWhenTheRootChanges() RoutingMode = DisseminationRoutingMode.AggregationTree, MembershipScope = DisseminationMembershipScope.ActiveMembers, }; - ns.Options.MaxCoalescingDelay = TimeSpan.FromMilliseconds(25); + ns.AggregationPeriod = TimeSpan.FromMilliseconds(25); var transport = new FakeTransport(local, peer); var handedOff = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); transport.SendBroadcastResponseHandler = (target, batch, _) => @@ -335,7 +311,7 @@ public async Task RootAcceptsAndDistributesACompleteTwoThousandSiloInventory() }; var defaults = new DeploymentLoadPublisherOptions().Dissemination; ns.Options.MaxPendingItemCount = defaults.MaxPendingItemCount; - ns.Options.MaxCoalescingDelay = defaults.MaxCoalescingDelay; + ns.AggregationPeriod = new DeploymentLoadPublisherOptions().DeploymentLoadPublisherRefreshTime; var transport = new FakeTransport(local, members[1..]); var protocol = CreateProtocol(transport, ns, timeProvider: new FakeTimeProvider()); try diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs index 876f5d4564b..aef884ce0c2 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs @@ -20,10 +20,8 @@ namespace UnitTests.Dissemination; public partial class DisseminationProtocolTests { - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task MembershipInventoryOnlyRepairConvergesWithoutHeartbeat(bool delta) + [Fact] + public async Task MembershipInventoryOnlyRepairConvergesWithoutHeartbeat() { var cancellationToken = TestContext.Current.CancellationToken; var local = CreateSilo(40301); @@ -43,14 +41,8 @@ public async Task MembershipInventoryOnlyRepairConvergesWithoutHeartbeat(bool de var receiver = CreateMembershipNamespace((IMembershipManager)receiverManager, serializer); var source = CreateMembershipNamespace((IMembershipManager)sourceManager, serializer); var before = Assert.Single(receiver.Digests); - var update = delta - ? new MembershipTableSnapshotUpdate - { - Diff = new MembershipTableSnapshotDiff( - new MembershipVersion(1), incoming.Version, [incoming.Entries[live.SiloAddress]], [], includesAllEntries: true), - } - : new MembershipTableSnapshotUpdate { Snapshot = incoming }; - var value = new DisseminationValue(DisseminationKey.Default, delta ? 1 : 0, 2, serializer.SerializeToArray(update)); + var update = new MembershipTableSnapshotUpdate { Snapshot = incoming }; + var value = new DisseminationValue(DisseminationKey.Default, 0, 2, serializer.SerializeToArray(update)); Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(value, cancellationToken)); AssertMembershipState(expected, receiverManager.MembershipTableSnapshot); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs index bc92807fc14..01a83bfae6e 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs @@ -122,7 +122,7 @@ public async Task CompletedApplicationRemainsAppliedWhenItsLocalDeadlineRacesCom } [Fact] - public async Task ChangedOversizedBatchCannotAcknowledgeOrApplyAMissingChainPrefix() + public async Task ChangedOversizedBatchRejectsNonFullValuesBeforeOwnerApplication() { var local = CreateSilo(39661); var peer = CreateSilo(39662); @@ -159,11 +159,20 @@ public async Task ChangedOversizedBatchCannotAcknowledgeOrApplyAMissingChainPref var prefix = await protocol.ReceiveBroadcast(changedBatch, TestContext.Current.CancellationToken); Assert.Equal(1, Assert.Single(prefix.Acknowledgments[second.Name]).Version); - var suffix = await protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + var nonFull = await protocol.ReceiveBroadcast(new DisseminationBroadcastBatch { Sender = peer, Values = CreateValueGroups(second.Name, second.CreateItem(peer, "stream", 2, fromVersion: 1)), }, TestContext.Current.CancellationToken); + Assert.Equal(1, Assert.Single(nonFull.Acknowledgments[second.Name]).Version); + Assert.Equal(1, second.GetVersion("stream")); + Assert.Equal(1, second.ApplyCounts["stream"]); + + var suffix = await protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = peer, + Values = CreateValueGroups(second.Name, second.CreateItem(peer, "stream", 2)), + }, TestContext.Current.CancellationToken); Assert.Equal(2, Assert.Single(suffix.Acknowledgments[second.Name]).Version); Assert.Equal(1, first.GetVersion("other")); Assert.Equal(2, second.GetVersion("stream")); @@ -201,14 +210,14 @@ public async Task PublicationRejectsSingleValueBeyondGlobalBatchBudget() } [Fact] - public async Task PublicationPagesCompleteRepairWithinActualSendBudgets() + public async Task PublicationUsesCurrentFullRepairWithinActualSendBudgets() { var local = CreateSilo(39681); var peer = CreateSilo(39682); - var ns = new FakeNamespace(local) { ReturnRepairChain = true }; + var ns = new FakeNamespace(local); ns.PublishValue(ns.CreateValue("chain", 1)); - ns.PublishValue(ns.CreateValue("chain", 2, fromVersion: 1)); - ns.PublishValue(ns.CreateValue("chain", 3, fromVersion: 2)); + ns.PublishValue(ns.CreateValue("chain", 2)); + ns.PublishValue(ns.CreateValue("chain", 3)); var transport = new FakeTransport(local, peer); var protocol = CreateProtocol(transport, ns, options => { @@ -219,10 +228,13 @@ public async Task PublicationPagesCompleteRepairWithinActualSendBudgets() { Assert.True(await protocol.Publish(ns, "chain", 3, TestContext.Current.CancellationToken)); await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - Assert.Equal(new long[] { 1, 2, 3 }, + Assert.Equal(new long[] { 3 }, transport.BroadcastBatches.Select(batch => Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion)); Assert.All(transport.BroadcastBatches, batch => - Assert.Equal(8, Assert.Single(GetBroadcastValues(batch.Batch)).Value.Payload.Length)); + { + Assert.Equal(8, Assert.Single(GetBroadcastValues(batch.Batch)).Value.Payload.Length); + Assert.Equal(0, Assert.Single(GetBroadcastValues(batch.Batch)).Value.FromVersion); + }); } finally { diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs index 7ed06b4fd31..17397795cc8 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs @@ -109,23 +109,22 @@ public void MembershipRepairSendsFullStateWhenPeerVersionIsAhead() var ns = CreateMembershipNamespace(new FakeMembershipManager(snapshot), serializer); var repair = ns.CreateRepair(new( - DisseminationKey.Default, 100, null, 1, 1024 * 1024, 1024 * 1024)); + DisseminationKey.Default, 100, 1024 * 1024, 1024 * 1024)); Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); Assert.Equal(2, repair.Version); - var value = Assert.Single(repair.Values); + var value = repair.Value; Assert.Equal(0, value.FromVersion); Assert.Equal(2, value.ToVersion); var update = Assert.IsType( serializer.Deserialize(value.Payload)); - Assert.Null(update.Diff); var repaired = Assert.IsType(update.Snapshot); Assert.Equal(snapshot.Version, repaired.Version); Assert.Equal(local, Assert.Single(repaired.Entries).Key); } [Fact] - public async Task MembershipHistoryResetDiscardsPriorIncarnationAndDelayedPublication() + public async Task MembershipResetSupersedesPriorIncarnationAndDelayedPublication() { var members = CreateSilos(20); var old = CreateMembershipSnapshot(1, members.Select( @@ -146,19 +145,17 @@ public async Task MembershipHistoryResetDiscardsPriorIncarnationAndDelayedPublic manager.CurrentSnapshot = reset; Assert.Equal(2, Assert.Single(ns.Digests).Version); - Assert.False(await ns.PublishAsync(new FakeDisseminationService(), old, TestContext.Current.CancellationToken)); - var obsoleteBaseline = ns.CreateRepair(new( - DisseminationKey.Default, null, 1, 1, 1024 * 1024, 1024 * 1024)); - Assert.Equal(DisseminationRepairStatus.Unavailable, obsoleteBaseline.Status); + var publications = new FakeDisseminationService(); + Assert.True(await ns.PublishAsync(publications, old, TestContext.Current.CancellationToken)); + Assert.Equal(2, Assert.Single(publications.Values).ToVersion); var repair = ns.CreateRepair(new( - DisseminationKey.Default, 1, null, 1, 1024 * 1024, 1024 * 1024)); + DisseminationKey.Default, 1, 1024 * 1024, 1024 * 1024)); Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); - var value = Assert.Single(repair.Values); + var value = repair.Value; Assert.Equal(0, value.FromVersion); Assert.Equal(2, value.ToVersion); var payload = Assert.IsType( serializer.Deserialize(value.Payload)); - Assert.Null(payload.Diff); var repaired = Assert.IsType(payload.Snapshot); Assert.Equal(members.Length, repaired.Entries.Count); Assert.All(repaired.Entries.Values, entry => Assert.Equal("new-table", entry.HostName)); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs index dd913cd95df..20cf5500359 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs @@ -1,4 +1,3 @@ -using System.Collections.Immutable; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; @@ -18,11 +17,15 @@ namespace UnitTests.Dissemination; public partial class DisseminationProtocolTests { [Theory] - [InlineData(1)] - [InlineData(2)] - public async Task MembershipNamespaceFullSnapshotPreservesMixedLivenessInRealManager(long incomingVersion) + [InlineData(1, false)] + [InlineData(2, false)] + [InlineData(1, true)] + [InlineData(2, true)] + public async Task MembershipNamespaceFullSnapshotPreservesMixedLivenessInRealManager( + long incomingVersion, bool complementaryPeers) { var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(39500); var newer = CreateSilo(39501); var stale = CreateSilo(39502); var previous = CreateMembershipSnapshot( @@ -35,7 +38,7 @@ public async Task MembershipNamespaceFullSnapshotPreservesMixedLivenessInRealMan previous.Entries[stale].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(5))); using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); var serializer = services.GetRequiredService(); - using var manager = CreateMembershipReviewManager(CreateSilo(39500), out _); + using var manager = CreateMembershipReviewManager(local, out _); await ((IMembershipManager)manager).ProcessGossipSnapshot(previous, cancellationToken); var ns = CreateMembershipNamespace((IMembershipManager)manager, serializer); var value = new DisseminationValue( @@ -45,18 +48,47 @@ public async Task MembershipNamespaceFullSnapshotPreservesMixedLivenessInRealMan serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = incoming })); Assert.True(incoming.IsSuccessorTo(previous)); - Assert.Equal(DisseminationApplyResult.Applied, await ns.ApplyValueAsync(value, cancellationToken)); + if (complementaryPeers) + { + var complementary = CreateMembershipSnapshot(incomingVersion, + previous.Entries[newer].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(15)), + previous.Entries[stale].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(40))); + var otherValue = new DisseminationValue(DisseminationKey.Default, 0, incomingVersion, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = complementary })); + var transport = new FakeTransport(local, newer, stale); + transport.ExchangeAntiEntropyHandler = (peer, _, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = CreateValueGroups(ns.Name, CreateDisseminationValue(peer, peer.Equals(newer) ? value : otherValue)), + }); + var protocol = CreateProtocol(transport, [ns], options => options.Overlay.AntiEntropyPeerCount = 2); + try + { + await protocol.RunAntiEntropyRound(cancellationToken); + Assert.Equal(2, transport.AntiEntropyRequests.Count); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + else + { + Assert.Equal(DisseminationApplyResult.Applied, await ns.ApplyValueAsync(value, cancellationToken)); + } var expected = CreateMembershipSnapshot( incomingVersion, incoming.Entries[newer], - previous.Entries[stale]); + complementaryPeers + ? previous.Entries[stale].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(40)) + : previous.Entries[stale]); AssertMembershipState(expected, manager.MembershipTableSnapshot); Assert.Equal(DateTime.UnixEpoch.AddSeconds(5), incoming.Entries[stale].IAmAliveTime); Assert.Equal(DateTime.UnixEpoch.AddSeconds(10), previous.Entries[newer].IAmAliveTime); // Repairs must contain the manager's merged state, not the unmerged incoming snapshot. - var repair = Assert.Single(ns.CreateRepair(MembershipReviewRepairRequest(incomingVersion)).Values); + var repair = GetMembershipRepair(ns, incomingVersion); Assert.Equal(0, repair.FromVersion); var repaired = Assert.IsType( serializer.Deserialize(repair.Payload)); @@ -68,7 +100,7 @@ public async Task MembershipNamespaceFullSnapshotPreservesMixedLivenessInRealMan [Theory] [InlineData(false)] [InlineData(true)] - public async Task MembershipNamespaceCompleteDiffPrunesDivergentInventoryAndRepairsDownstream(bool receiverAlreadyAtTargetVersion) + public async Task MembershipNamespaceFullSnapshotPrunesDivergentInventoryAndRepairsDownstream(bool receiverAlreadyAtTargetVersion) { var cancellationToken = TestContext.Current.CancellationToken; var retired = CreateSilo(39511); @@ -124,14 +156,13 @@ public async Task MembershipNamespaceCompleteDiffPrunesDivergentInventoryAndRepa cancellationToken); } - var value = Assert.Single(source.CreateRepair(MembershipReviewRepairRequest(sourceBaseline.Version)).Values); - Assert.Equal((sourceBaseline.Version, target.Version.Value), (value.FromVersion, value.ToVersion)); + var value = GetMembershipRepair(source, sourceBaseline.Version); + Assert.Equal((0L, target.Version.Value), (value.FromVersion, value.ToVersion)); var update = Assert.IsType( serializer.Deserialize(value.Payload)); - var diff = Assert.IsType(update.Diff); - Assert.True(diff.IncludesAllEntries); - Assert.Empty(diff.RemovedSilos); - Assert.Equal(successor, Assert.Single(diff.UpdatedEntries).SiloAddress); + var snapshot = Assert.IsType(update.Snapshot); + AssertMembershipState(target, snapshot); + Assert.Equal(successor, Assert.Single(snapshot.Entries).Key); Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(value, cancellationToken)); AssertMembershipState(target, receiverManager.MembershipTableSnapshot); @@ -152,15 +183,15 @@ public async Task MembershipNamespaceCompleteDiffPrunesDivergentInventoryAndRepa using var downstreamManager = CreateMembershipReviewManager(CreateSilo(39513), out _); await ((IMembershipManager)downstreamManager).ProcessGossipSnapshot(beforeCleanup, cancellationToken); var downstream = CreateMembershipNamespace((IMembershipManager)downstreamManager, serializer); - var forwarded = Assert.Single(receiver.CreateRepair(MembershipReviewRepairRequest(sourceBaseline.Version)).Values); - Assert.True(forwarded.FromVersion == 0 || forwarded.FromVersion == sourceBaseline.Version); + var forwarded = GetMembershipRepair(receiver, sourceBaseline.Version); + Assert.Equal(0, forwarded.FromVersion); Assert.Equal(target.Version.Value, forwarded.ToVersion); Assert.Equal(DisseminationApplyResult.Applied, await downstream.ApplyValueAsync(forwarded, cancellationToken)); AssertMembershipState(target, downstreamManager.MembershipTableSnapshot); } [Fact] - public async Task MembershipNamespaceCompleteDiffPreservesRealManagerLocalDeathEntry() + public async Task MembershipNamespaceFullSnapshotPreservesRealManagerLocalDeathEntry() { var cancellationToken = TestContext.Current.CancellationToken; var local = CreateSilo(39520); @@ -177,14 +208,9 @@ public async Task MembershipNamespaceCompleteDiffPreservesRealManagerLocalDeathE var ns = CreateMembershipNamespace((IMembershipManager)manager, serializer); var update = new MembershipTableSnapshotUpdate { - Diff = new MembershipTableSnapshotDiff( - previous.Version, - new MembershipVersion(2), - [previous.Entries[peer]], - [], - includesAllEntries: true), + Snapshot = CreateMembershipSnapshot(2, previous.Entries[peer]), }; - var value = new DisseminationValue(DisseminationKey.Default, 1, 2, serializer.SerializeToArray(update)); + var value = new DisseminationValue(DisseminationKey.Default, 0, 2, serializer.SerializeToArray(update)); Assert.Equal(DisseminationApplyResult.Applied, await ns.ApplyValueAsync(value, cancellationToken)); @@ -192,7 +218,7 @@ public async Task MembershipNamespaceCompleteDiffPreservesRealManagerLocalDeathE AssertMembershipState(expected, manager.MembershipTableSnapshot); Assert.Equal(SiloStatus.Dead, manager.CurrentStatus); fatalErrorHandler.Received(1).OnFatalException(manager, Arg.Any(), null); - var repair = Assert.Single(ns.CreateRepair(MembershipReviewRepairRequest(2)).Values); + var repair = GetMembershipRepair(ns, 2); Assert.Equal(0, repair.FromVersion); var repaired = Assert.IsType( serializer.Deserialize(repair.Payload)); @@ -200,67 +226,7 @@ public async Task MembershipNamespaceCompleteDiffPreservesRealManagerLocalDeathE } [Fact] - public async Task MembershipNamespaceDiffWireCompatibilityPreservesLegacyPartialUpdates() - { - var cancellationToken = TestContext.Current.CancellationToken; - var retained = CreateSilo(39531); - var updated = CreateSilo(39532); - var removed = CreateSilo(39533); - var previous = CreateMembershipSnapshot( - 1, - CreateMembershipEntry(retained, SiloStatus.Active, DateTime.UnixEpoch), - CreateMembershipEntry(updated, SiloStatus.Active, DateTime.UnixEpoch), - CreateMembershipEntry(removed, SiloStatus.Dead, DateTime.UnixEpoch)); - using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); - var serializer = services.GetRequiredService(); - var legacy = new MembershipReviewLegacyDiff - { - BaseVersion = previous.Version, - Version = new MembershipVersion(2), - UpdatedEntries = [previous.Entries[updated].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))], - RemovedSilos = [removed], - }; - - // Root codecs omit the type name when the expected and actual types agree, allowing - // these two schemas to exercise both missing-field and unknown-field wire compatibility. - var decoded = Assert.IsType( - serializer.Deserialize(serializer.SerializeToArray(legacy))); - Assert.False(decoded.IncludesAllEntries); - Assert.Equal(legacy.BaseVersion, decoded.BaseVersion); - Assert.Equal(legacy.Version, decoded.Version); - Assert.Equal(legacy.RemovedSilos, decoded.RemovedSilos); - Assert.Equal(updated, Assert.Single(decoded.UpdatedEntries).SiloAddress); - using var manager = CreateMembershipReviewManager(CreateSilo(39530), out _); - await ((IMembershipManager)manager).ProcessGossipSnapshot(previous, cancellationToken); - var ns = CreateMembershipNamespace((IMembershipManager)manager, serializer); - var value = new DisseminationValue( - DisseminationKey.Default, - 1, - 2, - serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Diff = decoded })); - - Assert.Equal(DisseminationApplyResult.Applied, await ns.ApplyValueAsync(value, cancellationToken)); - var expected = CreateMembershipSnapshot(2, previous.Entries[retained], legacy.UpdatedEntries[0]); - AssertMembershipState(expected, manager.MembershipTableSnapshot); - - var complete = new MembershipTableSnapshotDiff( - legacy.BaseVersion, - legacy.Version, - [.. expected.Entries.Values], - legacy.RemovedSilos, - includesAllEntries: true); - var oldPeer = Assert.IsType( - serializer.Deserialize(serializer.SerializeToArray(complete))); - Assert.Equal(complete.BaseVersion, oldPeer.BaseVersion); - Assert.Equal(complete.Version, oldPeer.Version); - Assert.Equal(complete.RemovedSilos, oldPeer.RemovedSilos); - AssertMembershipState( - expected, - CreateMembershipSnapshot(oldPeer.Version.Value, [.. oldPeer.UpdatedEntries])); - } - - [Fact] - public async Task MembershipNamespaceLegacyInventoryDivergenceConvergesWithoutHeartbeatWithFullRepair() + public async Task MembershipNamespaceInventoryDivergenceConvergesWithoutHeartbeatWithFullRepair() { var cancellationToken = TestContext.Current.CancellationToken; var retired = CreateSilo(39541); @@ -283,19 +249,8 @@ public async Task MembershipNamespaceLegacyInventoryDivergenceConvergesWithoutHe var receiver = CreateMembershipNamespace((IMembershipManager)receiverManager, serializer); _ = Assert.Single(source.Digests); await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(target, cancellationToken); - var delta = Assert.Single(source.CreateRepair(MembershipReviewRepairRequest(1)).Values); - Assert.Equal((1L, 2L), (delta.FromVersion, delta.ToVersion)); - var update = Assert.IsType( - serializer.Deserialize(delta.Payload)); - var legacy = Assert.IsType( - serializer.Deserialize( - serializer.SerializeToArray(Assert.IsType(update.Diff)))); - var legacyUpdate = new MembershipTableSnapshotUpdate - { - Diff = new MembershipTableSnapshotDiff(legacy.BaseVersion, legacy.Version, legacy.UpdatedEntries, legacy.RemovedSilos), - }; - var legacyValue = new DisseminationValue(DisseminationKey.Default, 1, 2, serializer.SerializeToArray(legacyUpdate)); - Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(legacyValue, cancellationToken)); + await ((IMembershipManager)receiverManager).ProcessGossipSnapshot( + CreateMembershipSnapshot(2, receiverBaseline.Entries[retired], target.Entries[successor]), cancellationToken); Assert.Equal(SiloStatus.Dead, receiverManager.MembershipTableSnapshot.GetSiloStatus(retired)); Assert.True(receiverManager.MembershipTableSnapshot.Entries.ContainsKey(retired)); Assert.Equal(target.Version, receiverManager.MembershipTableSnapshot.Version); @@ -379,22 +334,10 @@ private static MembershipDisseminationNamespace CreateMembershipNamespace(IMembe return new(manager, new TestOptionsMonitor(options), serializer); } - private static DisseminationRepairRequest MembershipReviewRepairRequest(long? fromVersion) => - new(DisseminationKey.Default, fromVersion, toVersion: null, maxItemCount: 1, maxBatchBytes: 1024 * 1024, maxPayloadBytes: 1024 * 1024); - - [GenerateSerializer] - internal sealed class MembershipReviewLegacyDiff + private static DisseminationValue GetMembershipRepair(MembershipDisseminationNamespace ns, long? fromVersion) { - [Id(0)] - public MembershipVersion BaseVersion { get; init; } - - [Id(1)] - public MembershipVersion Version { get; init; } - - [Id(2)] - public ImmutableArray UpdatedEntries { get; init; } - - [Id(3)] - public ImmutableArray RemovedSilos { get; init; } + var repair = ns.CreateRepair(new(DisseminationKey.Default, fromVersion, 1024 * 1024, 1024 * 1024)); + Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); + return repair.Value; } } diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs index 6e00d6b2406..ea0d6618317 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs @@ -1,4 +1,5 @@ using System.Diagnostics.Metrics; +using Microsoft.Extensions.Time.Testing; using Orleans.Runtime.Dissemination; using Xunit; @@ -22,9 +23,8 @@ public async Task MetricFailuresPreserveProtocolResults(string operation) var healthy = CreateSilo(41023); var transport = operation == "repair-failure" ? new FakeTransport(local, peer, healthy) : new FakeTransport(local, peer); var ns = new FakeNamespace(local, new DisseminationNamespace("metric-boundary-" + operation)); - ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); var logger = new Phase6ProtocolLogger(); - var protocol = CreatePhase6Protocol(transport, ns, logger, options => options.Overlay.AntiEntropyPeerCount = 2); + var protocol = CreateProtocol(transport, [ns], options => options.Overlay.AntiEntropyPeerCount = 2, new FakeTimeProvider(), logger); var metric = operation switch { "broadcast-receive" => "orleans-dissemination-broadcast-received", diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PerformanceReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PerformanceReview.cs index 178f29b4345..e7d7ebbaff3 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PerformanceReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PerformanceReview.cs @@ -51,9 +51,9 @@ public async Task LoadRepairReusesOnlySuccessfullyAppliedPayloads() DisseminationValue Repair() { - var repair = ns.CreateRepair(new(value.Key, null, null, 1, 1024, 1024)); + var repair = ns.CreateRepair(new(value.Key, null, 1024, 1024)); Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); - return Assert.Single(repair.Values); + return repair.Value; } } diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs index 9244e2fb189..060578ff730 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs @@ -30,7 +30,7 @@ public async Task ProtocolReviewPayloadObserverFailureDoesNotBlockLaterRepair() ns.Inner.SetValue("healthy", 2); ns.Options.MaxPayloadBytes = 8; ns.RepairHandler = request => request.Key == new DisseminationKey("oversize") - ? DisseminationRepairResult.Produced(1, [new(request.Key, 0, 1, new byte[9])]) + ? DisseminationRepairResult.Produced(new(request.Key, 0, 1, new byte[9])) : ns.Inner.CreateRepair(request); var logger = new Phase6ProtocolLogger(); var protocol = CreatePhase6Protocol(new FakeTransport(local, peer), ns, logger); @@ -77,7 +77,7 @@ public async Task ProtocolReviewValueMetricFailuresDoNotBlockRepairOrApplication ns.Inner.SetValue("healthy", 2); ns.Options.MaxPayloadBytes = 8; ns.RepairHandler = request => request.Key == new DisseminationKey("oversize") - ? DisseminationRepairResult.Produced(1, [new(request.Key, 0, 1, new byte[9])]) + ? DisseminationRepairResult.Produced(new(request.Key, 0, 1, new byte[9])) : ns.Inner.CreateRepair(request); var logger = new Phase6ProtocolLogger(); var protocol = CreatePhase6Protocol(new FakeTransport(local, peer), ns, logger); @@ -474,7 +474,7 @@ public async Task ProtocolReviewBroadcastCapsAllNamespacesAndAcknowledgesOnlyPre Sender = peer, Values = new() { - [first.Name] = [first.Inner.CreateItem(peer, "chain", 1), first.Inner.CreateItem(peer, "chain", 2, 1)], + [first.Name] = [first.Inner.CreateItem(peer, "chain", 1), first.Inner.CreateItem(peer, "chain", 2)], [second.Name] = [second.Inner.CreateItem(peer, "later", 3)], }, }; @@ -505,7 +505,7 @@ public async Task ProtocolReviewBroadcastCapsAllNamespacesAndAcknowledgesOnlyPre [Theory] [InlineData(false)] [InlineData(true)] - public async Task ProtocolReviewReceiveCursorPassesRejectedHeadAndPreservesChain(bool antiEntropy) + public async Task ProtocolReviewReceiveCursorPassesRejectedHeadAndAppliesFullUpdates(bool antiEntropy) { var local = CreateSilo(39551); var peer = CreateSilo(39552); @@ -515,7 +515,7 @@ public async Task ProtocolReviewReceiveCursorPassesRejectedHeadAndPreservesChain var values = CreateValueGroups( CreateDisseminationValue(peer, new DisseminationValue("bad", 0, 1, new byte[9])), ns.Inner.CreateItem(peer, "chain", 1), - ns.Inner.CreateItem(peer, "chain", 2, 1)); + ns.Inner.CreateItem(peer, "chain", 2)); var transport = new FakeTransport(local, peer); transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult( new DisseminationAntiEntropyResponse { Sender = peer, Values = values }); @@ -552,7 +552,7 @@ public async Task ProtocolReviewReceiveCursorPassesRejectedHeadAndPreservesChain Assert.Equal(0, ns.GetVersion("bad")); Assert.Equal(new long[] { 1, 2 }, ns.Attempts.Select(static value => value.ToVersion)); - Assert.Equal(new long[] { 0, 1 }, ns.Attempts.Select(static value => value.FromVersion)); + Assert.Equal(new long[] { 0, 0 }, ns.Attempts.Select(static value => value.FromVersion)); } finally { @@ -564,7 +564,7 @@ public async Task ProtocolReviewReceiveCursorPassesRejectedHeadAndPreservesChain [InlineData(2, 1024)] [InlineData(10, 16)] [InlineData(2, 16)] - public async Task ProtocolReviewAntiEntropyCapsEachResponseBeforeGrouping(int maxItems, int maxBytes) + public async Task ProtocolReviewAntiEntropyCapsEachResponseBeforeApplication(int maxItems, int maxBytes) { var local = CreateSilo(39561); var badPeer = CreateSilo(39562); @@ -582,7 +582,7 @@ public async Task ProtocolReviewAntiEntropyCapsEachResponseBeforeGrouping(int ma Sender = peer, Values = new() { - [ns.Name] = [first, ns.Inner.CreateItem(peer, "chain", 2, 1)], + [ns.Name] = [first, ns.Inner.CreateItem(peer, "chain", 2)], [other.Name] = [other.Inner.CreateItem(peer, "omitted", 3)], }, }); @@ -598,9 +598,9 @@ public async Task ProtocolReviewAntiEntropyCapsEachResponseBeforeGrouping(int ma await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); Assert.Equal(2, transport.AntiEntropyRequests.Count); Assert.Equal(2, ns.GetVersion("chain")); - Assert.Equal(new long[] { 1, 1, 2 }, ns.Attempts.Select(static value => value.ToVersion)); + Assert.Equal(new long[] { 1, 2, 2 }, ns.Attempts.Select(static value => value.ToVersion)); Assert.Equal(24, ns.Attempts.Sum(static value => value.Payload.Length)); - Assert.Equal(2, ns.Inner.ApplyCounts["chain"]); + Assert.Equal(1, ns.Inner.ApplyCounts["chain"]); Assert.Empty(other.Attempts); Assert.Equal(0, other.GetVersion("omitted")); @@ -755,29 +755,33 @@ public async Task ProtocolReviewInvalidResponseBudgetsHaveNoReceiveSideEffects( var local = CreateSilo(39601); var peer = CreateSilo(39602); var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); ns.SetValue("pending", 1); var transport = new FakeTransport(local, peer); var protocol = CreateProtocol(transport, ns, timeProvider: new FakeTimeProvider()); try { - Assert.True(await protocol.Publish(ns, "pending", 1, TestContext.Current.CancellationToken)); - var repairRequests = ns.RepairRequestCount; - var exception = Assert.Throws(() => + var publication = BeforeBroadcastPumpsRun(() => { - _ = protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + var pending = protocol.Publish(ns, "pending", 1, TestContext.Current.CancellationToken); + var repairRequests = ns.RepairRequestCount; + var exception = Assert.Throws(() => { - Sender = peer, - SupportedNamespaces = [ns.Name], - Digests = CreateAntiEntropyRequestDigest(ns.Name, ("pending", 1)), - MaxResponseItems = maxResponseItems, - MaxResponseBytes = maxResponseBytes, - }, TestContext.Current.CancellationToken); + _ = protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = CreateAntiEntropyRequestDigest(ns.Name, ("pending", 1)), + MaxResponseItems = maxResponseItems, + MaxResponseBytes = maxResponseBytes, + }, TestContext.Current.CancellationToken); + }); + Assert.Equal(parameterName, exception.ParamName); + Assert.Equal(maxResponseItems ?? maxResponseBytes, Assert.IsType(exception.ActualValue)); + Assert.Equal(repairRequests, ns.RepairRequestCount); + Assert.Equal(peer, Assert.Single(protocol.GetUnconfirmedPeers(ns))); + return pending; }); - Assert.Equal(parameterName, exception.ParamName); - Assert.Equal(maxResponseItems ?? maxResponseBytes, Assert.IsType(exception.ActualValue)); - Assert.Equal(repairRequests, ns.RepairRequestCount); - Assert.Equal(peer, Assert.Single(protocol.GetUnconfirmedPeers(ns))); + Assert.True(await publication); await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken) .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); @@ -846,17 +850,14 @@ public async Task ProtocolReviewResponseBudgetsRoundTripAndBoundEveryRepairProbe Assert.Equal(keys.Take(count), GetAntiEntropyResponseValues(response).Select(static item => item.Value.Key)); Assert.Equal(count * sizeof(long), GetAntiEntropyResponseValues(response).Sum(static item => item.Value.Payload.Length)); Assert.True(response.Truncated); - Assert.Equal(count + 2, requests.Count); - for (var index = 0; index <= count; index++) + var probes = count + (count < maxItems ? 1 : 0); + Assert.Equal(probes, requests.Count); + for (var index = 0; index < probes; index++) { Assert.Equal(keys[index], requests[index].Key); - Assert.Equal(maxItems - index, requests[index].MaxItemCount); - Assert.Equal(maxBytes - index * sizeof(long), requests[index].MaxBatchBytes); + Assert.Equal(maxBytes, requests[index].MaxBatchBytes); + Assert.Equal(ns.Options.MaxPayloadBytes, requests[index].MaxPayloadBytes); } - - Assert.Equal(keys[count], requests[^1].Key); - Assert.Equal(maxItems, requests[^1].MaxItemCount); - Assert.Equal(maxBytes, requests[^1].MaxBatchBytes); } finally { @@ -867,16 +868,17 @@ public async Task ProtocolReviewResponseBudgetsRoundTripAndBoundEveryRepairProbe [Theory] [InlineData(1)] [InlineData(10)] - public async Task ProtocolReviewLargerBroadcastSenderContinuesFromAcknowledgedPrefix(int senderLimit) + public async Task LargerBroadcastSenderRetriesOnlyUnacknowledgedKeys(int senderLimit) { var sourceAddress = CreateSilo(39581); var receiverAddress = CreateSilo(39582); var clock = new FakeTimeProvider(); - var source = new FakeNamespace(sourceAddress) { ReturnRepairChain = true }; - source.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); - source.PublishValue(source.CreateValue("chain", 1)); - source.PublishValue(source.CreateValue("chain", 2, 1)); - source.PublishValue(source.CreateValue("chain", 3, 2)); + var source = new FakeNamespace(sourceAddress); + DisseminationKey[] keys = ["first", "second", "third"]; + for (var index = 0; index < keys.Length; index++) + { + source.SetValue(keys[index], index + 1); + } var receiver = new ProtocolReviewNamespace(receiverAddress); var receivingProtocol = CreateProtocol(new FakeTransport(receiverAddress, sourceAddress), [receiver], options => { @@ -893,34 +895,54 @@ public async Task ProtocolReviewLargerBroadcastSenderContinuesFromAcknowledgedPr var acknowledgment = await receivingProtocol.ReceiveBroadcast(batch, token); compactResponses.Add(acknowledgment.AllVersionsAcknowledged); acknowledgments.Add(acknowledgment.AllVersionsAcknowledged - ? receiver.GetVersion("chain") + ? GetBroadcastValues(batch).Max(static item => item.Value.ToVersion) : Assert.Single(acknowledgment.Acknowledgments[source.Name]).Version); return acknowledgment; }; - var sendingProtocol = CreateProtocol(transport, source, options => + var sendingQueue = CreateBroadcastQueue(transport, [source], options => { options.MaxBatchItems = senderLimit; options.MaxBatchBytes = 1024; }, clock); + using var schedules = new BroadcastScheduleObserver(); + var retry = senderLimit > 1 ? WaitForRetry() : null; try { - Assert.True(await sendingProtocol.Publish(source, "chain", 3, TestContext.Current.CancellationToken)); - await sendingProtocol.FlushPendingBroadcast(TestContext.Current.CancellationToken) + Assert.True(sendingQueue.NotifyBatch(receiverAddress, source, + [.. keys.Select((key, index) => new DisseminationBroadcastQueue.KeyNotification(key, index + 1, true))])); + for (var round = 0; round < 2 && retry is not null; round++) + { + var scheduled = await retry; + Assert.Equal(TimeSpan.FromMilliseconds(100), scheduled.DueTime); + Assert.Equal(1, scheduled.Attempt); + Assert.Equal(Enumerable.Range(1, round + 1).Select(static version => (long)version), acknowledgments); + retry = round == 0 ? WaitForRetry() : null; + clock.Advance(scheduled.DueTime); + } + + await sendingQueue.FlushPendingBroadcast(TestContext.Current.CancellationToken) .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(new long[] { 1, 2, 3 }, acknowledgments); int[] expectedCounts = senderLimit == 1 ? [1, 1, 1] : [3, 2, 1]; bool[] expectedCompact = senderLimit == 1 ? [true, true, true] : [false, false, true]; Assert.Equal(expectedCounts, sentItemCounts); Assert.Equal(expectedCompact, compactResponses); - Assert.Equal(new long[] { 0, 1, 2 }, receiver.Attempts.Select(static value => value.FromVersion)); + Assert.Equal(new long[] { 0, 0, 0 }, receiver.Attempts.Select(static value => value.FromVersion)); Assert.Equal(new long[] { 1, 2, 3 }, receiver.Attempts.Select(static value => value.ToVersion)); - Assert.Equal(3, receiver.GetVersion("chain")); + Assert.Equal(keys, receiver.Attempts.Select(static value => value.Key)); + Assert.Equal(new long[] { 1, 2, 3 }, keys.Select(receiver.GetVersion)); } finally { - await sendingProtocol.StopAsync(TestContext.Current.CancellationToken); + source.Options.Enabled = false; + await sendingQueue.StopAsync(TestContext.Current.CancellationToken); await receivingProtocol.StopAsync(TestContext.Current.CancellationToken); } + + Task WaitForRetry() => schedules.WaitAsync( + scheduled => scheduled.LocalSilo.Equals(sourceAddress) && scheduled.Peer.Equals(receiverAddress) + && scheduled.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); } private sealed class ProtocolReviewNamespace(SiloAddress local, DisseminationNamespace? name = null) : IDisseminationNamespace diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs index 2862cc31ea3..133e505f201 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs @@ -45,7 +45,6 @@ private static async Task VerifyPumpFailureMetricIsolation(int failedTimerChange var logger = new PumpDiagnosticsLogger(); var ns = new FakeNamespace(local); ns.Options.MaxPendingItemCount = 1; - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); ns.SetValue("original", 7); ns.SetValue("next", 9); var transport = new FakeTransport(local, peer); @@ -154,11 +153,10 @@ private static async Task VerifyPumpFailureMetricIsolation(int failedTimerChange && value.Reason == DisseminationBroadcastScheduleReason.Immediate, TimeSpan.FromSeconds(5), cancellationToken); Assert.True(queue.Notify(peer, ns, "original")); - await WaitForPhase(initialSchedule, "initial timer armed"); + await WaitForPhase(initialSchedule, "initial immediate send scheduled"); var initial = await initialSchedule; Assert.Equal(TimeSpan.Zero, initial.DueTime); Assert.Equal(0, initial.Attempt); - clock.Advance(initial.DueTime); await WaitForPhase(started[0].Task, "original send in flight"); Assert.False(queue.Notify(peer, ns, "next")); @@ -219,7 +217,7 @@ private static async Task VerifyPumpFailureMetricIsolation(int failedTimerChange await WaitForPhase(retrySchedule!, "recovered retry timer armed"); var retry = await retrySchedule!; Assert.Equal(failedTimerChanges + 1, retry.Attempt); - Assert.Equal(TimeSpan.FromSeconds(failedTimerChanges + 1), retry.DueTime); + Assert.Equal(TimeSpan.FromMilliseconds(100 * (failedTimerChanges + 1)), retry.DueTime); Assert.Equal(initial.Epoch, retry.Epoch); Assert.Equal(1, Volatile.Read(ref sendCount)); @@ -242,9 +240,9 @@ private static async Task VerifyPumpFailureMetricIsolation(int failedTimerChange Assert.True(queue.Notify(peer, ns, "next")); await WaitForPhase(nextSchedule, "freed capacity schedules the next key"); var next = await nextSchedule; + Assert.Equal(TimeSpan.Zero, next.DueTime); Assert.Equal(0, next.Attempt); Assert.Equal(retry.Epoch + 1, next.Epoch); - clock.Advance(next.DueTime); await WaitForPhase(started[2].Task, "next key admitted to transport"); var nextFlush = queue.FlushPendingBroadcast(cancellationToken); Assert.False(nextFlush.IsCompleted); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs index edff81df901..5fcf23fc33a 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs @@ -32,22 +32,26 @@ public async Task SingleKeyNotificationAvoidsTemporaryAllocations() var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); try { - Assert.True(queue.Notify(peer, ns, "value")); - for (var iteration = 0; iteration < 128; iteration++) + var allocated = BeforeBroadcastPumpsRun(() => { - Assert.True(queue.Notify(peer, ns, "value", force: false)); - } + Assert.True(queue.Notify(peer, ns, "value")); + for (var iteration = 0; iteration < 128; iteration++) + { + Assert.True(queue.Notify(peer, ns, "value", force: false)); + } - const int iterations = 1024; - var accepted = true; - var before = GC.GetAllocatedBytesForCurrentThread(); - for (var iteration = 0; iteration < iterations; iteration++) - { - accepted &= queue.Notify(peer, ns, "value", force: false); - } - var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + const int iterations = 1024; + var accepted = true; + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var iteration = 0; iteration < iterations; iteration++) + { + accepted &= queue.Notify(peer, ns, "value", force: false); + } - Assert.True(accepted); + var result = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.True(accepted); + return result; + }); Assert.True(allocated < 512, $"Repeated single-key notifications allocated {allocated} bytes."); await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); Assert.Equal(1, Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)).Value.ToVersion); @@ -90,7 +94,7 @@ public async Task BatchNotificationPreservesLaterUpdatesWhenAdmissionRejectsAKey await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); ns.SetValue("value", 2); Assert.False(queue.NotifyBatch(peer, ns, - [new("rejected", 1, true), new("value", 2, true)], immediate: true)); + [new("rejected", 1, true), new("value", 2, true)])); releaseFirst.TrySetResult(); await firstFlush.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); await queue.FlushPendingBroadcast(cancellationToken); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index 582459bee56..076cc52486d 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -38,10 +38,7 @@ public partial class DisseminationProtocolTests [Fact] public async Task PublishPropagatesCancellationBeforeQueueing() { - var local = CreateSilo(39101); - var peer = CreateSilo(39102); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(39101, 39102); ns.SetValue(FakeNamespace.DefaultKey, version: 1); var protocol = CreateProtocol(transport, ns); using var cancellation = new CancellationTokenSource(); @@ -84,7 +81,7 @@ public void PushBroadcastUsesRequestResponseSemantics() } [Fact] - public async Task PublishSendsBroadcastToDeterministicTreeChildren() + public async Task PublishQueuesBroadcastToDeterministicTreeChildrenWithoutCapabilityProbing() { var local = CreateSilo(11111); var peers = Enumerable.Range(11112, 6).Select(CreateSilo).ToArray(); @@ -104,29 +101,7 @@ public async Task PublishSendsBroadcastToDeterministicTreeChildren() var expectedChildren = GetOriginatorTreeTargets(local, peers, fanout: 2); Assert.Equal(expectedChildren.OrderBy(static peer => peer), transport.BroadcastBatches.Select(batch => batch.Peer).OrderBy(static peer => peer)); Assert.All(transport.BroadcastBatches, batch => Assert.Equal(item, GetBroadcastValues(batch.Batch).Single().Value)); - } - - [Fact] - public async Task PublishQueuesTreeBroadcastWithoutCapabilityProbing() - { - var local = CreateSilo(11111); - var peers = Enumerable.Range(11112, 6).Select(CreateSilo).ToArray(); - var transport = new FakeTransport(local, peers); - var ns = new FakeNamespace(local); - var protocol = CreateProtocol(transport, ns, options => - { - options.MaxConcurrentSends = 1; - options.Overlay.FanOutFactor = static _ => 2; - }); - var item = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1); - - var result = await PublishValue(protocol, ns, item, TestContext.Current.CancellationToken); - await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - - Assert.True(result); - Assert.Equal( - GetOriginatorTreeTargets(local, peers, fanout: 2).OrderBy(static peer => peer), - transport.BroadcastBatches.Select(batch => batch.Peer).OrderBy(static peer => peer)); + Assert.Empty(transport.AntiEntropyRequests); } [Fact] @@ -210,10 +185,7 @@ public async Task BroadcastSendsHonorMaxConcurrentSends() [Fact] public async Task PublishRejectsInvalidValuesBeforeQueueing() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, _, transport, ns) = CreatePeerFixture(); var protocol = CreateProtocol(transport, ns); ns.SetValue("obsolete", version: 10); @@ -319,8 +291,10 @@ public async Task NewPublicationWakesFailedPeerWithoutWaitingForBackoff() Assert.Equal(2, GetBroadcastValues(batch.Batch).Single().Value.ToVersion); } - [Fact] - public async Task SendFailureRetriesAutomaticallyWithBoundedBackoff() + [Theory] + [InlineData(4000, 100)] + [InlineData(50, 50)] + public async Task SendFailureRetriesAutomaticallyWithBoundedBackoff(int antiEntropyMilliseconds, int retryMilliseconds) { var local = CreateSilo(11111); var peer = CreateSilo(11112); @@ -343,26 +317,28 @@ public async Task SendFailureRetriesAutomaticallyWithBoundedBackoff() }; var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); var protocol = CreateProtocol( transport, ns, - options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(4), + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromMilliseconds(antiEntropyMilliseconds), timeProvider); + using var schedule = new BroadcastScheduleObserver(); + var scheduledRetry = schedule.WaitAsync( + e => e.LocalSilo.Equals(local) && e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.True(await PublishValue( protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken)); - using var schedule = new BroadcastScheduleObserver(); - timeProvider.Advance(TimeSpan.FromSeconds(1)); await firstAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await schedule.WaitAsync( - e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry, - TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var retry = await scheduledRetry; + Assert.Equal(TimeSpan.FromMilliseconds(retryMilliseconds), retry.DueTime); - timeProvider.Advance(TimeSpan.FromSeconds(1)); + timeProvider.Advance(retry.DueTime - TimeSpan.FromMilliseconds(1)); + Assert.False(secondAttempt.Task.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); await secondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(2, sendCount); @@ -399,47 +375,46 @@ public async Task NewNotificationResetsRetryBackoff() }; var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); var protocol = CreateProtocol( transport, ns, options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(4), timeProvider); + using var schedule = new BroadcastScheduleObserver(); + var firstRetry = schedule.WaitAsync( + e => e.LocalSilo.Equals(local) && e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry && e.Attempt == 1, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.True(await PublishValue( protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken)); - using var schedule = new BroadcastScheduleObserver(); - timeProvider.Advance(TimeSpan.FromSeconds(1)); await firstAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await schedule.WaitAsync( - e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry && e.Attempt == 1, + Assert.Equal(TimeSpan.FromMilliseconds(100), (await firstRetry).DueTime); + var secondRetry = schedule.WaitAsync( + e => e.LocalSilo.Equals(local) && e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry && e.Attempt == 2, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - timeProvider.Advance(TimeSpan.FromSeconds(1)); + timeProvider.Advance(TimeSpan.FromMilliseconds(100)); await secondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await schedule.WaitAsync( - e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry && e.Attempt == 2, - TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(TimeSpan.FromMilliseconds(200), (await secondRetry).DueTime); + var beforeNotification = timeProvider.GetTimestamp(); Assert.True(await PublishValue( protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), TestContext.Current.CancellationToken)); - timeProvider.Advance(TimeSpan.FromMilliseconds(999)); - Assert.False(thirdAttempt.Task.IsCompleted); - timeProvider.Advance(TimeSpan.FromMilliseconds(1)); await thirdAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(TimeSpan.Zero, timeProvider.GetElapsedTime(beforeNotification)); Assert.Equal(3, sendCount); Assert.Equal(2, Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)).Value.ToVersion); await protocol.StopAsync(TestContext.Current.CancellationToken); } [Fact] - public async Task HighPriorityNamespaceFlushesWithoutWaitingForCoalescingWindow() + public async Task NamespaceNotificationFlushesImmediately() { var local = CreateSilo(11111); var peer = CreateSilo(11112); @@ -454,11 +429,9 @@ public async Task HighPriorityNamespaceFlushesWithoutWaitingForCoalescingWindow( }; var ns = new FakeNamespace(local); - ns.Options.Priority = DisseminationPriority.High; - // A long window would delay a normal namespace; a high-priority namespace must ignore it. - ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); using var schedule = new BroadcastScheduleObserver(); + var start = timeProvider.GetTimestamp(); Assert.True(await PublishValue( protocol, @@ -467,220 +440,16 @@ public async Task HighPriorityNamespaceFlushesWithoutWaitingForCoalescingWindow( TestContext.Current.CancellationToken)); var scheduled = await schedule.WaitAsync( - e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Priority, + e => e.LocalSilo.Equals(local) && e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Immediate, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(TimeSpan.Zero, scheduled.DueTime); - // Advancing far less than the coalescing window still delivers the update immediately. - timeProvider.Advance(TimeSpan.FromMilliseconds(1)); var batch = await sent.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(TimeSpan.Zero, timeProvider.GetElapsedTime(start)); Assert.Equal(1, Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); await protocol.StopAsync(TestContext.Current.CancellationToken); } - [Fact] - public async Task HighPriorityRetryIsCappedByAntiEntropyInterval() - { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var timeProvider = new FakeTimeProvider(); - var firstAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var secondAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var sendCount = 0; - transport.SendBroadcastHandler = (target, batch, cancellationToken) => - { - if (Interlocked.Increment(ref sendCount) == 1) - { - firstAttempt.TrySetResult(); - throw new InvalidOperationException("transient send failure"); - } - - transport.BroadcastBatches.Add((target, batch)); - secondAttempt.TrySetResult(); - return Task.CompletedTask; - }; - - var normalNamespace = new FakeNamespace(local, new DisseminationNamespace("normal")); - normalNamespace.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); - var highNamespace = new FakeNamespace(local, new DisseminationNamespace("high")); - highNamespace.Options.Priority = DisseminationPriority.High; - var protocol = CreateProtocol( - transport, - new IDisseminationNamespace[] { normalNamespace, highNamespace }, - options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(1), - timeProvider); - using var schedule = new BroadcastScheduleObserver(); - - Assert.True(await PublishValue( - protocol, - highNamespace, - highNamespace.CreateValue(FakeNamespace.DefaultKey, sequence: 1), - TestContext.Current.CancellationToken)); - timeProvider.Advance(TimeSpan.FromMilliseconds(1)); - await firstAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - var retry = await schedule.WaitAsync( - e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry, - TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - - Assert.Equal(TimeSpan.FromSeconds(1), retry.DueTime); - timeProvider.Advance(TimeSpan.FromMilliseconds(999)); - Assert.False(secondAttempt.Task.IsCompleted); - timeProvider.Advance(TimeSpan.FromMilliseconds(1)); - await secondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await protocol.StopAsync(TestContext.Current.CancellationToken); - } - - [Fact] - public async Task HighPriorityNotificationPullsPendingCoalescedFlushForward() - { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var timeProvider = new FakeTimeProvider(); - var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - transport.SendBroadcastHandler = (target, batch, cancellationToken) => - { - transport.BroadcastBatches.Add((target, batch)); - sent.TrySetResult(); - return Task.CompletedTask; - }; - - var normalNamespace = new FakeNamespace(local, new DisseminationNamespace("normal")); - normalNamespace.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); - var highNamespace = new FakeNamespace(local, new DisseminationNamespace("high")); - highNamespace.Options.Priority = DisseminationPriority.High; - var protocol = CreateProtocol( - transport, - new IDisseminationNamespace[] { normalNamespace, highNamespace }, - timeProvider: timeProvider); - using var schedule = new BroadcastScheduleObserver(); - - // A normal update on the peer pump arms a distant coalescing timer. - Assert.True(await PublishValue( - protocol, - normalNamespace, - normalNamespace.CreateValue(FakeNamespace.DefaultKey, sequence: 1), - TestContext.Current.CancellationToken)); - var coalesced = await schedule.WaitAsync( - e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Coalesce, - TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.Equal(TimeSpan.FromMinutes(1), coalesced.DueTime); - Assert.False(sent.Task.IsCompleted); - - // A high-priority update on the same pump must pull that flush forward to now. - Assert.True(await PublishValue( - protocol, - highNamespace, - highNamespace.CreateValue(FakeNamespace.DefaultKey, sequence: 1), - TestContext.Current.CancellationToken)); - var pulled = await schedule.WaitAsync( - e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Priority, - TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.Equal(TimeSpan.Zero, pulled.DueTime); - - timeProvider.Advance(TimeSpan.FromMilliseconds(1)); - await sent.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await protocol.StopAsync(TestContext.Current.CancellationToken); - } - - [Fact] - public async Task NormalPriorityNamespaceStillWaitsForCoalescingWindow() - { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var timeProvider = new FakeTimeProvider(); - var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - transport.SendBroadcastHandler = (target, batch, cancellationToken) => - { - transport.BroadcastBatches.Add((target, batch)); - sent.TrySetResult(); - return Task.CompletedTask; - }; - - var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); - Assert.Equal(DisseminationPriority.Normal, ns.Options.Priority); - var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); - using var schedule = new BroadcastScheduleObserver(); - - Assert.True(await PublishValue( - protocol, - ns, - ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), - TestContext.Current.CancellationToken)); - var scheduled = await schedule.WaitAsync( - e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Coalesce, - TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.Equal(TimeSpan.FromSeconds(1), scheduled.DueTime); - - // A sub-window advance must not release the coalesced batch. - timeProvider.Advance(TimeSpan.FromMilliseconds(999)); - Assert.False(sent.Task.IsCompleted); - - // Crossing the window flushes it. - timeProvider.Advance(TimeSpan.FromMilliseconds(1)); - await sent.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - await protocol.StopAsync(TestContext.Current.CancellationToken); - } - - [Fact] - public async Task HighPriorityNamespaceValuesAreSentAheadOfNormalPriority() - { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var timeProvider = new FakeTimeProvider(); - transport.SendBroadcastResponseHandler = (target, batch, cancellationToken) => - { - transport.BroadcastBatches.Add((target, batch)); - var acknowledgments = new Dictionary>(); - foreach (var (namespaceName, values) in batch.Values) - { - acknowledgments[namespaceName] = - [.. values.Select(static value => new DigestEntry(value.Value.Key, value.Value.ToVersion))]; - } - - return Task.FromResult(new DisseminationBroadcastResponse { Acknowledgments = acknowledgments }); - }; - - var normalNamespace = new FakeNamespace(local, new DisseminationNamespace("normal")); - normalNamespace.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); - var highNamespace = new FakeNamespace(local, new DisseminationNamespace("high")); - highNamespace.Options.Priority = DisseminationPriority.High; - DisseminationOptions? optionsRef = null; - var protocol = CreateProtocol( - transport, - new IDisseminationNamespace[] { normalNamespace, highNamespace }, - options => - { - optionsRef = options; - options.MaxBatchItems = 10; - }, - timeProvider); - - Assert.True(await PublishValue( - protocol, - normalNamespace, - normalNamespace.CreateValue(FakeNamespace.DefaultKey, sequence: 1), - TestContext.Current.CancellationToken)); - Assert.True(await PublishValue( - protocol, - highNamespace, - highNamespace.CreateValue(FakeNamespace.DefaultKey, sequence: 1), - TestContext.Current.CancellationToken)); - - // One value per batch, so the recorded send sequence reflects the drain order. - optionsRef!.MaxBatchItems = 1; - await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - - Assert.Equal( - new[] { highNamespace.Name, normalNamespace.Name }, - transport.BroadcastBatches.Select(batch => Assert.Single(batch.Batch.Values.Keys))); - await protocol.StopAsync(TestContext.Current.CancellationToken); - } - [Fact] public async Task PeerVersionAdvancesOnlyFromExplicitAcknowledgment() { @@ -754,7 +523,6 @@ public async Task UnsupportedNamespaceResponseStopsAutomaticRetries() }; var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); Assert.True(await PublishValue( @@ -790,7 +558,6 @@ public async Task UnsupportedNamespaceResponseCompletesQueuedFlushWaiter() }; var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); Assert.True(await PublishValue( @@ -798,7 +565,6 @@ public async Task UnsupportedNamespaceResponseCompletesQueuedFlushWaiter() ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken)); - timeProvider.Advance(ns.Options.MaxCoalescingDelay); await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.True(await PublishValue( protocol, @@ -828,15 +594,15 @@ public async Task RemovedPendingKeyIsDroppedWithoutRetry() }; var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); - Assert.True(await PublishValue( - protocol, - ns, - ns.CreateValue("removed", sequence: 1), - TestContext.Current.CancellationToken)); - ns.RemoveValue("removed"); + var publication = BeforeBroadcastPumpsRun(() => + { + var pending = PublishValue(protocol, ns, ns.CreateValue("removed", sequence: 1), TestContext.Current.CancellationToken); + ns.RemoveValue("removed"); + return pending; + }); + Assert.True(await publication); await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); var repairRequestCount = ns.RepairRequestCount; timeProvider.Advance(TimeSpan.FromMinutes(1)); @@ -867,7 +633,6 @@ public async Task BroadcastBatchingStopsPeerFlushAfterSendFailure() }; var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); DisseminationOptions? optionsRef = null; var protocol = CreateProtocol(transport, ns, options => { @@ -875,59 +640,25 @@ public async Task BroadcastBatchingStopsPeerFlushAfterSendFailure() options.MaxBatchItems = 10; }, timeProvider); - Assert.True(await PublishValue(protocol, ns, ns.CreateValue("first", sequence: 1), TestContext.Current.CancellationToken)); - Assert.True(await PublishValue(protocol, ns, ns.CreateValue("second", sequence: 1), TestContext.Current.CancellationToken)); - Assert.True(await PublishValue(protocol, ns, ns.CreateValue("third", sequence: 1), TestContext.Current.CancellationToken)); - optionsRef!.MaxBatchItems = 1; - await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - - Assert.Equal(1, sendCount); - Assert.Empty(transport.BroadcastBatches); - } - - [Fact] - public async Task OrderedBroadcastBatchingRetriesFailedChainBeforeNewerValues() - { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var attempts = new List(); - var failed = false; - transport.SendBroadcastHandler = (target, batch, cancellationToken) => + var publications = BeforeBroadcastPumpsRun(() => { - var value = Assert.Single(GetBroadcastValues(batch)); - attempts.Add(value.Value.ToVersion); - if (value.Value.ToVersion == 2 && !failed) + var pending = new[] { - failed = true; - throw new InvalidOperationException("transient send failure"); - } - - transport.BroadcastBatches.Add((target, batch)); - return Task.CompletedTask; - }; - - var ns = new FakeNamespace(local) { ReturnRepairChain = true }; - ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); - DisseminationOptions? optionsRef = null; - var protocol = CreateProtocol(transport, ns, options => - { - optionsRef = options; - options.MaxBatchItems = 10; + PublishValue(protocol, ns, ns.CreateValue("first", sequence: 1), TestContext.Current.CancellationToken), + PublishValue(protocol, ns, ns.CreateValue("second", sequence: 1), TestContext.Current.CancellationToken), + PublishValue(protocol, ns, ns.CreateValue("third", sequence: 1), TestContext.Current.CancellationToken), + }; + optionsRef!.MaxBatchItems = 1; + return pending; }); - - Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken)); - Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2, fromVersion: 1), TestContext.Current.CancellationToken)); - Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 3, fromVersion: 2), TestContext.Current.CancellationToken)); - optionsRef!.MaxBatchItems = 1; - await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - - Assert.Equal(new long[] { 1, 2 }, attempts); - - Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 4, fromVersion: 3), TestContext.Current.CancellationToken)); + foreach (var publication in publications) + { + Assert.True(await publication); + } await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - Assert.Equal(new long[] { 1, 2, 2, 3, 4 }, attempts); + Assert.Equal(1, sendCount); + Assert.Empty(transport.BroadcastBatches); } [Fact] @@ -975,17 +706,18 @@ public async Task BroadcastPeerFailureDoesNotBlockOtherPeerPumps() [Fact] public async Task MembershipRefreshDropsPendingBroadcastForRemovedPeers() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + var (_, peer, transport, ns) = CreatePeerFixture(); var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1); - Assert.True(await PublishValue(protocol, ns, ns.CreateValue("before-removal", sequence: 1), TestContext.Current.CancellationToken)); - - transport.Peers.Remove(peer); - Assert.True(await PublishValue(protocol, ns, ns.CreateValue("during-removal", sequence: 2), TestContext.Current.CancellationToken)); + var publications = BeforeBroadcastPumpsRun(() => + { + var before = PublishValue(protocol, ns, ns.CreateValue("before-removal", sequence: 1), TestContext.Current.CancellationToken); + transport.Peers.Remove(peer); + var after = PublishValue(protocol, ns, ns.CreateValue("during-removal", sequence: 2), TestContext.Current.CancellationToken); + return (before, after); + }); + Assert.True(await publications.before); + Assert.True(await publications.after); await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); Assert.Empty(transport.BroadcastBatches); @@ -994,15 +726,14 @@ public async Task MembershipRefreshDropsPendingBroadcastForRemovedPeers() [Fact] public async Task StopDrainsPendingBroadcastPumps() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); + var (_, peer, transport, ns) = CreatePeerFixture(); var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1); - Assert.True(await PublishValue(protocol, ns, ns.CreateValue("before-stop", sequence: 1), TestContext.Current.CancellationToken)); - await protocol.StopAsync(TestContext.Current.CancellationToken); + var pending = BeforeBroadcastPumpsRun(() => ( + Publication: PublishValue(protocol, ns, ns.CreateValue("before-stop", sequence: 1), TestContext.Current.CancellationToken), + Stop: protocol.StopAsync(TestContext.Current.CancellationToken))); + Assert.True(await pending.Publication); + await pending.Stop; var batch = Assert.Single(transport.BroadcastBatches); Assert.Equal(peer, batch.Peer); @@ -1026,11 +757,9 @@ public async Task FlushPendingBroadcastWaitsForInFlightFlush() }; var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1, timeProvider); Assert.True(await PublishValue(protocol, ns, ns.CreateValue("in-flight", sequence: 1), TestContext.Current.CancellationToken)); - timeProvider.Advance(ns.Options.MaxCoalescingDelay); await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); @@ -1049,7 +778,7 @@ public async Task FlushPendingBroadcastWaitsForInFlightFlush() } [Fact] - public async Task NotificationDuringInFlightSendRepairsFromAcknowledgedVersion() + public async Task NotificationDuringInFlightSendRepairsLatestFullValueAfterAcknowledgment() { var local = CreateSilo(11111); var peer = CreateSilo(11112); @@ -1070,8 +799,7 @@ public async Task NotificationDuringInFlightSendRepairsFromAcknowledgedVersion() return FakeTransport.CreateAcknowledgment(batch); }; - var ns = new FakeNamespace(local) { ReturnRepairChain = true }; - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var ns = new FakeNamespace(local); var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); Assert.True(await PublishValue( @@ -1079,19 +807,18 @@ public async Task NotificationDuringInFlightSendRepairsFromAcknowledgedVersion() ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken)); - timeProvider.Advance(ns.Options.MaxCoalescingDelay); await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.True(await PublishValue( protocol, ns, - ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2, fromVersion: 1), + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), TestContext.Current.CancellationToken)); releaseFirstSend.TrySetResult(); await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal( - new[] { (From: 0L, To: 1L), (From: 1L, To: 2L) }, + new[] { (From: 0L, To: 1L), (From: 0L, To: 2L) }, transport.BroadcastBatches.Select(batch => { var value = Assert.Single(GetBroadcastValues(batch.Batch)); @@ -1115,11 +842,9 @@ public async Task MembershipRefreshCompletesFlushWaitersForRemovedPeers() }; var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1, timeProvider); Assert.True(await PublishValue(protocol, ns, ns.CreateValue("in-flight", sequence: 1), TestContext.Current.CancellationToken)); - timeProvider.Advance(ns.Options.MaxCoalescingDelay); await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.True(await PublishValue(protocol, ns, ns.CreateValue("pending", sequence: 1), TestContext.Current.CancellationToken)); var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); @@ -1291,79 +1016,33 @@ await protocol.ReceiveBroadcast( Assert.False(ns.ApplyCounts.ContainsKey(FakeNamespace.DefaultKey)); } - [Fact] - public async Task ReceiveBroadcastAppliesFullValueOverOlderLocalVersion() - { - var root = CreateSilo(11111); - var local = CreateSilo(11112); - var transport = new FakeTransport(local, root); - var ns = new FakeNamespace(local); - var protocol = CreateProtocol(transport, ns); - ns.SetValue(FakeNamespace.DefaultKey, version: 1); - var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 3, fromVersion: 0); - - await protocol.ReceiveBroadcast(CreateBroadcastBatch(root, item), TestContext.Current.CancellationToken); - - Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); - Assert.Equal(1, ns.ApplyCounts[FakeNamespace.DefaultKey]); - } - - [Fact] - public async Task ReceiveBroadcastAppliesDeltaValueOnlyWhenFromVersionMatches() - { - var root = CreateSilo(11111); - var local = CreateSilo(11112); - var transport = new FakeTransport(local, root); - var ns = new FakeNamespace(local); - var protocol = CreateProtocol(transport, ns); - ns.SetValue(FakeNamespace.DefaultKey, version: 2); - var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 3, fromVersion: 2); - - await protocol.ReceiveBroadcast(CreateBroadcastBatch(root, item), TestContext.Current.CancellationToken); - - Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); - Assert.Equal(1, ns.ApplyCounts[FakeNamespace.DefaultKey]); - } - - [Fact] - public async Task ReceiveBroadcastRejectsDeltaValueWhenFromVersionIsMissing() - { - var root = CreateSilo(11111); - var local = CreateSilo(11112); - var transport = new FakeTransport(local, root); - var ns = new FakeNamespace(local); - var protocol = CreateProtocol(transport, ns); - ns.SetValue(FakeNamespace.DefaultKey, version: 1); - var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 3, fromVersion: 2); - - await protocol.ReceiveBroadcast(CreateBroadcastBatch(root, item), TestContext.Current.CancellationToken); - - Assert.Equal(1, ns.GetVersion(FakeNamespace.DefaultKey)); - Assert.False(ns.ApplyCounts.ContainsKey(FakeNamespace.DefaultKey)); - } - - [Fact] - public async Task ReceiveBroadcastAcknowledgesCurrentVersionAfterRejectedRepair() - { - var sender = CreateSilo(11111); - var local = CreateSilo(11112); - var transport = new FakeTransport(local, sender); - var ns = new FakeNamespace(local); + [Theory] + [InlineData(1, 0, 3, 3, 1)] + [InlineData(2, 2, 3, 2, 0)] + [InlineData(1, 2, 3, 1, 0)] + [InlineData(1, -1, 3, 1, 0)] + [InlineData(1, 0, 0, 1, 0)] + [InlineData(1, 0, -1, 1, 0)] + public async Task ReceiveBroadcastAppliesOnlyPositiveFullValues( + long localVersion, long fromVersion, long toVersion, long expectedVersion, int expectedApplyCount) + { + var (_, root, transport, ns) = CreatePeerFixture(11112, 11111); var protocol = CreateProtocol(transport, ns); - ns.SetValue(FakeNamespace.DefaultKey, version: 1); - var item = ns.CreateItem( - sender, - FakeNamespace.DefaultKey, - sequence: 3, - fromVersion: 2); + ns.SetValue(FakeNamespace.DefaultKey, localVersion); + var item = ns.CreateItem(root, FakeNamespace.DefaultKey, toVersion, fromVersion); + using var rejected = new Phase6ApplyObserver(ns.Name, FakeNamespace.DefaultKey, transport.LocalSilo, root); var response = await protocol.ReceiveBroadcast( - CreateBroadcastBatch(sender, item), + CreateBroadcastBatch(root, item), TestContext.Current.CancellationToken); + Assert.Equal(expectedVersion, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(expectedApplyCount, ns.ApplyCounts.GetValueOrDefault(FakeNamespace.DefaultKey)); + Assert.Equal(expectedApplyCount != 0, ns.ApplyCounts.ContainsKey(FakeNamespace.DefaultKey)); + Assert.Equal(expectedApplyCount == 0 ? 1 : 0, rejected.Events.Count); var acknowledgment = Assert.Single(response.Acknowledgments[ns.Name]); Assert.Equal(FakeNamespace.DefaultKey, acknowledgment.Key); - Assert.Equal(1, acknowledgment.Version); + Assert.Equal(expectedVersion, acknowledgment.Version); } [Fact] @@ -1453,16 +1132,16 @@ public async Task TreeRoutingInvalidatesCachedTopologyWhenActivePeersChange() } [Fact] - public async Task BroadcastBatchingCoalescesSamePeerValuesByLatestVersion() + public async Task BroadcastMaterializesLatestQueuedVersion() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(); var protocol = CreateProtocol(transport, ns); - var first = await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken); - var second = await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), TestContext.Current.CancellationToken); + var publications = BeforeBroadcastPumpsRun(() => ( + First: PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken), + Second: PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), TestContext.Current.CancellationToken))); + var first = await publications.First; + var second = await publications.Second; await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); Assert.True(first); @@ -1474,12 +1153,12 @@ public async Task BroadcastBatchingCoalescesSamePeerValuesByLatestVersion() } [Fact] - public async Task BroadcastRepairsFromAcknowledgedPeerVersion() + public async Task BroadcastRepairsLatestFullValueAfterAcknowledgment() { var local = CreateSilo(11111); var peer = CreateSilo(11112); var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local) { ReturnRepairChain = true }; + var ns = new FakeNamespace(local); var protocol = CreateProtocol(transport, ns); Assert.True(await PublishValue( @@ -1491,12 +1170,12 @@ public async Task BroadcastRepairsFromAcknowledgedPeerVersion() Assert.True(await PublishValue( protocol, ns, - ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2, fromVersion: 1), + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), TestContext.Current.CancellationToken)); await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); Assert.Equal( - new[] { (From: 0L, To: 1L), (From: 1L, To: 2L) }, + new[] { (From: 0L, To: 1L), (From: 0L, To: 2L) }, transport.BroadcastBatches.Select(batch => { var value = Assert.Single(GetBroadcastValues(batch.Batch)); @@ -1504,52 +1183,31 @@ public async Task BroadcastRepairsFromAcknowledgedPeerVersion() })); } - [Fact] - public async Task BroadcastBatchingPreservesOrderedSameKeyValues() - { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local) { ReturnRepairChain = true }; - var protocol = CreateProtocol(transport, ns); - - Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken)); - Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2, fromVersion: 1), TestContext.Current.CancellationToken)); - Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 3, fromVersion: 2), TestContext.Current.CancellationToken)); - await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - - var batch = Assert.Single(transport.BroadcastBatches); - Assert.Equal( - new[] { (From: 0L, To: 1L), (From: 1L, To: 2L), (From: 2L, To: 3L) }, - GetBroadcastValues(batch.Batch).Select(static value => (value.Value.FromVersion, value.Value.ToVersion))); - } - - [Fact] - public async Task BroadcastBatchingLetsFullValueSupersedeOrderedValues() + [Theory] + [InlineData(3)] + [InlineData(4)] + public async Task BroadcastPublicationSupersedesMissingHistoryWithLatestFullValue(long latestVersion) { var local = CreateSilo(11111); var peer = CreateSilo(11112); var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local) { ReturnRepairChain = true }; + var ns = new FakeNamespace(local); var protocol = CreateProtocol(transport, ns); - Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2, fromVersion: 1), TestContext.Current.CancellationToken)); - Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 3, fromVersion: 2), TestContext.Current.CancellationToken)); - Assert.True(await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 4), TestContext.Current.CancellationToken)); + ns.SetValue(FakeNamespace.DefaultKey, latestVersion); + Assert.True(await protocol.Publish(ns, FakeNamespace.DefaultKey, 1, TestContext.Current.CancellationToken)); await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); var value = Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)); Assert.Equal(0, value.Value.FromVersion); - Assert.Equal(4, value.Value.ToVersion); + Assert.Equal(latestVersion, value.Value.ToVersion); + Assert.Equal(latestVersion, BitConverter.ToInt64(value.Value.Payload.Span)); } [Fact] public async Task BroadcastPeerSenderCachesSystemTarget() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(); var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1); Assert.True(await PublishValue(protocol, ns, ns.CreateValue("first", sequence: 1), TestContext.Current.CancellationToken)); @@ -1561,73 +1219,6 @@ public async Task BroadcastPeerSenderCachesSystemTarget() Assert.Equal(1, transport.GetTargetResolutionCount(peer)); } - [Fact] - public async Task BroadcastBatchingWakesScheduledFlushWhenBatchLimitIsReached() - { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - transport.SendBroadcastHandler = (target, batch, cancellationToken) => - { - transport.BroadcastBatches.Add((target, batch)); - sent.TrySetResult(true); - return Task.CompletedTask; - }; - - var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); - var protocol = CreateProtocol(transport, ns, options => options.MaxBatchItems = 2); - - var first = await PublishValue(protocol, ns, ns.CreateValue("first", sequence: 1), TestContext.Current.CancellationToken); - await Task.Delay(TimeSpan.FromMilliseconds(50), TestContext.Current.CancellationToken); - var second = await PublishValue(protocol, ns, ns.CreateValue("second", sequence: 1), TestContext.Current.CancellationToken); - - await sent.Task.WaitAsync(TimeSpan.FromSeconds(2), TestContext.Current.CancellationToken); - Assert.True(first); - Assert.True(second); - var batch = Assert.Single(transport.BroadcastBatches); - Assert.Equal(new DisseminationKey[] { "first", "second" }, GetBroadcastValues(batch.Batch).Select(static value => value.Value.Key)); - } - - [Fact] - public async Task BroadcastBatchingUsesShortestConfiguredCoalescingDelay() - { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var timeProvider = new FakeTimeProvider(); - var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - transport.SendBroadcastHandler = (target, batch, cancellationToken) => - { - transport.BroadcastBatches.Add((target, batch)); - sent.TrySetResult(); - return Task.CompletedTask; - }; - - var slowNamespace = new FakeNamespace(local, "slow"); - slowNamespace.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); - var fastNamespace = new FakeNamespace(local, "fast"); - fastNamespace.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); - var protocol = CreateProtocol( - transport, - new IDisseminationNamespace[] { slowNamespace, fastNamespace }, - options => options.Overlay.FanOutFactor = static _ => 1, - timeProvider); - - Assert.True(await PublishValue(protocol, slowNamespace, slowNamespace.CreateValue("slow", sequence: 1), TestContext.Current.CancellationToken)); - Assert.True(await PublishValue(protocol, fastNamespace, fastNamespace.CreateValue("fast", sequence: 1), TestContext.Current.CancellationToken)); - - timeProvider.Advance(TimeSpan.FromMilliseconds(999)); - Assert.False(sent.Task.IsCompleted); - timeProvider.Advance(TimeSpan.FromMilliseconds(1)); - await sent.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - - var batch = Assert.Single(transport.BroadcastBatches); - Assert.Equal(2, batch.Batch.Values.Count); - await protocol.StopAsync(TestContext.Current.CancellationToken); - } - [Fact] public void FixedTreeRoutingSatisfiesReachabilityAndFanoutInvariants() { @@ -1691,59 +1282,44 @@ public void AntiEntropyPeerSelectionSamplesDistinctMembers() }); } - [Fact] - public async Task AntiEntropyResponseIncludesNewerLocalValues() + [Theory] + [InlineData(3, false)] + [InlineData(3, true)] + [InlineData(5, false)] + public async Task AntiEntropyResponseIncludesOnlyRequestedNewerValues(long peerVersion, bool hasUnrequestedKey) { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(); var protocol = CreateProtocol(transport, ns); - ns.SetValue(FakeNamespace.DefaultKey, version: 5); - - var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + var key = hasUnrequestedKey ? new DisseminationKey("requested") : FakeNamespace.DefaultKey; + ns.SetValue(key, version: 5); + if (hasUnrequestedKey) { - Sender = peer, - Digests = CreateAntiEntropyRequestDigest( - ns.Name, - (FakeNamespace.DefaultKey, 3)), - }, TestContext.Current.CancellationToken); - - var item = Assert.Single(GetAntiEntropyResponseValues(response)); - Assert.Equal(5, item.Value.ToVersion); - Assert.False(response.Truncated); - } - - [Fact] - public async Task AntiEntropyResponseOnlyIncludesRequestedDigestKeys() - { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); - var protocol = CreateProtocol(transport, ns); - ns.SetValue("requested", version: 5); - ns.SetValue("omitted", version: 5); + ns.SetValue("omitted", version: 5); + } var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest { Sender = peer, - Digests = CreateAntiEntropyRequestDigest( - ns.Name, - ("requested", 3)), + Digests = CreateAntiEntropyRequestDigest(ns.Name, (key, peerVersion)), }, TestContext.Current.CancellationToken); - var item = Assert.Single(GetAntiEntropyResponseValues(response)); - Assert.Equal(new DisseminationKey("requested"), item.Value.Key); + if (peerVersion == 5) + { + Assert.Empty(response.Values); + } + else + { + var item = Assert.Single(GetAntiEntropyResponseValues(response)); + Assert.Equal(key, item.Value.Key); + Assert.Equal(5, item.Value.ToVersion); + } + Assert.False(response.Truncated); } [Fact] public async Task OversizedAntiEntropyKeyDoesNotStarveLaterRepairs() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(); ns.Options.MaxPayloadBytes = sizeof(long); ns.PublishValue(new DisseminationValue( "oversized", @@ -1770,10 +1346,7 @@ public async Task OversizedAntiEntropyKeyDoesNotStarveLaterRepairs() [Fact] public async Task AntiEntropyDigestRequestsAreSuppressedUntilExpectedCadenceExpires() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(); var timeProvider = new TestTimeProvider(); ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(2); var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); @@ -1805,10 +1378,7 @@ await protocol.ReceiveBroadcast( [Fact] public async Task AntiEntropyCapabilityExchangeConfirmsNamespaceWithoutRepairDigests() { - var local = CreateSilo(11113); - var peer = CreateSilo(11114); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(11113, 11114); transport.ExchangeAntiEntropyHandler = (target, request, _) => { Assert.Empty(request.Digests); @@ -1869,10 +1439,7 @@ public async Task AntiEntropyCapabilityOnlyRoundUsesNamespaceStaleItemTtl() [Fact] public async Task DuplicateBroadcastDoesNotPostponeAntiEntropyProbe() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(); var timeProvider = new TestTimeProvider(); ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(2); var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); @@ -1923,10 +1490,7 @@ public async Task AntiEntropySendsOneDigestRequestPerPeer() [Fact] public async Task AntiEntropyExchangeFailureDoesNotBackOffPeer() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, _, transport, ns) = CreatePeerFixture(); ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); var exchangeCount = 0; transport.ExchangeAntiEntropyHandler = (target, request, _) => @@ -1954,10 +1518,7 @@ public async Task AntiEntropyExchangeFailureDoesNotBackOffPeer() [Fact] public async Task AntiEntropyExchangePropagatesCancellation() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, _, transport, ns) = CreatePeerFixture(); ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); var exchangeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); using var cancellation = new CancellationTokenSource(); @@ -1981,10 +1542,7 @@ await Assert.ThrowsAnyAsync( [Fact] public async Task AntiEntropyResponseHonorsCancellation() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(); var protocol = CreateProtocol(transport, ns); using var cancellation = new CancellationTokenSource(); cancellation.Cancel(); @@ -2139,7 +1697,7 @@ await Task.WhenAll(fastResponseReturned.Task, slowExchangeStarted.Task) } [Fact] - public async Task AntiEntropyRanksStaggeredResponsesWhichCompleteWithinRoundLifetime() + public async Task AntiEntropyAppliesStaggeredFullResponsesAfterExchangesComplete() { var local = CreateSilo(11111); var fastPeer = CreateSilo(11112); @@ -2152,7 +1710,7 @@ public async Task AntiEntropyRanksStaggeredResponsesWhichCompleteWithinRoundLife var releaseSlowResponse = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); transport.ExchangeAntiEntropyHandler = async (target, request, cancellationToken) => { - var requestedVersion = Assert.Single(request.Digests[ns.Name]).Version; + Assert.Equal(1, Assert.Single(request.Digests[ns.Name]).Version); var responseVersion = target.Equals(fastPeer) ? 2 : 3; if (target.Equals(slowerPeer)) { @@ -2170,8 +1728,7 @@ public async Task AntiEntropyRanksStaggeredResponsesWhichCompleteWithinRoundLife Values = CreateValueGroups(ns.CreateItem( target, FakeNamespace.DefaultKey, - sequence: responseVersion, - fromVersion: requestedVersion)), + sequence: responseVersion)), }; }; var protocol = CreateProtocol( @@ -2188,15 +1745,13 @@ await Task.WhenAll(fastResponseReturned.Task, slowExchangeStarted.Task) await round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(2, ns.ApplyCounts[FakeNamespace.DefaultKey]); } [Fact] public async Task AntiEntropyAppliesReturnedRepairItemsWithoutForwarding() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(); var protocol = CreateProtocol(transport, ns); ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); var repairItem = ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 7); @@ -2219,24 +1774,21 @@ public async Task AntiEntropyAppliesReturnedRepairItemsWithoutForwarding() } [Fact] - public async Task AntiEntropyAppliesRepairChainInOrder() + public async Task AntiEntropyRejectsNonFullRepairAndAppliesLaterFullValue() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); - ns.SetValue(FakeNamespace.DefaultKey, version: 1); - var protocol = CreateProtocol(transport, ns); - var second = ns.CreateItem( + var (local, peer, transport, _) = CreatePeerFixture(); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.SetValue(FakeNamespace.DefaultKey, version: 1); + var protocol = CreateProtocol(transport, [ns]); + var second = ns.Inner.CreateItem( peer, FakeNamespace.DefaultKey, sequence: 2, fromVersion: 1); - var third = ns.CreateItem( + var third = ns.Inner.CreateItem( peer, FakeNamespace.DefaultKey, - sequence: 3, - fromVersion: 2); + sequence: 3); transport.ExchangeAntiEntropyHandler = (target, request, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse { Sender = target, @@ -2246,28 +1798,32 @@ public async Task AntiEntropyAppliesRepairChainInOrder() await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); - Assert.Equal(2, ns.ApplyCounts[FakeNamespace.DefaultKey]); + Assert.Equal(1, ns.Inner.ApplyCounts[FakeNamespace.DefaultKey]); + Assert.Equal(new long[] { 3 }, ns.Attempts.Select(static value => value.ToVersion)); Assert.Empty(transport.BroadcastBatches); } - [Fact] - public async Task AntiEntropyAppliesNewestRepairFirstForEachStream() + [Theory] + [InlineData(2, 3, 2)] + [InlineData(3, 2, 1)] + [InlineData(2, 2, 1)] + public async Task AntiEntropyAppliesFullResponsesWithoutRegressingState( + long firstVersion, long secondVersion, int expectedApplyCount) { var local = CreateSilo(11111); var peers = new[] { CreateSilo(11112), CreateSilo(11113) }; var transport = new FakeTransport(local, peers); - var ns = new FakeNamespace(local); - ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.SetValue(FakeNamespace.DefaultKey, version: 1); var exchangeCount = 0; transport.ExchangeAntiEntropyHandler = (target, request, _) => { - var requestedVersion = Assert.Single(request.Digests[ns.Name]).Version; - var responseVersion = Interlocked.Increment(ref exchangeCount) == 1 ? 2 : 3; - var repair = ns.CreateItem( + Assert.Equal(1, Assert.Single(request.Digests[ns.Name]).Version); + var responseVersion = Interlocked.Increment(ref exchangeCount) == 1 ? firstVersion : secondVersion; + var repair = ns.Inner.CreateItem( target, FakeNamespace.DefaultKey, - sequence: responseVersion, - fromVersion: requestedVersion); + sequence: responseVersion); return ValueTask.FromResult(new DisseminationAntiEntropyResponse { Sender = target, @@ -2277,34 +1833,16 @@ public async Task AntiEntropyAppliesNewestRepairFirstForEachStream() var protocol = CreateProtocol( transport, - ns, + [ns], options => options.Overlay.AntiEntropyPeerCount = peers.Length); await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); - Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(Math.Max(firstVersion, secondVersion), ns.GetVersion(FakeNamespace.DefaultKey)); Assert.Equal(peers.Length, exchangeCount); - } - - [Fact] - public async Task AntiEntropyResponseOmitsNamespacesWithoutRepairs() - { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); - ns.SetValue(FakeNamespace.DefaultKey, version: 5); - var protocol = CreateProtocol(transport, ns); - - var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest - { - Sender = peer, - Digests = CreateAntiEntropyRequestDigest( - ns.Name, - (FakeNamespace.DefaultKey, 5)), - }, TestContext.Current.CancellationToken); - - Assert.Empty(response.Values); + Assert.Equal(expectedApplyCount, ns.Inner.ApplyCounts[FakeNamespace.DefaultKey]); + long[] expectedAttempts = secondVersion < firstVersion ? [firstVersion] : [firstVersion, secondVersion]; + Assert.Equal(expectedAttempts, ns.Attempts.Select(static value => value.ToVersion)); } [Fact] @@ -2350,10 +1888,7 @@ await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest [Fact] public async Task AntiEntropyAppliesValidItemsAfterFailedRepairItem() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(); var protocol = CreateProtocol(transport, ns); ns.SetValue(FakeNamespace.DefaultKey, version: 1); var badRepairItem = new DisseminationBroadcastValue @@ -2376,10 +1911,7 @@ public async Task AntiEntropyAppliesValidItemsAfterFailedRepairItem() [Fact] public async Task AntiEntropyAppliesValidItemsAfterFailedRepairRound() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(); var protocol = CreateProtocol(transport, ns); ns.SetValue(FakeNamespace.DefaultKey, version: 1); @@ -2457,7 +1989,7 @@ public async Task MonotonicDisseminationModelConformsToAccordantSpec() #endif [Fact] - public async Task MembershipNamespaceReturnsAndAppliesDiffWhenPeerVersionIsRetained() + public async Task MembershipNamespaceRepairsCurrentFullSnapshotWithoutRetainingPeerBaseline() { var local = CreateSilo(11111); var peer = CreateSilo(11112); @@ -2479,18 +2011,15 @@ public async Task MembershipNamespaceReturnsAndAppliesDiffWhenPeerVersionIsRetai var repair = sourceNamespace.CreateRepair(new DisseminationRepairRequest( localDigest.Key, peerDigest.Version, - toVersion: null, - maxItemCount: 1, maxBatchBytes: 1024 * 1024, maxPayloadBytes: 1024 * 1024)); Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); - var value = Assert.Single(repair.Values); - Assert.Equal(peerDigest.Version, value.FromVersion); + var value = repair.Value; + Assert.Equal(0, value.FromVersion); Assert.Equal(localDigest.Version, value.ToVersion); var update = Assert.IsType( serializer.Deserialize(value.Payload)); - Assert.NotNull(update.Diff); - Assert.Null(update.Snapshot); + AssertMembershipState(updatedSnapshot, Assert.IsType(update.Snapshot)); var receiverManager = new FakeMembershipManager(baseSnapshot); var receiverNamespace = CreateMembershipNamespace(receiverManager, serializer); var result = await receiverNamespace.ApplyValueAsync(value, TestContext.Current.CancellationToken); @@ -2500,48 +2029,6 @@ public async Task MembershipNamespaceReturnsAndAppliesDiffWhenPeerVersionIsRetai Assert.True(receiverManager.CurrentSnapshot.Entries.ContainsKey(peer)); } - [Fact] - public async Task MembershipNamespaceAppliesSameVersionDiffWhenLivenessAdvances() - { - var local = CreateSilo(11111); - var receiverSnapshot = CreateMembershipSnapshot( - version: 2, - CreateMembershipEntry( - local, - SiloStatus.Active, - DateTime.UnixEpoch, - iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2))); - var updatedEntry = CreateMembershipEntry( - local, - SiloStatus.Active, - DateTime.UnixEpoch, - iAmAliveTime: DateTime.UnixEpoch.AddSeconds(3)); - using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); - var serializer = serviceProvider.GetRequiredService(); - var update = new MembershipTableSnapshotUpdate - { - Diff = new MembershipTableSnapshotDiff( - new MembershipVersion(1), - new MembershipVersion(2), - [updatedEntry], - []), - }; - var value = new DisseminationValue( - DisseminationKey.Default, - fromVersion: 1, - toVersion: 2, - serializer.SerializeToArray(update)); - var receiverManager = new FakeMembershipManager(receiverSnapshot); - var receiverNamespace = CreateMembershipNamespace(receiverManager, serializer); - - var result = await receiverNamespace.ApplyValueAsync(value, TestContext.Current.CancellationToken); - - Assert.Equal(DisseminationApplyResult.Applied, result); - Assert.Equal( - DateTime.UnixEpoch.AddSeconds(3), - receiverManager.CurrentSnapshot.Entries[local].IAmAliveTime); - } - [Fact] public void MembershipNamespaceInvalidatesCachedPayloadForSameVersionUpdate() { @@ -2567,8 +2054,6 @@ public void MembershipNamespaceInvalidatesCachedPayloadForSameVersionUpdate() var request = new DisseminationRepairRequest( DisseminationKey.Default, fromVersion: null, - toVersion: null, - maxItemCount: 1, maxBatchBytes: 1024 * 1024, maxPayloadBytes: 1024 * 1024); @@ -2576,8 +2061,10 @@ public void MembershipNamespaceInvalidatesCachedPayloadForSameVersionUpdate() manager.CurrentSnapshot = updatedSnapshot; var updatedRepair = disseminationNamespace.CreateRepair(request); - var firstValue = Assert.Single(firstRepair.Values); - var updatedValue = Assert.Single(updatedRepair.Values); + Assert.Equal(DisseminationRepairStatus.Produced, firstRepair.Status); + Assert.Equal(DisseminationRepairStatus.Produced, updatedRepair.Status); + var firstValue = firstRepair.Value; + var updatedValue = updatedRepair.Value; var firstUpdate = Assert.IsType( serializer.Deserialize(firstValue.Payload)); var updatedUpdate = Assert.IsType( @@ -2628,13 +2115,12 @@ public async Task MembershipNamespaceRetainsPostApplySnapshotForRepair() var repair = disseminationNamespace.CreateRepair(new DisseminationRepairRequest( DisseminationKey.Default, fromVersion: null, - toVersion: appliedSnapshot.Version.Value, - maxItemCount: 1, maxBatchBytes: 1024 * 1024, maxPayloadBytes: 1024 * 1024)); Assert.Equal(DisseminationApplyResult.Applied, result); - var repairedValue = Assert.Single(repair.Values); + Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); + var repairedValue = repair.Value; var repairedUpdate = Assert.IsType( serializer.Deserialize(repairedValue.Payload)); var repairedSnapshot = Assert.IsType(repairedUpdate.Snapshot); @@ -2743,7 +2229,7 @@ public async Task MembershipAntiEntropyRepairsSameVersionSuccessor() } [Fact] - public async Task MembershipNamespacePublishesOrderedDiffChain() + public async Task MembershipNamespacePublishesIndependentlyApplicableFullSnapshots() { var local = CreateSilo(11111); var peer = CreateSilo(11112); @@ -2788,26 +2274,32 @@ public async Task MembershipNamespacePublishesOrderedDiffChain() disseminationService.Values[2]); Assert.Equal((0L, 1L), (first.FromVersion, first.ToVersion)); - Assert.Equal((1L, 2L), (second.FromVersion, second.ToVersion)); - Assert.Equal((2L, 3L), (third.FromVersion, third.ToVersion)); + Assert.Equal((0L, 2L), (second.FromVersion, second.ToVersion)); + Assert.Equal((0L, 3L), (third.FromVersion, third.ToVersion)); Assert.NotNull(Assert.IsType( serializer.Deserialize(first.Payload)).Snapshot); Assert.NotNull(Assert.IsType( - serializer.Deserialize(second.Payload)).Diff); + serializer.Deserialize(second.Payload)).Snapshot); Assert.NotNull(Assert.IsType( - serializer.Deserialize(third.Payload)).Diff); + serializer.Deserialize(third.Payload)).Snapshot); var receiverManager = new FakeMembershipManager(CreateMembershipSnapshot(MembershipVersion.MinValue.Value)); var receiverNamespace = CreateMembershipNamespace(receiverManager, serializer); Assert.Equal(DisseminationApplyResult.Applied, await receiverNamespace.ApplyValueAsync(first, TestContext.Current.CancellationToken)); Assert.Equal(DisseminationApplyResult.Applied, await receiverNamespace.ApplyValueAsync(second, TestContext.Current.CancellationToken)); Assert.Equal(DisseminationApplyResult.Applied, await receiverNamespace.ApplyValueAsync(third, TestContext.Current.CancellationToken)); - Assert.Equal(thirdSnapshot.Version, receiverManager.CurrentSnapshot.Version); - Assert.Equal(SiloStatus.Active, receiverManager.CurrentSnapshot.Entries[peer].Status); + AssertMembershipState(thirdSnapshot, receiverManager.CurrentSnapshot); + + var lateReceiver = new FakeMembershipManager(CreateMembershipSnapshot(MembershipVersion.MinValue.Value)); + var lateNamespace = CreateMembershipNamespace(lateReceiver, serializer); + Assert.Equal(DisseminationApplyResult.Applied, await lateNamespace.ApplyValueAsync(third, TestContext.Current.CancellationToken)); + AssertMembershipState(thirdSnapshot, lateReceiver.CurrentSnapshot); } - [Fact] - public async Task MembershipNamespacePublishesFullSnapshotAfterFailedBaseline() + [Theory] + [InlineData(false, 2)] + [InlineData(true, 3)] + public async Task MembershipNamespacePublishesCurrentSnapshotWhenNotificationIsSuperseded(bool baselineAccepted, long nextVersion) { var local = CreateSilo(11111); var peer = CreateSilo(11112); @@ -2816,7 +2308,7 @@ public async Task MembershipNamespacePublishesFullSnapshotAfterFailedBaseline() CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); var secondSnapshot = CreateMembershipSnapshot( - version: 2, + version: nextVersion, CreateMembershipEntry( local, SiloStatus.Active, @@ -2828,20 +2320,22 @@ public async Task MembershipNamespacePublishesFullSnapshotAfterFailedBaseline() var sourceManager = new FakeMembershipManager(firstSnapshot); var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); var disseminationService = new FakeDisseminationService(); - disseminationService.Results.Enqueue(false); + disseminationService.Results.Enqueue(baselineAccepted); disseminationService.Results.Enqueue(true); - Assert.False(await sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken)); + Assert.Equal(baselineAccepted, await sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken)); sourceManager.CurrentSnapshot = secondSnapshot; - Assert.True(await sourceNamespace.PublishAsync(disseminationService, secondSnapshot, TestContext.Current.CancellationToken)); + Assert.True(await sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken)); - Assert.Equal(0, disseminationService.Values[1].FromVersion); - Assert.NotNull(Assert.IsType( - serializer.Deserialize(disseminationService.Values[1].Payload)).Snapshot); + var value = disseminationService.Values[1]; + Assert.Equal((0L, nextVersion), (value.FromVersion, value.ToVersion)); + var update = Assert.IsType( + serializer.Deserialize(value.Payload)); + AssertMembershipState(secondSnapshot, Assert.IsType(update.Snapshot)); } [Fact] - public async Task MembershipNamespacePublishesDiffWhenTopologyChanges() + public async Task MembershipNamespacePublishesFullInventoryWhenTopologyChanges() { var local = CreateSilo(11111); var firstPeer = CreateSilo(11112); @@ -2865,41 +2359,10 @@ public async Task MembershipNamespacePublishesDiffWhenTopologyChanges() sourceManager.CurrentSnapshot = secondSnapshot; Assert.True(await sourceNamespace.PublishAsync(disseminationService, secondSnapshot, TestContext.Current.CancellationToken)); - Assert.Equal(1, disseminationService.Values[1].FromVersion); - Assert.NotNull(Assert.IsType( - serializer.Deserialize(disseminationService.Values[1].Payload)).Diff); - } - - [Fact] - public async Task MembershipNamespacePublishesDirectDiffAcrossVersionGap() - { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var firstSnapshot = CreateMembershipSnapshot( - version: 1, - CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), - CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); - var thirdSnapshot = CreateMembershipSnapshot( - version: 3, - CreateMembershipEntry( - local, - SiloStatus.Active, - DateTime.UnixEpoch, - iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2)), - CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); - using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); - var serializer = serviceProvider.GetRequiredService(); - var sourceManager = new FakeMembershipManager(firstSnapshot); - var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); - var disseminationService = new FakeDisseminationService(); - - Assert.True(await sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken)); - sourceManager.CurrentSnapshot = thirdSnapshot; - Assert.True(await sourceNamespace.PublishAsync(disseminationService, thirdSnapshot, TestContext.Current.CancellationToken)); - - Assert.Equal(1, disseminationService.Values[1].FromVersion); - Assert.NotNull(Assert.IsType( - serializer.Deserialize(disseminationService.Values[1].Payload)).Diff); + Assert.Equal(0, disseminationService.Values[1].FromVersion); + var update = Assert.IsType( + serializer.Deserialize(disseminationService.Values[1].Payload)); + AssertMembershipState(secondSnapshot, Assert.IsType(update.Snapshot)); } [Fact] @@ -2981,9 +2444,6 @@ public void DisseminationKeyRoundTripsDefaultAndSiloAddressKeys() Assert.Equal(expected[..expectedLength].ToString(), actual[..actualLength].ToString()); Assert.True(roundTripMembershipValue.Key.TryFormat(Span.Empty, out var nullKeyLength, default, null)); Assert.Equal(0, nullKeyLength); - Assert.Equal( - Math.Sign(earlierPeer.CompareTo(peer)), - Math.Sign(Comparer.Default.Compare(earlierPeer, peer))); Assert.Equal( Math.Sign(earlierPeer.CompareTo(peer)), Math.Sign(new DisseminationKey(earlierPeer).CompareTo(new DisseminationKey(peer)))); @@ -3034,10 +2494,12 @@ public void OptionsValidatorRejectsInvalidFanoutBounds() } [Fact] - public void NamespaceOptionsValidatorAllowsStaleTtlShorterThanCoalescingDelay() + public void NamespaceOptionsValidatorAllowsStaleTtlShorterThanPublicationPeriod() { - var options = new DeploymentLoadPublisherOptions(); - options.Dissemination.MaxCoalescingDelay = TimeSpan.FromSeconds(1); + var options = new DeploymentLoadPublisherOptions + { + DeploymentLoadPublisherRefreshTime = TimeSpan.FromSeconds(1), + }; options.Dissemination.StaleItemTtl = TimeSpan.FromMilliseconds(1); var result = new DeploymentLoadPublisherOptionsValidator().Validate(Options.DefaultName, options); @@ -3090,57 +2552,25 @@ public void OptionsValidatorRejectsUnsupportedAntiEntropyIntervals(long millisec } [Theory] - [InlineData(0)] - [InlineData(4294967295)] - public void NamespaceOptionsValidatorRejectsUnsupportedCoalescingDelays(long milliseconds) - { - var options = new DeploymentLoadPublisherOptions(); - options.Dissemination.MaxCoalescingDelay = TimeSpan.FromMilliseconds(milliseconds); - options.Dissemination.StaleItemTtl = options.Dissemination.MaxCoalescingDelay + TimeSpan.FromSeconds(1); - var result = new DeploymentLoadPublisherOptionsValidator().Validate(Options.DefaultName, options); - - Assert.True(result.Failed); - } - - [Fact] - public async Task ReceiveBroadcastUsesRelativeLifetimeAcrossClockSkew() - { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); - var timeProvider = new FakeTimeProvider(DateTimeOffset.MaxValue - TimeSpan.FromSeconds(1)); + [InlineData(1, true)] + [InlineData(0, false)] + public async Task ReceiveBroadcastUsesOnlyPositiveRelativeLifetime(int milliseconds, bool skewClock) + { + var (_, peer, transport, ns) = CreatePeerFixture(); + var timeProvider = skewClock + ? new FakeTimeProvider(DateTimeOffset.MaxValue - TimeSpan.FromSeconds(1)) + : TimeProvider.System; var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); var item = ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 1); item = new DisseminationBroadcastValue { Value = item.Value, - TimeToLive = TimeSpan.FromMilliseconds(1), - }; - - await protocol.ReceiveBroadcast(CreateBroadcastBatch(peer, item), TestContext.Current.CancellationToken); - - Assert.Equal(1, ns.GetVersion(FakeNamespace.DefaultKey)); - } - - [Fact] - public async Task ReceiveBroadcastDropsNonPositiveRelativeLifetime() - { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); - var protocol = CreateProtocol(transport, ns); - var item = ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 1); - item = new DisseminationBroadcastValue - { - Value = item.Value, - TimeToLive = TimeSpan.Zero, + TimeToLive = TimeSpan.FromMilliseconds(milliseconds), }; await protocol.ReceiveBroadcast(CreateBroadcastBatch(peer, item), TestContext.Current.CancellationToken); - Assert.Equal(0, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(milliseconds > 0 ? 1 : 0, ns.GetVersion(FakeNamespace.DefaultKey)); } [Fact] @@ -3168,7 +2598,6 @@ public async Task BroadcastTransportCancelsWhenRelativeLifetimeExpires() }; var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); @@ -3177,7 +2606,6 @@ public async Task BroadcastTransportCancelsWhenRelativeLifetimeExpires() ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken)); - timeProvider.Advance(ns.Options.MaxCoalescingDelay); await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); var flush = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); @@ -3204,7 +2632,6 @@ public void OptionsValidatorRejectsInvalidExpectedUpdateCadence() public void NamespaceOptionsUseExpectedUpdateCadenceDefaults() { Assert.Equal(TimeSpan.FromSeconds(5), new DeploymentLoadPublisherOptions().Dissemination.ExpectedUpdateCadence); - Assert.Equal(TimeSpan.FromMilliseconds(100), new DeploymentLoadPublisherOptions().Dissemination.MaxCoalescingDelay); Assert.Equal(8192, new DeploymentLoadPublisherOptions().Dissemination.MaxPendingItemCount); Assert.Equal(TimeSpan.FromSeconds(10), new ClusterMembershipOptions().Dissemination.ExpectedUpdateCadence); } @@ -3400,7 +2827,6 @@ public async Task DisabledNamespaceBetweenPublishAndReschedule_UsesFiniteBounded }; var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); var protocol = CreateProtocol( transport, ns, @@ -3418,7 +2844,6 @@ public async Task DisabledNamespaceBetweenPublishAndReschedule_UsesFiniteBounded ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken)); - timeProvider.Advance(TimeSpan.FromSeconds(1)); await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); ns.Options.Enabled = false; @@ -3474,7 +2899,6 @@ public async Task UnexpectedPeerPumpIterationFailure_IsRetriedAndCompletesFlushA }; var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); var protocol = CreateProtocolWithBroadcastLogger( transport, ns, @@ -3493,7 +2917,6 @@ public async Task UnexpectedPeerPumpIterationFailure_IsRetriedAndCompletesFlushA ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken)); - timeProvider.Advance(TimeSpan.FromSeconds(1)); await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); @@ -3503,7 +2926,7 @@ public async Task UnexpectedPeerPumpIterationFailure_IsRetriedAndCompletesFlushA await flushTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); var retry = await recoveredRetryScheduled; - Assert.Equal(TimeSpan.FromSeconds(2), retry.DueTime); + Assert.Equal(TimeSpan.FromMilliseconds(200), retry.DueTime); Assert.Equal(1, logger.WarningCount); Assert.DoesNotContain(TimeSpan.MaxValue, timeProvider.TimerDueTimes); @@ -3545,7 +2968,6 @@ public async Task PermanentPeerPumpRecoveryFailure_FaultsFlushAndStopDrainWaiter }; var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(1); var protocol = CreateProtocol( transport, ns, @@ -3557,7 +2979,6 @@ public async Task PermanentPeerPumpRecoveryFailure_FaultsFlushAndStopDrainWaiter ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken)); - timeProvider.Advance(TimeSpan.FromSeconds(1)); await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); @@ -3572,6 +2993,64 @@ public async Task PermanentPeerPumpRecoveryFailure_FaultsFlushAndStopDrainWaiter Assert.Same(flushException, stopException); } + private static T BeforeBroadcastPumpsRun(Func action) + { + var context = new DeferredBroadcastContext(); + var previous = SynchronizationContext.Current; + SynchronizationContext.SetSynchronizationContext(context); + try + { + return action(); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previous); + context.Resume(); + } + } + + private sealed class DeferredBroadcastContext : SynchronizationContext + { + private readonly List<(SendOrPostCallback Callback, object? State)> _pending = []; + private bool _resumed; + + public override void Post(SendOrPostCallback callback, object? state) + { + lock (_pending) + { + if (!_resumed) + { + _pending.Add((callback, state)); + return; + } + } + + ThreadPool.QueueUserWorkItem(static item => item.Callback(item.State), (Callback: callback, State: state), preferLocal: false); + } + + public void Resume() + { + lock (_pending) + { + _resumed = true; + foreach (var item in _pending) + { + ThreadPool.QueueUserWorkItem(static work => work.Callback(work.State), item, preferLocal: false); + } + + _pending.Clear(); + } + } + } + + private static (SiloAddress Local, SiloAddress Peer, FakeTransport Transport, FakeNamespace Namespace) CreatePeerFixture( + int localPort = 11111, int peerPort = 11112) + { + var local = CreateSilo(localPort); + var peer = CreateSilo(peerPort); + return (local, peer, new FakeTransport(local, peer), new FakeNamespace(local)); + } + private static DisseminationProtocol CreateProtocol( FakeTransport transport, FakeNamespace ns, @@ -3597,7 +3076,9 @@ private static DisseminationProtocol CreateProtocol( FakeTransport transport, IReadOnlyList namespaces, Action? configure = null, - TimeProvider? timeProvider = null) + TimeProvider? timeProvider = null, + Microsoft.Extensions.Logging.ILogger? logger = null, + Microsoft.Extensions.Logging.ILogger? broadcastLogger = null) { var options = new DisseminationOptions { Enabled = true }; configure?.Invoke(options); @@ -3612,8 +3093,8 @@ private static DisseminationProtocol CreateProtocol( new TestOptionsMonitor(options), namespaces, timeProvider ?? TimeProvider.System, - NullLogger.Instance, - NullLogger.Instance); + logger ?? NullLogger.Instance, + broadcastLogger ?? NullLogger.Instance); } private static DisseminationBroadcastQueue CreateBroadcastQueue( @@ -3653,24 +3134,8 @@ private static DisseminationProtocol CreateProtocolWithBroadcastLogger( FakeNamespace ns, Action configure, TimeProvider timeProvider, - Microsoft.Extensions.Logging.ILogger broadcastLogger) - { - var options = new DisseminationOptions { Enabled = true }; - configure(options); - var localSiloDetails = new FakeLocalSiloDetails(transport.LocalSilo); - return new DisseminationProtocol( - localSiloDetails, - transport.GrainFactory, - new DisseminationMembership( - transport.MembershipManager, - localSiloDetails, - Options.Create(options)), - new TestOptionsMonitor(options), - [ns], - timeProvider, - NullLogger.Instance, - broadcastLogger); - } + Microsoft.Extensions.Logging.ILogger broadcastLogger) => + CreateProtocol(transport, [ns], configure, timeProvider, broadcastLogger: broadcastLogger); private static DisseminationOverlayOptions CreateOverlayOptions(int fanout) => new() { @@ -3942,14 +3407,18 @@ private sealed class FakeNamespace : IDisseminationNamespace public static readonly DisseminationKey DefaultKey = new("value"); private readonly object _lock = new(); private readonly Dictionary _versions = new(); - private readonly Dictionary> _publishedValues = new(); + private readonly Dictionary _publishedValues = new(); private readonly DisseminationNamespace _name; private int _repairRequestCount; - public FakeNamespace(SiloAddress localSilo, DisseminationNamespace? name = null) + public FakeNamespace( + SiloAddress localSilo, + DisseminationNamespace? name = null, + DisseminationMembershipScope membershipScope = DisseminationMembershipScope.AllMembers) { _ = localSilo; _name = name ?? DefaultName; + MembershipScope = membershipScope; } public Dictionary ApplyCounts { get; } = new(); @@ -3964,7 +3433,7 @@ public FakeNamespace(SiloAddress localSilo, DisseminationNamespace? name = null) public DisseminationNamespaceOptions Options { get; } = new() { Enabled = true }; - public bool ReturnRepairChain { get; set; } + public TimeSpan AggregationPeriod { get; set; } = TimeSpan.FromSeconds(1); public int RepairRequestCount => Volatile.Read(ref _repairRequestCount); @@ -3983,15 +3452,9 @@ public void PublishValue(DisseminationValue value) { lock (_lock) { - if (!_publishedValues.TryGetValue(value.Key, out var values)) - { - values = []; - _publishedValues.Add(value.Key, values); - } - - values[value.ToVersion] = value; - if (!_versions.TryGetValue(value.Key, out var version) || value.ToVersion > version) + if (!_versions.TryGetValue(value.Key, out var version) || value.ToVersion >= version) { + _publishedValues[value.Key] = value; _versions[value.Key] = value.ToVersion; } } @@ -4026,7 +3489,7 @@ public void Clear() .Select(entry => ( entry.Key, entry.Value, - BitConverter.ToInt64(_publishedValues[entry.Key][entry.Value].Payload.Span))) + BitConverter.ToInt64(_publishedValues[entry.Key].Payload.Span))) .ToArray(); } } @@ -4069,101 +3532,23 @@ public IEnumerable Digests public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) { Interlocked.Increment(ref _repairRequestCount); - long version; - List candidates; lock (_lock) { - if (!_versions.TryGetValue(request.Key, out var currentVersion)) + if (!_versions.TryGetValue(request.Key, out var version)) { return DisseminationRepairResult.Unavailable(version: 0); } - version = request.ToVersion ?? currentVersion; - if (version > currentVersion) - { - return DisseminationRepairResult.Unavailable(currentVersion); - } - if (request.FromVersion is { } peerVersion && peerVersion >= version) { return DisseminationRepairResult.Current(version); } - if (ReturnRepairChain - && _publishedValues.TryGetValue(request.Key, out var publishedValues)) - { - candidates = []; - long? expectedVersion = request.FromVersion; - foreach (var value in publishedValues.Values) - { - if (value.ToVersion > version - || expectedVersion is { } expected && value.ToVersion <= expected) - { - continue; - } - - if (expectedVersion is null && value.FromVersion != 0 - || expectedVersion is { } fromVersion - && value.FromVersion != 0 - && value.FromVersion != fromVersion) - { - continue; - } - - candidates.Add(value); - expectedVersion = value.ToVersion; - if (value.ToVersion == version) - { - break; - } - } - } - else - { - candidates = []; - } - - if (candidates.Count == 0 || candidates[^1].ToVersion != version) - { - var value = _publishedValues.TryGetValue(request.Key, out var valuesByVersion) - && valuesByVersion.TryGetValue(version, out var publishedValue) - && publishedValue.FromVersion == 0 - ? publishedValue - : CreateValue(request.Key, version); - candidates = [value]; - } - } - - if (request.MaxItemCount <= 0) - { - return DisseminationRepairResult.InsufficientCapacity(version); - } - - var values = ImmutableArray.CreateBuilder(); - var byteCount = 0; - foreach (var value in candidates) - { - if (values.Count >= request.MaxItemCount - || value.Payload.Length > request.MaxPayloadBytes - || value.Payload.Length > request.MaxBatchBytes - byteCount) - { - break; - } - - values.Add(value); - byteCount += value.Payload.Length; + var value = _publishedValues[request.Key]; + return value.Payload.Length <= request.MaxPayloadBytes && value.Payload.Length <= request.MaxBatchBytes + ? DisseminationRepairResult.Produced(value) + : DisseminationRepairResult.InsufficientCapacity(version); } - - if (values.Count == 0) - { - return DisseminationRepairResult.InsufficientCapacity(version); - } - - var isComplete = values[^1].ToVersion == version; - return DisseminationRepairResult.Produced( - version, - values.ToImmutable(), - isComplete); } public ValueTask ApplyValueAsync( @@ -4192,13 +3577,7 @@ public ValueTask ApplyValueAsync( } _versions[value.Key] = version; - if (!_publishedValues.TryGetValue(value.Key, out var values)) - { - values = []; - _publishedValues.Add(value.Key, values); - } - - values[version] = CreateValue(value.Key, version); + _publishedValues[value.Key] = CreateValue(value.Key, version); ApplyCounts[value.Key] = ApplyCounts.TryGetValue(value.Key, out var count) ? count + 1 : 1; } @@ -4231,19 +3610,17 @@ public async ValueTask Publish( var repair = disseminationNamespace.CreateRepair(new DisseminationRepairRequest( key, _knownVersions.TryGetValue(stream, out var knownVersion) ? knownVersion : null, - version, - maxItemCount: 1024, maxBatchBytes: 1024 * 1024, maxPayloadBytes: disseminationNamespace.Options.MaxPayloadBytes)); - if (repair.Status is not DisseminationRepairStatus.Produced || !repair.IsComplete) + if (repair.Status is not DisseminationRepairStatus.Produced) { return false; } - Values.AddRange(repair.Values); + Values.Add(repair.Value); var result = PublishHandler is null ? Results.Count == 0 || Results.Dequeue() - : await PublishHandler(repair.Values[^1], cancellationToken); + : await PublishHandler(repair.Value, cancellationToken); if (result) { _knownVersions[stream] = repair.Version; @@ -4502,8 +3879,6 @@ public Task RepairPeer(long peerVersion, CancellationToken var repair = _topic.CreateRepair(new DisseminationRepairRequest( FakeNamespace.DefaultKey, peerVersion, - toVersion: null, - maxItemCount: 1, maxBatchBytes: 1024, maxPayloadBytes: 1024)); return Task.FromResult(repair.Status is DisseminationRepairStatus.Produced @@ -4892,139 +4267,71 @@ public void OnError(Exception error) public void Dispose() => _subscription.Dispose(); } - [Fact] - public async Task PublishRoutesDeploymentLoadOnlyToActiveSilos() - { - var topology = CreateScopeRoutingTopology(); - var transport = topology.Transport; - var ns = new ScopedNamespace( - topology.Local, - new DisseminationNamespace("deployment-load-publish"), - DisseminationMembershipScope.ActiveMembers); - var protocol = CreateProtocol( - transport, - [ns], - static options => options.Overlay.FanOutFactor = static _ => 32); - - Assert.True(await PublishScopedValue( - protocol, - ns, - ns.CreateValue("load", sequence: 1), - TestContext.Current.CancellationToken)); - await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - - Assert.Equal( - new[] { topology.ActiveOne, topology.ActiveTwo }, - GetBroadcastPeersForNamespace(transport, ns.Name)); - Assert.DoesNotContain(topology.Joining, GetBroadcastPeersForNamespace(transport, ns.Name)); - Assert.DoesNotContain(topology.ShuttingDown, GetBroadcastPeersForNamespace(transport, ns.Name)); - Assert.DoesNotContain(topology.Stopping, GetBroadcastPeersForNamespace(transport, ns.Name)); - Assert.DoesNotContain(topology.Dead, GetBroadcastPeersForNamespace(transport, ns.Name)); - await protocol.StopAsync(TestContext.Current.CancellationToken); - } - - [Fact] - public async Task PublishRoutesMembershipToJoiningActiveShuttingDownAndStoppingSilos() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task PublishRoutesOnlyToNamespaceMembershipScope(bool activeOnly) { + var scope = activeOnly ? DisseminationMembershipScope.ActiveMembers : DisseminationMembershipScope.AllMembers; var topology = CreateScopeRoutingTopology(); var transport = topology.Transport; - var ns = new ScopedNamespace( - topology.Local, - new DisseminationNamespace("membership-publish"), - DisseminationMembershipScope.AllMembers); + var ns = new FakeNamespace(topology.Local, "scoped-publish", scope); var protocol = CreateProtocol( transport, [ns], static options => options.Overlay.FanOutFactor = static _ => 32); - Assert.True(await PublishScopedValue( - protocol, - ns, - ns.CreateValue("membership", sequence: 1), - TestContext.Current.CancellationToken)); - await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - - Assert.Equal( - new[] - { - topology.ActiveOne, - topology.ActiveTwo, - topology.Joining, - topology.ShuttingDown, - topology.Stopping, - }.OrderBy(static silo => silo), - GetBroadcastPeersForNamespace(transport, ns.Name).OrderBy(static silo => silo)); - Assert.DoesNotContain(topology.Dead, GetBroadcastPeersForNamespace(transport, ns.Name)); - await protocol.StopAsync(TestContext.Current.CancellationToken); - } - - [Fact] - public async Task ReceiveBroadcastForwardsDeploymentLoadOnlyToActiveSilos() - { - var topology = CreateForwardingScopeTopology(); - var ns = new ScopedNamespace( - topology.Local, - new DisseminationNamespace("deployment-load-forward"), - DisseminationMembershipScope.ActiveMembers); - var protocol = CreateProtocol( - topology.Transport, - [ns], - static options => options.Overlay.FanOutFactor = static _ => 2); - - await protocol.ReceiveBroadcast( - CreateScopedBroadcastBatch( - topology.Sender, - ns, - ns.CreateItem(topology.Sender, "load", sequence: 1)), - TestContext.Current.CancellationToken); + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue("load", sequence: 1), + TestContext.Current.CancellationToken)); await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - Assert.Equal( - new[] { topology.LastActive }, - GetBroadcastPeersForNamespace(topology.Transport, ns.Name)); - Assert.DoesNotContain(topology.Joining, GetBroadcastPeersForNamespace(topology.Transport, ns.Name)); - Assert.DoesNotContain(topology.Dead, GetBroadcastPeersForNamespace(topology.Transport, ns.Name)); - Assert.Equal(1, ns.GetVersion("load")); + Assert.Equal(topology.GetPeers(scope), GetBroadcastPeersForNamespace(transport, ns.Name)); + Assert.DoesNotContain(topology.Dead, GetBroadcastPeersForNamespace(transport, ns.Name)); await protocol.StopAsync(TestContext.Current.CancellationToken); } - [Fact] - public async Task ReceiveBroadcastForwardsMembershipToAllEligibleSilos() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ReceiveBroadcastForwardsOnlyToNamespaceMembershipScope(bool activeOnly) { + var scope = activeOnly ? DisseminationMembershipScope.ActiveMembers : DisseminationMembershipScope.AllMembers; var topology = CreateForwardingScopeTopology(); - var ns = new ScopedNamespace( - topology.Local, - new DisseminationNamespace("membership-forward"), - DisseminationMembershipScope.AllMembers); + var ns = new FakeNamespace(topology.Local, "scoped-forward", scope); var protocol = CreateProtocol( topology.Transport, [ns], static options => options.Overlay.FanOutFactor = static _ => 2); await protocol.ReceiveBroadcast( - CreateScopedBroadcastBatch( - topology.Sender, - ns, - ns.CreateItem(topology.Sender, "membership", sequence: 1)), + new DisseminationBroadcastBatch + { + Sender = topology.Sender, + Values = CreateValueGroups(ns.Name, ns.CreateItem(topology.Sender, "load", sequence: 1)), + }, TestContext.Current.CancellationToken); await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - Assert.Equal( - new[] { topology.LastActive, topology.Joining }, - GetBroadcastPeersForNamespace(topology.Transport, ns.Name)); + SiloAddress[] expected = scope == DisseminationMembershipScope.ActiveMembers + ? [topology.LastActive] + : [topology.LastActive, topology.Joining]; + Assert.Equal(expected, GetBroadcastPeersForNamespace(topology.Transport, ns.Name)); Assert.DoesNotContain(topology.Dead, GetBroadcastPeersForNamespace(topology.Transport, ns.Name)); - Assert.Equal(1, ns.GetVersion("membership")); + Assert.Equal(1, ns.GetVersion("load")); await protocol.StopAsync(TestContext.Current.CancellationToken); } - [Fact] - public async Task AntiEntropyRoutesDeploymentLoadOnlyToActiveSilos() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task AntiEntropyRoutesOnlyToNamespaceMembershipScope(bool activeOnly) { + var scope = activeOnly ? DisseminationMembershipScope.ActiveMembers : DisseminationMembershipScope.AllMembers; var topology = CreateScopeRoutingTopology(); - var ns = new ScopedNamespace( - topology.Local, - new DisseminationNamespace("deployment-load-anti-entropy"), - DisseminationMembershipScope.ActiveMembers); + var ns = new FakeNamespace(topology.Local, "scoped-anti-entropy", scope); ns.SetValue("load", version: 1); var protocol = CreateProtocol( topology.Transport, @@ -5033,40 +4340,7 @@ public async Task AntiEntropyRoutesDeploymentLoadOnlyToActiveSilos() await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); - Assert.Equal( - new[] { topology.ActiveOne, topology.ActiveTwo }, - GetAntiEntropyPeersForNamespace(topology.Transport, ns.Name)); - Assert.DoesNotContain(topology.Joining, GetAntiEntropyPeersForNamespace(topology.Transport, ns.Name)); - Assert.DoesNotContain(topology.Dead, GetAntiEntropyPeersForNamespace(topology.Transport, ns.Name)); - await protocol.StopAsync(TestContext.Current.CancellationToken); - } - - [Fact] - public async Task AntiEntropyRoutesMembershipToAllEligibleSilos() - { - var topology = CreateScopeRoutingTopology(); - var ns = new ScopedNamespace( - topology.Local, - new DisseminationNamespace("membership-anti-entropy"), - DisseminationMembershipScope.AllMembers); - ns.SetValue("membership", version: 1); - var protocol = CreateProtocol( - topology.Transport, - [ns], - static options => options.Overlay.AntiEntropyPeerCount = 32); - - await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); - - Assert.Equal( - new[] - { - topology.ActiveOne, - topology.ActiveTwo, - topology.Joining, - topology.ShuttingDown, - topology.Stopping, - }.OrderBy(static silo => silo), - GetAntiEntropyPeersForNamespace(topology.Transport, ns.Name).OrderBy(static silo => silo)); + Assert.Equal(topology.GetPeers(scope), GetAntiEntropyPeersForNamespace(topology.Transport, ns.Name)); Assert.DoesNotContain(topology.Dead, GetAntiEntropyPeersForNamespace(topology.Transport, ns.Name)); await protocol.StopAsync(TestContext.Current.CancellationToken); } @@ -5075,11 +4349,11 @@ public async Task AntiEntropyRoutesMembershipToAllEligibleSilos() public async Task AntiEntropyPreservesCursorFairnessAcrossNamespaceScopes() { var topology = CreateScopeRoutingTopology(); - var load = new ScopedNamespace( + var load = new FakeNamespace( topology.Local, new DisseminationNamespace("deployment-load-fairness"), DisseminationMembershipScope.ActiveMembers); - var membership = new ScopedNamespace( + var membership = new FakeNamespace( topology.Local, new DisseminationNamespace("membership-fairness"), DisseminationMembershipScope.AllMembers); @@ -5147,46 +4421,34 @@ public async Task QueuePrunePreservesPeerStateRequiredByAnotherNamespaceScope() var peer = CreateSilo(35002); var transport = new FakeTransport(local, peer); var timeProvider = new FakeTimeProvider(); - var load = new ScopedNamespace( + var load = new FakeNamespace( local, new DisseminationNamespace("deployment-load-prune"), DisseminationMembershipScope.ActiveMembers); - var membership = new ScopedNamespace( + var membership = new FakeNamespace( local, new DisseminationNamespace("membership-prune"), DisseminationMembershipScope.AllMembers); - load.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); - membership.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); var protocol = CreateProtocol( transport, [load, membership], static options => options.Overlay.FanOutFactor = static _ => 1, timeProvider); - using var schedule = new BroadcastScheduleObserver(); - var loadScheduled = schedule.WaitAsync( - scheduled => scheduled.Peer.Equals(peer), - TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - - Assert.True(await PublishScopedValue( - protocol, - load, - load.CreateValue("load", sequence: 1), - TestContext.Current.CancellationToken)); - await loadScheduled; - Assert.True(await PublishScopedValue( - protocol, - membership, - membership.CreateValue("membership", sequence: 1), - TestContext.Current.CancellationToken)); - - transport.PeerStatuses[peer] = SiloStatus.Joining; - await protocol.ReceiveBroadcast( - new DisseminationBroadcastBatch + var pending = BeforeBroadcastPumpsRun(() => + { + var loadPublication = PublishValue(protocol, load, load.CreateValue("load", 1), TestContext.Current.CancellationToken); + var membershipPublication = PublishValue(protocol, membership, membership.CreateValue("membership", 1), TestContext.Current.CancellationToken); + transport.PeerStatuses[peer] = SiloStatus.Joining; + var receive = protocol.ReceiveBroadcast(new DisseminationBroadcastBatch { Sender = peer, Values = [], - }, - TestContext.Current.CancellationToken); + }, TestContext.Current.CancellationToken); + return (loadPublication, membershipPublication, receive); + }); + Assert.True(await pending.loadPublication); + Assert.True(await pending.membershipPublication); + await pending.receive; await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); var batch = Assert.Single(transport.BroadcastBatches); @@ -5201,10 +4463,7 @@ await protocol.ReceiveBroadcast( [Fact] public async Task PeerQueueRejectsNewDistinctKeyAtNamespacePendingLimit() { - var local = CreateSilo(36001); - var peer = CreateSilo(36002); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(36001, 36002); ns.Options.MaxPendingItemCount = 2; var queue = CreateBroadcastQueue(transport, [ns]); var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -5237,10 +4496,7 @@ public async Task PeerQueueRejectsNewDistinctKeyAtNamespacePendingLimit() [Fact] public async Task PeerQueueAllowsLatestReplacementForAdmittedKeyAtPendingLimit() { - var local = CreateSilo(36011); - var peer = CreateSilo(36012); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(36011, 36012); ns.Options.MaxPendingItemCount = 1; var queue = CreateBroadcastQueue(transport, [ns]); var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -5438,10 +4694,7 @@ void AssertMetric(string name, params (string Name, object Value)[] expectedTags [Fact] public async Task PeerQueueAdmitsNewKeyAfterAcknowledgedDrain() { - var local = CreateSilo(36041); - var peer = CreateSilo(36042); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(36041, 36042); ns.Options.MaxPendingItemCount = 1; var queue = CreateBroadcastQueue(transport, [ns]); ns.SetValue("first", version: 1); @@ -5465,12 +4718,11 @@ public async Task PeerQueueAdmitsNewKeyAfterMembershipPrune() var local = CreateSilo(36051); var peer = CreateSilo(36052); var transport = new FakeTransport(local, peer); - var ns = new ScopedNamespace( + var ns = new FakeNamespace( local, new DisseminationNamespace("prune-recovery"), DisseminationMembershipScope.ActiveMembers); ns.Options.MaxPendingItemCount = 1; - ns.Options.MaxCoalescingDelay = TimeSpan.FromMinutes(1); var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); transport.SendBroadcastHandler = async (target, batch, cancellationToken) => @@ -5488,14 +4740,14 @@ public async Task PeerQueueAdmitsNewKeyAfterMembershipPrune() static options => options.Overlay.FanOutFactor = static _ => 1, new FakeTimeProvider()); - Assert.True(await PublishScopedValue( + Assert.True(await PublishValue( protocol, ns, ns.CreateValue("before-prune", sequence: 1), TestContext.Current.CancellationToken)); await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); using var observer = new AdmissionRejectionObserver(); - Assert.False(await PublishScopedValue( + Assert.False(await PublishValue( protocol, ns, ns.CreateValue("rejected-before-prune", sequence: 1), @@ -5506,7 +4758,7 @@ await protocol.ReceiveBroadcast( new DisseminationBroadcastBatch { Sender = peer, Values = [] }, TestContext.Current.CancellationToken); transport.PeerStatuses[peer] = SiloStatus.Active; - Assert.True(await PublishScopedValue( + Assert.True(await PublishValue( protocol, ns, ns.CreateValue("after-prune", sequence: 1), @@ -5584,15 +4836,15 @@ public async Task PeerQueueHardBoundIsIndependentPerPeerAndNamespace() [Fact] public async Task PeerQueueMaterializesNamespaceRepairAtSendTime() { - var local = CreateSilo(36071); - var peer = CreateSilo(36072); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(36071, 36072); var queue = CreateBroadcastQueue(transport, [ns]); ns.SetValue("latest", version: 1); - queue.Notify(peer, ns, "latest"); - - ns.SetValue("latest", version: 2); + Assert.True(BeforeBroadcastPumpsRun(() => + { + var accepted = queue.Notify(peer, ns, "latest"); + ns.SetValue("latest", version: 2); + return accepted; + })); await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); var value = Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)).Value; @@ -5617,24 +4869,31 @@ public async Task PeerQueueAdvancesKnownVersionOnlyFromResponseAcknowledgment() ? new DisseminationBroadcastResponse() : FakeTransport.CreateAcknowledgment(batch)); }; - var ns = new FakeNamespace(local) { ReturnRepairChain = true }; + var ns = new ProtocolReviewNamespace(local); + var baselines = new List(); + ns.RepairHandler = request => + { + baselines.Add(request.FromVersion); + return ns.Inner.CreateRepair(request); + }; ns.Options.MaxPendingItemCount = 1; - var queue = CreateBroadcastQueue(transport, [ns]); - ns.PublishValue(ns.CreateValue("ack", sequence: 1)); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + ns.Inner.SetValue("ack", 1); queue.Notify(peer, ns, "ack"); await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); - ns.PublishValue(ns.CreateValue("ack", sequence: 2, fromVersion: 1)); + ns.Inner.SetValue("ack", 2); queue.Notify(peer, ns, "ack"); await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); - ns.PublishValue(ns.CreateValue("ack", sequence: 3, fromVersion: 2)); + ns.Inner.SetValue("ack", 3); queue.Notify(peer, ns, "ack"); await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); Assert.Equal(3, sendCount); Assert.Equal(new long[] { 1 }, sentVersions[0]); - Assert.Equal(new long[] { 1, 2 }, sentVersions[1]); + Assert.Equal(new long[] { 2 }, sentVersions[1]); Assert.Equal(new long[] { 3 }, sentVersions[2]); + Assert.Equal(new long?[] { null, null, 2 }, baselines); await queue.StopAsync(TestContext.Current.CancellationToken); } @@ -5753,32 +5012,6 @@ private static ForwardingScopeTopology CreateForwardingScopeTopology() return new(sender, local, lastActive, joining, dead, transport); } - private static ValueTask PublishScopedValue( - DisseminationProtocol protocol, - ScopedNamespace disseminationNamespace, - DisseminationValue value, - CancellationToken cancellationToken) - { - disseminationNamespace.PublishValue(value); - return protocol.Publish( - disseminationNamespace, - value.Key, - value.ToVersion, - cancellationToken); - } - - private static DisseminationBroadcastBatch CreateScopedBroadcastBatch( - SiloAddress sender, - ScopedNamespace disseminationNamespace, - params DisseminationBroadcastValue[] values) => new() - { - Sender = sender, - Values = new() - { - [disseminationNamespace.Name] = [.. values], - }, - }; - private static SiloAddress[] GetBroadcastPeersForNamespace( FakeTransport transport, DisseminationNamespace namespaceName) @@ -5819,7 +5052,13 @@ private readonly record struct ScopeRoutingTopology( SiloAddress ShuttingDown, SiloAddress Stopping, SiloAddress Dead, - FakeTransport Transport); + FakeTransport Transport) + { + public SiloAddress[] GetPeers(DisseminationMembershipScope scope) => + scope == DisseminationMembershipScope.ActiveMembers + ? [ActiveOne, ActiveTwo] + : [ActiveOne, ActiveTwo, Joining, ShuttingDown, Stopping]; + } private readonly record struct ForwardingScopeTopology( SiloAddress Sender, @@ -5829,46 +5068,6 @@ private readonly record struct ForwardingScopeTopology( SiloAddress Dead, FakeTransport Transport); - private sealed class ScopedNamespace( - SiloAddress localSilo, - DisseminationNamespace name, - DisseminationMembershipScope membershipScope) : IDisseminationNamespace - { - private readonly FakeNamespace _inner = new(localSilo, name); - - public DisseminationNamespace Name => _inner.Name; - - public DisseminationMembershipScope MembershipScope { get; } = membershipScope; - - public DisseminationNamespaceOptions Options => _inner.Options; - - public IEnumerable Digests => _inner.Digests; - - public DisseminationValue CreateValue(DisseminationKey key, long sequence, long fromVersion = 0) => - _inner.CreateValue(key, sequence, fromVersion); - - public DisseminationBroadcastValue CreateItem( - SiloAddress originator, - DisseminationKey key, - long sequence, - long fromVersion = 0) => - _inner.CreateItem(originator, key, sequence, fromVersion); - - public void PublishValue(DisseminationValue value) => _inner.PublishValue(value); - - public void SetValue(DisseminationKey key, long version) => _inner.SetValue(key, version); - - public long GetVersion(DisseminationKey key) => _inner.GetVersion(key); - - public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) => - _inner.CreateRepair(request); - - public ValueTask ApplyValueAsync( - DisseminationValue value, - CancellationToken cancellationToken) => - _inner.ApplyValueAsync(value, cancellationToken); - } - [Fact] public async Task QueueAcceptanceDoesNotConfirmPeerNamespaceSupport() { @@ -5879,14 +5078,14 @@ public async Task QueueAcceptanceDoesNotConfirmPeerNamespaceSupport() var ns = new FakeNamespace(local); var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); - Assert.True(await PublishValue( - protocol, - ns, - ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), - TestContext.Current.CancellationToken)); - - Assert.Equal([peer], protocol.GetUnconfirmedPeers(ns)); - Assert.Empty(transport.BroadcastBatches); + var publication = BeforeBroadcastPumpsRun(() => + { + var pending = PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, 1), TestContext.Current.CancellationToken); + Assert.Equal([peer], protocol.GetUnconfirmedPeers(ns)); + Assert.Empty(transport.BroadcastBatches); + return pending; + }); + Assert.True(await publication); await protocol.StopAsync(TestContext.Current.CancellationToken); } @@ -5912,10 +5111,7 @@ public async Task InboundNamespaceTrafficConfirmsPeerSupport() [Fact] public async Task ResponseAcknowledgmentConfirmsPeerNamespaceSupport() { - var local = CreateSilo(31005); - var peer = CreateSilo(31006); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, _, transport, ns) = CreatePeerFixture(31005, 31006); var protocol = CreateProtocol(transport, ns); Assert.True(await PublishValue( @@ -5932,10 +5128,7 @@ public async Task ResponseAcknowledgmentConfirmsPeerNamespaceSupport() [Fact] public async Task UnsupportedNamespaceResponseRevokesPeerSupport() { - var local = CreateSilo(31007); - var peer = CreateSilo(31008); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(31007, 31008); var protocol = CreateProtocol(transport, ns); await protocol.ReceiveAntiEntropy( new DisseminationAntiEntropyRequest @@ -6037,7 +5230,7 @@ public async Task GetUnconfirmedPeersReturnsOnlyActiveUnconfirmedPeersForNamespa var transport = new FakeTransport(local, confirmed, unconfirmed, joining, dead); transport.PeerStatuses[joining] = SiloStatus.Joining; transport.PeerStatuses[dead] = SiloStatus.Dead; - var ns = new ScopedNamespace(local, "active-only", DisseminationMembershipScope.ActiveMembers); + var ns = new FakeNamespace(local, "active-only", DisseminationMembershipScope.ActiveMembers); var protocol = CreateProtocol(transport, [ns]); await protocol.ReceiveAntiEntropy( new DisseminationAntiEntropyRequest @@ -6056,10 +5249,7 @@ await protocol.ReceiveAntiEntropy( [Fact] public async Task DisseminationPushBroadcastWireContractPreservesAcknowledgmentsAndUnsupportedNamespaces() { - var local = CreateSilo(31018); - var peer = CreateSilo(31019); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(31018, 31019); var protocol = CreateProtocol(transport, ns); var unsupported = new DisseminationNamespace("unsupported"); var receivedNamespaces = new ConcurrentBag(); @@ -6112,42 +5302,31 @@ public async Task DisseminationPushBroadcastWireContractPreservesAcknowledgments Assert.Equal([ns.Name], receivedNamespaces); } - [Fact] - public async Task DeploymentLoadPublisherDirectSendsToAllActivePeersWhileTreeSupportIsIncomplete() - { - var harness = CreateDeploymentLoadPublisherHarness(); - harness.Dissemination.UnconfirmedPeers = [harness.ActiveTwo]; - - await harness.PublishStatistics(TestContext.Current.CancellationToken); - - AssertDirectRecipients(harness, harness.ActiveOne, harness.ActiveTwo); - Assert.Single(harness.Dissemination.PublishCalls); - Assert.Single(harness.Dissemination.QueryCalls); - } - - [Fact] - public async Task DeploymentLoadPublisherDoesNotDirectSendToConfirmedPeersAfterQueueAcceptance() + [Theory] + [InlineData("partially-confirmed")] + [InlineData("confirmed")] + [InlineData("inactive")] + [InlineData("unconfirmed")] + public async Task DeploymentLoadPublisherDirectFallbackRequiresUnconfirmedActivePeers(string support) { var harness = CreateDeploymentLoadPublisherHarness(); - harness.Dissemination.UnconfirmedPeers = []; + SiloAddress[] unconfirmed = support switch + { + "partially-confirmed" => [harness.ActiveTwo], + "confirmed" => [], + "inactive" => [harness.Joining], + _ => [harness.ActiveOne, harness.ActiveTwo], + }; + harness.Dissemination.UnconfirmedPeers = unconfirmed; await harness.PublishStatistics(TestContext.Current.CancellationToken); - AssertDirectRecipients(harness); + SiloAddress[] expected = support is "confirmed" or "inactive" ? [] : [harness.ActiveOne, harness.ActiveTwo]; + AssertDirectRecipients(harness, expected); Assert.Single(harness.Dissemination.PublishCalls); Assert.Single(harness.Dissemination.QueryCalls); - } - - [Fact] - public async Task DeploymentLoadPublisherDoesNotDirectSendToNonActiveUnconfirmedPeers() - { - var harness = CreateDeploymentLoadPublisherHarness(); - harness.Dissemination.UnconfirmedPeers = [harness.Joining]; - - await harness.PublishStatistics(TestContext.Current.CancellationToken); - - AssertDirectRecipients(harness); - Assert.Single(harness.Dissemination.QueryCalls); + Assert.True(Assert.Single(harness.Dissemination.PublishResults)); + Assert.Equal(unconfirmed, harness.Dissemination.UnconfirmedPeers.OrderBy(static peer => peer)); } [Fact] @@ -6162,11 +5341,15 @@ public async Task DeploymentLoadPublisherFallsBackToAllActivePeersWhenDisseminat Assert.Empty(harness.Dissemination.QueryCalls); } - [Fact] - public async Task DeploymentLoadPublisherFallsBackToAllActivePeersWhenDisseminationReturnsFalse() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DeploymentLoadPublisherFallsBackToAllActivePeersWhenDisseminationFails(bool throws) { var harness = CreateDeploymentLoadPublisherHarness(); - harness.Dissemination.PublishHandler = (_, _, _, _) => ValueTask.FromResult(false); + harness.Dissemination.PublishHandler = (_, _, _, _) => throws + ? ValueTask.FromException(new InvalidOperationException("test failure")) + : ValueTask.FromResult(false); await harness.PublishStatistics(TestContext.Current.CancellationToken); @@ -6227,35 +5410,6 @@ await Assert.ThrowsAnyAsync( Assert.Empty(harness.Dissemination.QueryCalls); } - [Fact] - public async Task DeploymentLoadPublisherFallsBackToAllActivePeersWhenDisseminationThrows() - { - var harness = CreateDeploymentLoadPublisherHarness(); - harness.Dissemination.PublishHandler = (_, _, _, _) => - ValueTask.FromException(new InvalidOperationException("test failure")); - - await harness.PublishStatistics(TestContext.Current.CancellationToken); - - AssertDirectRecipients(harness, harness.ActiveOne, harness.ActiveTwo); - Assert.Single(harness.Dissemination.PublishCalls); - Assert.Empty(harness.Dissemination.QueryCalls); - } - - [Fact] - public async Task DeploymentLoadPublisherQueueAcceptanceAloneNeverSuppressesLegacyPeerDelivery() - { - var harness = CreateDeploymentLoadPublisherHarness(); - harness.Dissemination.UnconfirmedPeers = [harness.ActiveOne, harness.ActiveTwo]; - - await harness.PublishStatistics(TestContext.Current.CancellationToken); - - AssertDirectRecipients(harness, harness.ActiveOne, harness.ActiveTwo); - Assert.True(Assert.Single(harness.Dissemination.PublishResults)); - Assert.Equal( - [harness.ActiveOne, harness.ActiveTwo], - harness.Dissemination.UnconfirmedPeers.OrderBy(static peer => peer)); - } - private static DeploymentLoadPublisherHarness CreateDeploymentLoadPublisherHarness( bool namespaceEnabled = true, FakeTimeProvider? timeProvider = null) @@ -6452,6 +5606,8 @@ private sealed class Phase4FakeSiloStatusOracle : ISiloStatusOracle { private readonly ConcurrentDictionary _statuses = new(); + public Func? GetStatusHandler { get; set; } + public SiloStatus CurrentStatus => SiloStatus.Active; public string SiloName => "local"; @@ -6463,9 +5619,12 @@ private sealed class Phase4FakeSiloStatusOracle : ISiloStatusOracle public SiloAddress[] GetActiveSilos() => [.. _statuses.Where(static entry => entry.Value == SiloStatus.Active).Select(static entry => entry.Key)]; - public SiloStatus GetApproximateSiloStatus(SiloAddress siloAddress) => + public SiloStatus GetStoredStatus(SiloAddress siloAddress) => _statuses.TryGetValue(siloAddress, out var status) ? status : SiloStatus.None; + public SiloStatus GetApproximateSiloStatus(SiloAddress siloAddress) => + GetStatusHandler?.Invoke(siloAddress) ?? GetStoredStatus(siloAddress); + public Dictionary GetApproximateSiloStatuses(bool onlyActive = false) => _statuses .Where(entry => !onlyActive || entry.Value == SiloStatus.Active) @@ -6481,7 +5640,7 @@ public bool TryGetSiloName( public bool IsFunctionalDirectory(SiloAddress siloAddress) => true; - public bool IsDeadSilo(SiloAddress silo) => GetApproximateSiloStatus(silo) == SiloStatus.Dead; + public bool IsDeadSilo(SiloAddress silo) => GetStoredStatus(silo) == SiloStatus.Dead; public bool SubscribeToSiloStatusEvents(ISiloStatusListener observer) => true; @@ -6505,89 +5664,39 @@ public void OnActive(IActivationWorkingSetMember member) { } - public void OnDeactivating(IActivationWorkingSetMember member) - { - } - - public void OnDeactivated(IActivationWorkingSetMember member) - { - } - } - - private sealed class Phase4FakeDeploymentLoadPublisherTarget : IDeploymentLoadPublisher - { - public List<(SiloAddress Source, SiloRuntimeStatistics Statistics)> Updates { get; } = []; - - public Task UpdateRuntimeStatistics( - SiloAddress siloAddress, - SiloRuntimeStatistics siloStats, - CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - Updates.Add((siloAddress, siloStats)); - return Task.CompletedTask; - } - } - - [Fact] - public async Task LoadNotificationAppliesQueuedUpdatesInOrderLatestWins() - { - var harness = CreatePhase5DeploymentLoadPublisherHarness(); - var notifications = new List(); - var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( - onUpdate: (silo, statistics) => - { - notifications.Add(Phase5Notification.Update(silo, statistics)); - if (silo.Equals(harness.Local)) - { - callbackEntered.TrySetResult(); - releaseCallback.Task.GetAwaiter().GetResult(); - } - })); - - var firstUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( - harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); - Task update10; - Task update20; - Task update30; - try + public void OnDeactivating(IActivationWorkingSetMember member) { - await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - update10 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( - harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken); - update20 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( - harness.ActiveOne, CreatePhase5Statistics(20), TestContext.Current.CancellationToken); - update30 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( - harness.ActiveOne, CreatePhase5Statistics(30), TestContext.Current.CancellationToken); - Assert.False(update10.IsCompleted); - Assert.False(update20.IsCompleted); - Assert.False(update30.IsCompleted); - Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); } - finally + + public void OnDeactivated(IActivationWorkingSetMember member) { - releaseCallback.TrySetResult(); } + } - Assert.All(await Task.WhenAll(firstUpdate, update10, update20, update30), - static result => Assert.Equal(DisseminationApplyResult.Applied, result)); - Assert.Equal(30, harness.Publisher.PeriodicStatistics[harness.ActiveOne].ActivationCount); - Assert.Equal( - [ - Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1)), - Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(10)), - Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(20)), - Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(30)), - ], - notifications); + private sealed class Phase4FakeDeploymentLoadPublisherTarget : IDeploymentLoadPublisher + { + public List<(SiloAddress Source, SiloRuntimeStatistics Statistics)> Updates { get; } = []; + + public Task UpdateRuntimeStatistics( + SiloAddress siloAddress, + SiloRuntimeStatistics siloStats, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Updates.Add((siloAddress, siloStats)); + return Task.CompletedTask; + } } - [Fact] - public async Task LoadNotificationPreservesCrossSiloPublicationOrder() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task LoadNotificationAppliesQueuedUpdatesInPublicationOrder(bool interleaveSilos) { var harness = CreatePhase5DeploymentLoadPublisherHarness(); + (SiloAddress Silo, int Version)[] publications = interleaveSilos + ? [(harness.ActiveOne, 10), (harness.ActiveTwo, 20), (harness.ActiveOne, 30), (harness.ActiveTwo, 40)] + : [(harness.ActiveOne, 10), (harness.ActiveOne, 20), (harness.ActiveOne, 30)]; var notifications = new List(); var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -6604,40 +5713,34 @@ public async Task LoadNotificationPreservesCrossSiloPublicationOrder() var firstUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); - Task update10; - Task update20; - Task update30; - Task update40; + Task[] updates; try { await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - update10 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( - harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken); - update20 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( - harness.ActiveTwo, CreatePhase5Statistics(20), TestContext.Current.CancellationToken); - update30 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( - harness.ActiveOne, CreatePhase5Statistics(30), TestContext.Current.CancellationToken); - update40 = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( - harness.ActiveTwo, CreatePhase5Statistics(40), TestContext.Current.CancellationToken); + updates = publications.Select(publication => harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + publication.Silo, CreatePhase5Statistics(publication.Version), TestContext.Current.CancellationToken)).ToArray(); + Assert.All(updates, static update => Assert.False(update.IsCompleted)); + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); } finally { releaseCallback.TrySetResult(); } - Assert.All(await Task.WhenAll(firstUpdate, update10, update20, update30, update40), + Assert.All(await Task.WhenAll(updates.Prepend(firstUpdate)), static result => Assert.Equal(DisseminationApplyResult.Applied, result)); + Assert.Equal(30, harness.Publisher.PeriodicStatistics[harness.ActiveOne].ActivationCount); Assert.Equal( [ Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1)), - Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(10)), - Phase5Notification.Update(harness.ActiveTwo, CreatePhase5Statistics(20)), - Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(30)), - Phase5Notification.Update(harness.ActiveTwo, CreatePhase5Statistics(40)), + .. publications.Select(static publication => + Phase5Notification.Update(publication.Silo, CreatePhase5Statistics(publication.Version))), ], notifications); - Assert.Equal(30, harness.Publisher.PeriodicStatistics[harness.ActiveOne].ActivationCount); - Assert.Equal(40, harness.Publisher.PeriodicStatistics[harness.ActiveTwo].ActivationCount); + if (interleaveSilos) + { + Assert.Equal(40, harness.Publisher.PeriodicStatistics[harness.ActiveTwo].ActivationCount); + } } [Fact] @@ -7061,7 +6164,7 @@ private static Phase5DeploymentLoadPublisherHarness CreatePhase5DeploymentLoadPu var local = CreateSilo(32101); var activeOne = CreateSilo(32102); var activeTwo = CreateSilo(32103); - var statusOracle = new Phase5FakeSiloStatusOracle(); + var statusOracle = new Phase4FakeSiloStatusOracle(); statusOracle.SetStatus(local, SiloStatus.Active); statusOracle.SetStatus(activeOne, SiloStatus.Active); statusOracle.SetStatus(activeTwo, SiloStatus.Active); @@ -7092,7 +6195,7 @@ private static Phase5DeploymentLoadPublisherHarness CreatePhase5DeploymentLoadPu private sealed record Phase5DeploymentLoadPublisherHarness( DeploymentLoadPublisher Publisher, - Phase5FakeSiloStatusOracle StatusOracle, + Phase4FakeSiloStatusOracle StatusOracle, SiloAddress Local, SiloAddress ActiveOne, SiloAddress ActiveTwo); @@ -7115,144 +6218,90 @@ public void SiloStatisticsChangeNotification(SiloAddress updatedSilo, SiloRuntim public void RemoveSilo(SiloAddress removedSilo) => onRemove?.Invoke(removedSilo); } - private sealed class Phase5FakeSiloStatusOracle : ISiloStatusOracle - { - private readonly ConcurrentDictionary _statuses = new(); - - public Func? GetStatusHandler { get; set; } - - public SiloStatus CurrentStatus => SiloStatus.Active; - - public string SiloName => "local"; - - public SiloAddress SiloAddress => _statuses.Keys.OrderBy(static silo => silo).First(); - - public void SetStatus(SiloAddress silo, SiloStatus status) => _statuses[silo] = status; - - public SiloStatus GetStoredStatus(SiloAddress silo) => - _statuses.TryGetValue(silo, out var status) ? status : SiloStatus.None; - - public SiloAddress[] GetActiveSilos() => - [.. _statuses.Where(static entry => entry.Value == SiloStatus.Active).Select(static entry => entry.Key)]; - - public SiloStatus GetApproximateSiloStatus(SiloAddress siloAddress) => - GetStatusHandler?.Invoke(siloAddress) ?? GetStoredStatus(siloAddress); - - public Dictionary GetApproximateSiloStatuses(bool onlyActive = false) => - _statuses - .Where(entry => !onlyActive || entry.Value == SiloStatus.Active) - .ToDictionary(static entry => entry.Key, static entry => entry.Value); - - public bool TryGetSiloName( - SiloAddress siloAddress, - [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out string? siloName) - { - siloName = siloAddress.ToParsableString(); - return true; - } - - public bool IsFunctionalDirectory(SiloAddress siloAddress) => true; - - public bool IsDeadSilo(SiloAddress silo) => GetStoredStatus(silo) == SiloStatus.Dead; - - public bool SubscribeToSiloStatusEvents(ISiloStatusListener observer) => true; - - public bool UnSubscribeFromSiloStatusEvents(ISiloStatusListener observer) => true; - } - - [Fact] - public async Task ReceiveBroadcastContinuesAfterNamespaceApplyThrows() + [Theory] + [InlineData(false, false, 1, 2)] + [InlineData(false, true, 1, 2)] + [InlineData(true, false, 1, 3)] + [InlineData(false, false, 4, 5)] + [InlineData(false, false, 6, 7)] + public async Task ReceiveBroadcastRejectsAndDiagnosesApplyFailureWithoutBlockingLaterValues( + bool unrelatedCancellation, bool forward, long failedVersion, long validVersion) { var sender = CreateSilo(33001); var local = CreateSilo(33002); - var transport = new FakeTransport(local, sender); + var child = CreateSilo(33003); + var transport = new FakeTransport(local, forward ? [sender, child] : [sender]); var failedKey = new DisseminationKey("throws"); var validKey = new DisseminationKey("valid"); + using var cancellation = new CancellationTokenSource(); + Exception failure = unrelatedCancellation + ? new OperationCanceledException("unrelated", cancellation.Token) + : new InvalidOperationException("phase 6 apply failure"); var ns = new Phase6ThrowingNamespace(local, failedKey) { - Failure = static (_, _) => throw new InvalidOperationException("phase 6 apply failure"), - }; - var protocol = CreatePhase6Protocol(transport, ns); - - var response = await protocol.ReceiveBroadcast( - CreateBroadcastBatch( - sender, - ns.CreateItem(sender, failedKey, sequence: 1), - ns.CreateItem(sender, validKey, sequence: 2)), - TestContext.Current.CancellationToken); - - Assert.Equal([failedKey, validKey], ns.ApplyAttempts); - Assert.Equal(0, ns.GetVersion(failedKey)); - Assert.Equal(2, ns.GetVersion(validKey)); - var acknowledgments = response.Acknowledgments[ns.Name].ToDictionary(static entry => entry.Key); - Assert.Equal(0, acknowledgments[failedKey].Version); - Assert.Equal(2, acknowledgments[validKey].Version); - Assert.Empty(response.UnsupportedNamespaces); - } - - [Fact] - public async Task ReceiveBroadcastForwardsRemainingValuesAfterNamespaceApplyThrows() - { - var sender = CreateSilo(33003); - var local = CreateSilo(33004); - var child = CreateSilo(33005); - var transport = new FakeTransport(local, sender, child); - var failedKey = new DisseminationKey("throws"); - var validKey = new DisseminationKey("forwarded"); - var ns = new Phase6ThrowingNamespace(local, failedKey) - { - Failure = static (_, _) => throw new InvalidOperationException("phase 6 apply failure"), + Failure = (_, _) => throw failure, }; + var logger = new Phase6ProtocolLogger(); var protocol = CreatePhase6Protocol( - transport, - ns, + transport, ns, logger, configure: static options => options.Overlay.FanOutFactor = static _ => 1); - - await protocol.ReceiveBroadcast( - CreateBroadcastBatch( - sender, - ns.CreateItem(sender, failedKey, sequence: 1), - ns.CreateItem(sender, validKey, sequence: 2)), - TestContext.Current.CancellationToken); - await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - - Assert.Equal([failedKey, validKey], ns.ApplyAttempts); - var forwarded = Assert.Single(transport.BroadcastBatches); - Assert.Equal(child, forwarded.Peer); - var forwardedValue = Assert.Single(GetBroadcastValues(forwarded.Batch)); - Assert.Equal(validKey, forwardedValue.Value.Key); - Assert.Equal(2, forwardedValue.Value.ToVersion); - } - - [Fact] - public async Task ReceiveBroadcastConvertsUnrelatedOperationCanceledExceptionToRejected() - { - var sender = CreateSilo(33006); - var local = CreateSilo(33007); - var transport = new FakeTransport(local, sender); - var failedKey = new DisseminationKey("unrelated-cancellation"); - var validKey = new DisseminationKey("valid"); - using var unrelatedCancellation = new CancellationTokenSource(); - var ns = new Phase6ThrowingNamespace(local, failedKey) + using var observer = new Phase6ApplyObserver(ns.Name, failedKey, local, sender); + try { - Failure = (_, _) => throw new OperationCanceledException("unrelated", unrelatedCancellation.Token), - }; - var protocol = CreatePhase6Protocol(transport, ns); - - var response = await protocol.ReceiveBroadcast( - CreateBroadcastBatch( - sender, - ns.CreateItem(sender, failedKey, sequence: 1), - ns.CreateItem(sender, validKey, sequence: 3)), - TestContext.Current.CancellationToken); + var response = await protocol.ReceiveBroadcast( + CreateBroadcastBatch( + sender, + ns.CreateItem(sender, failedKey, failedVersion), + ns.CreateItem(sender, validKey, validVersion)), + TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - Assert.False(unrelatedCancellation.IsCancellationRequested); - Assert.Equal([failedKey, validKey], ns.ApplyAttempts); - Assert.Equal(0, ns.GetVersion(failedKey)); - Assert.Equal(3, ns.GetVersion(validKey)); - var acknowledgments = response.Acknowledgments[ns.Name].ToDictionary(static entry => entry.Key); - Assert.Equal(0, acknowledgments[failedKey].Version); - Assert.Equal(3, acknowledgments[validKey].Version); + Assert.False(cancellation.IsCancellationRequested); + Assert.Equal([failedKey, validKey], ns.ApplyAttempts); + Assert.Equal(0, ns.GetVersion(failedKey)); + Assert.Equal(validVersion, ns.GetVersion(validKey)); + var acknowledgments = response.Acknowledgments[ns.Name].ToDictionary(static entry => entry.Key); + Assert.Equal(0, acknowledgments[failedKey].Version); + Assert.Equal(validVersion, acknowledgments[validKey].Version); + Assert.Empty(response.UnsupportedNamespaces); + + var rejection = Assert.Single(observer.Events); + Assert.Equal(ns.Name, rejection.Namespace); + Assert.Equal(local, rejection.LocalSilo); + Assert.Equal(sender, rejection.Peer); + Assert.Equal(failedKey, rejection.Key); + Assert.Equal(0, rejection.FromVersion); + Assert.Equal(failedVersion, rejection.ToVersion); + Assert.Equal(nameof(DisseminationApplyResult.Rejected), rejection.Result); + Assert.Equal(sizeof(long), rejection.PayloadBytes); + + var entry = Assert.Single(logger.Entries); + Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Debug, entry.Level); + Assert.Equal(1360205587, entry.EventId.Id); + Assert.Equal("LogDebugDisseminationValueApplyFailed", entry.EventId.Name); + Assert.Same(failure, entry.Exception); + Assert.Equal(sender, entry.State["Sender"]); + Assert.Equal(ns.Name, entry.State["Namespace"]); + Assert.Equal(failedKey, entry.State["Key"]); + Assert.Equal(failedVersion, entry.State["Version"]); + + if (forward) + { + var forwarded = Assert.Single(transport.BroadcastBatches); + Assert.Equal(child, forwarded.Peer); + var forwardedValue = Assert.Single(GetBroadcastValues(forwarded.Batch)); + Assert.Equal(validKey, forwardedValue.Value.Key); + Assert.Equal(validVersion, forwardedValue.Value.ToVersion); + } + else + { + Assert.Empty(transport.BroadcastBatches); + } + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } } [Fact] @@ -7292,107 +6341,6 @@ public async Task ReceiveBroadcastPropagatesCallerCancellation() Assert.Empty(transport.BroadcastBatches); } - [Fact] - public async Task ReceiveBroadcastStopsBeforeNextValueWhenCallerIsCanceledAfterApply() - { - var sender = CreateSilo(33014); - var local = CreateSilo(33015); - var transport = new FakeTransport(local, sender); - var cancelingKey = new DisseminationKey("canceling-apply"); - var laterKey = new DisseminationKey("must-not-apply"); - using var cancellation = new CancellationTokenSource(); - var ns = new Phase6ThrowingNamespace(local, cancelingKey) - { - Failure = (_, _) => - { - cancellation.Cancel(); - return ValueTask.FromResult(DisseminationApplyResult.Applied); - }, - }; - var protocol = CreatePhase6Protocol(transport, ns); - - var exception = await Assert.ThrowsAnyAsync(() => protocol.ReceiveBroadcast( - CreateBroadcastBatch( - sender, - ns.CreateItem(sender, cancelingKey, sequence: 1), - ns.CreateItem(sender, laterKey, sequence: 2)), - cancellation.Token)); - - Assert.Equal(cancellation.Token, exception.CancellationToken); - Assert.Equal([cancelingKey], ns.ApplyAttempts); - Assert.Equal(0, ns.GetVersion(laterKey)); - Assert.Empty(transport.BroadcastBatches); - } - - [Fact] - public async Task ReceiveBroadcastEmitsRejectedDiagnosticWhenNamespaceApplyThrows() - { - var sender = CreateSilo(33010); - var local = CreateSilo(33011); - var transport = new FakeTransport(local, sender); - var failedKey = new DisseminationKey("diagnostic"); - var validKey = new DisseminationKey("valid"); - var ns = new Phase6ThrowingNamespace(local, failedKey) - { - Failure = static (_, _) => throw new InvalidOperationException("phase 6 apply failure"), - }; - var protocol = CreatePhase6Protocol(transport, ns); - using var observer = new Phase6ApplyObserver(ns.Name, failedKey, local, sender); - - await protocol.ReceiveBroadcast( - CreateBroadcastBatch( - sender, - ns.CreateItem(sender, failedKey, sequence: 4), - ns.CreateItem(sender, validKey, sequence: 5)), - TestContext.Current.CancellationToken); - - var rejection = Assert.Single(observer.Events); - Assert.Equal(ns.Name, rejection.Namespace); - Assert.Equal(local, rejection.LocalSilo); - Assert.Equal(sender, rejection.Peer); - Assert.Equal(failedKey, rejection.Key); - Assert.Equal(0, rejection.FromVersion); - Assert.Equal(4, rejection.ToVersion); - Assert.Equal(nameof(DisseminationApplyResult.Rejected), rejection.Result); - Assert.Equal(sizeof(long), rejection.PayloadBytes); - Assert.Equal(5, ns.GetVersion(validKey)); - } - - [Fact] - public async Task ReceiveBroadcastLogsNamespaceApplyFailure() - { - var sender = CreateSilo(33012); - var local = CreateSilo(33013); - var transport = new FakeTransport(local, sender); - var failedKey = new DisseminationKey("logged"); - var validKey = new DisseminationKey("valid"); - var ns = new Phase6ThrowingNamespace(local, failedKey) - { - Failure = static (_, _) => throw new InvalidOperationException("phase 6 apply failure"), - }; - var logger = new Phase6ProtocolLogger(); - var protocol = CreatePhase6Protocol(transport, ns, logger); - - await protocol.ReceiveBroadcast( - CreateBroadcastBatch( - sender, - ns.CreateItem(sender, failedKey, sequence: 6), - ns.CreateItem(sender, validKey, sequence: 7)), - TestContext.Current.CancellationToken); - - var entry = Assert.Single(logger.Entries); - Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Debug, entry.Level); - Assert.Equal(1360205587, entry.EventId.Id); - Assert.Equal("LogDebugDisseminationValueApplyFailed", entry.EventId.Name); - var exception = Assert.IsType(entry.Exception); - Assert.Equal("phase 6 apply failure", exception.Message); - Assert.Equal(sender, entry.State["Sender"]); - Assert.Equal(ns.Name, entry.State["Namespace"]); - Assert.Equal(failedKey, entry.State["Key"]); - Assert.Equal(6L, entry.State["Version"]); - Assert.Equal(7, ns.GetVersion(validKey)); - } - [Fact] public async Task ReceiveBroadcastContinuesWhenApplyDiagnosticObserverThrows() { @@ -7427,8 +6375,10 @@ public async Task ReceiveBroadcastContinuesWhenApplyDiagnosticObserverThrows() && Equals(entry.State["Namespace"], ns.Name)); } - [Fact] - public async Task AntiEntropyStopsBeforeNextValueWhenCallerIsCanceledAfterApply() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ReceivingStopsBeforeNextValueWhenCallerIsCanceledAfterApply(bool antiEntropy) { var sender = CreateSilo(39001); var local = CreateSilo(39002); @@ -7446,30 +6396,29 @@ public async Task AntiEntropyStopsBeforeNextValueWhenCallerIsCanceledAfterApply( return ValueTask.FromResult(DisseminationApplyResult.Applied); }, }; + var values = CreateValueGroups( + ns.CreateItem(sender, cancelingKey, sequence: 1), + ns.CreateItem(sender, laterKey, sequence: 2)); transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse { Sender = sender, - Values = new() - { - [ns.Name] = - [ - ns.CreateItem(sender, cancelingKey, sequence: 1), - ns.CreateItem(sender, laterKey, sequence: 2), - ], - }, + Values = values, }); var protocol = CreateProtocol(transport, [ns]); try { var exception = await Assert.ThrowsAnyAsync( - () => protocol.RunAntiEntropyRound(cancellation.Token)); + () => antiEntropy + ? protocol.RunAntiEntropyRound(cancellation.Token) + : protocol.ReceiveBroadcast(new() { Sender = sender, Values = values }, cancellation.Token)); Assert.True(cancellation.IsCancellationRequested); Assert.True(applyCancellation.IsCancellationRequested); Assert.Equal(cancellation.Token, exception.CancellationToken); Assert.Equal([cancelingKey], ns.ApplyAttempts); Assert.Equal(0, ns.GetVersion(laterKey)); + Assert.Empty(transport.BroadcastBatches); } finally { @@ -7577,10 +6526,7 @@ MembershipTableSnapshot CreateSnapshot(int aliveSeconds) => [Fact] public async Task PeerQueueAdmitsNewKeyAfterOversizedRepair() { - var local = CreateSilo(39003); - var peer = CreateSilo(39004); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(39003, 39004); ns.Options.MaxPendingItemCount = 1; ns.Options.MaxPayloadBytes = sizeof(long); ns.PublishValue(new DisseminationValue("oversized", 0, 1, new byte[sizeof(long) + 1])); @@ -7610,20 +6556,21 @@ public async Task PeerQueueAdmitsNewKeyAfterOversizedRepair() [Fact] public async Task DisablingDisseminationStopsPreviouslyQueuedBroadcasts() { - var local = CreateSilo(39025); - var peer = CreateSilo(39026); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, _, transport, ns) = CreatePeerFixture(39025, 39026); DisseminationOptions? options = null; var protocol = CreateProtocol( transport, ns, value => options = value, timeProvider: new FakeTimeProvider()); try { - Assert.True(await PublishValue( - protocol, ns, ns.CreateValue("value", 1), TestContext.Current.CancellationToken)); - Assert.NotNull(options); - options.Enabled = false; + var publication = BeforeBroadcastPumpsRun(() => + { + var pending = PublishValue(protocol, ns, ns.CreateValue("value", 1), TestContext.Current.CancellationToken); + Assert.NotNull(options); + options.Enabled = false; + return pending; + }); + Assert.True(await publication); await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); @@ -7638,10 +6585,7 @@ public async Task DisablingDisseminationStopsPreviouslyQueuedBroadcasts() [Fact] public async Task PublishReturnsFalseWhenPeerQueueRejectsNewKey() { - var local = CreateSilo(39005); - var peer = CreateSilo(39006); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, _, transport, ns) = CreatePeerFixture(39005, 39006); ns.Options.MaxPendingItemCount = 1; var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -7673,12 +6617,8 @@ public async Task PublishReturnsFalseWhenPeerQueueRejectsNewKey() [Fact] public async Task BroadcastDeadlineCompletesFlushBeforeRemoteCancellationAcknowledgment() { - var local = CreateSilo(39011); - var peer = CreateSilo(39012); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, peer, transport, ns) = CreatePeerFixture(39011, 39012); ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(10); ns.SetValue("value", version: 1); var clock = new FakeTimeProvider(); var response = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -7718,12 +6658,8 @@ public async Task BroadcastDeadlineCompletesFlushBeforeRemoteCancellationAcknowl [InlineData(32, false)] public async Task TimedOutBroadcastReleasesLocalSlotBeforeRpcCompletes(int maxConcurrentSends, bool lateFault) { - var local = CreateSilo(39013); - var peer = CreateSilo(39014); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (local, peer, transport, ns) = CreatePeerFixture(39013, 39014); ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); - ns.Options.MaxCoalescingDelay = TimeSpan.FromSeconds(10); ns.SetValue("value", version: 1); var clock = new FakeTimeProvider(); var response = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -7786,10 +6722,7 @@ await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken) [Fact] public async Task PublishReturnsFalseAfterBroadcastQueueStops() { - var local = CreateSilo(39007); - var peer = CreateSilo(39008); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (_, _, transport, ns) = CreatePeerFixture(39007, 39008); var protocol = CreateProtocol(transport, ns); await protocol.StopAsync(TestContext.Current.CancellationToken); @@ -7802,10 +6735,7 @@ public async Task PublishReturnsFalseAfterBroadcastQueueStops() [InlineData(true)] public async Task BroadcastNotificationDiagnosticsRunOutsideQueueLock(bool batch) { - var local = CreateSilo(39009); - var peer = CreateSilo(39010); - var transport = new FakeTransport(local, peer); - var ns = new FakeNamespace(local); + var (local, peer, transport, ns) = CreatePeerFixture(39009, 39010); ns.SetValue("value", version: 1); var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -7819,7 +6749,7 @@ public async Task BroadcastNotificationDiagnosticsRunOutsideQueueLock(bool batch static name => name == DisseminationEvents.BroadcastScheduledEventName); var notification = Task.Run( () => batch - ? queue.NotifyBatch(peer, ns, [new("value", 1, true)], immediate: true) + ? queue.NotifyBatch(peer, ns, [new("value", 1, true)]) : queue.Notify(peer, ns, "value"), TestContext.Current.CancellationToken); @@ -7849,7 +6779,6 @@ public async Task BroadcastLocalCompletionReleasesCapacityAcrossPeerRemovalAndRe var transport = new FakeTransport(local, blocked, healthy); var ns = new FakeNamespace(local); ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); - ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); ns.SetValue("value", 1); var clock = new FakeTimeProvider(); var blockedResponse = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -7946,7 +6875,6 @@ public async Task CancelingFlushObserverPreservesQueuedBroadcastAttempt() var waiting = CreateSilo(39333); var transport = new FakeTransport(local, first, waiting); var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); ns.SetValue("value", 1); var firstResponse = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -8009,7 +6937,6 @@ public async Task WaitingBroadcastMaterializesLatestValueAfterAdmissionAndRotate var waiting = CreateSilo(39313); var transport = new FakeTransport(local, large, waiting); var ns = new FakeNamespace(local); - ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); ns.SetValue("large-first", 1); ns.SetValue("large-second", 2); ns.SetValue("large-third", 3); @@ -8104,7 +7031,6 @@ public async Task RepairAttemptsReleaseLocalCapacityBeforeLateCompletion(bool ca ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(1); - ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); var clock = new FakeTimeProvider(); var responses = blocked.ToDictionary(static peer => peer, static _ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); @@ -8400,7 +7326,7 @@ public async Task MembershipRepairConvergesAcrossJoinDepartureAndRestartWithoutR } var delayedDepartedValue = network[departed].Namespace.CreateRepair(new DisseminationRepairRequest( - DisseminationKey.Default, null, null, 1, 1024 * 1024, 1024 * 1024)).Values.Single(); + DisseminationKey.Default, null, 1024 * 1024, 1024 * 1024)).Value; var restartedSnapshot = CreateMembershipSnapshot(3, CreateMembershipEntry(first, SiloStatus.Active, DateTime.UnixEpoch), CreateMembershipEntry(second, SiloStatus.Active, DateTime.UnixEpoch), @@ -8477,7 +7403,7 @@ public async Task MembershipAntiEntropyConvergesSameVersionLivenessAndTerminates var repairCount = network.Replies.Sum(static response => GetAntiEntropyResponseValues(response).Count()); await network.RunSweep(network.Cancellation); var duplicate = source.Namespace.CreateRepair(new DisseminationRepairRequest( - DisseminationKey.Default, null, null, 1, 1024 * 1024, 1024 * 1024)).Values.Single(); + DisseminationKey.Default, null, 1024 * 1024, 1024 * 1024)).Value; foreach (var member in members.Skip(1)) { var acknowledgment = await network[member].Protocol.ReceiveBroadcast(new DisseminationBroadcastBatch @@ -8614,8 +7540,6 @@ public RecoveryNetwork( foreach (var member in members) { var ns = createNamespace(member); - ns.Options.Priority = DisseminationPriority.Normal; - ns.Options.MaxCoalescingDelay = TimeSpan.FromHours(1); ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(1); var transport = new FakeTransport(member, members.Where(peer => !peer.Equals(member)).ToArray()); var protocol = CreateProtocol(transport, [ns], options => @@ -8835,24 +7759,8 @@ private static DisseminationProtocol CreatePhase6Protocol( FakeTransport transport, IDisseminationNamespace disseminationNamespace, Microsoft.Extensions.Logging.ILogger? logger = null, - Action? configure = null) - { - var options = new DisseminationOptions { Enabled = true }; - configure?.Invoke(options); - var localSiloDetails = new FakeLocalSiloDetails(transport.LocalSilo); - return new DisseminationProtocol( - localSiloDetails, - transport.GrainFactory, - new DisseminationMembership( - transport.MembershipManager, - localSiloDetails, - Options.Create(options)), - new TestOptionsMonitor(options), - [disseminationNamespace], - TimeProvider.System, - logger ?? NullLogger.Instance, - NullLogger.Instance); - } + Action? configure = null) => + CreateProtocol(transport, [disseminationNamespace], configure, logger: logger); private sealed class Phase6ThrowingNamespace( SiloAddress localSilo, diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs index 9429c657d26..bce80ac1cd9 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs @@ -316,7 +316,7 @@ public async Task FullCohortPreservesPeerItemAndByteSplitting(int count) var queue = new DisseminationBroadcastQueue( rig.Clock, rig.Members[0], factory, new TestOptionsMonitor(rig.Options), [rig.Namespace], NullLogger.Instance); - rig.OnDispatch = attempt => queue.NotifyBatch(rig.Members[1], rig.Namespace, attempt.Notifications, immediate: false); + rig.OnDispatch = attempt => queue.NotifyBatch(rig.Members[1], rig.Namespace, attempt.Notifications); try { var receipts = new List> { await rig.StartPublicationAsync(0) }; @@ -479,13 +479,15 @@ public async Task OlderInFlightRetryUsesSealedSignalWhileNewerVersionWaitsForNex } [Theory] - [InlineData(false, 1L)] - [InlineData(true, 1L)] - [InlineData(true, 2L)] - public async Task PartialAdmissionRejectsReceiptButRetriesNewestHintAndForce(bool newForce, long version) + [InlineData(false, false, 1L)] + [InlineData(false, true, 1L)] + [InlineData(false, true, 2L)] + [InlineData(true, false, 1L)] + public async Task RejectedDispatchCachesReceiptAndRetriesNewestHintAndForce(bool callbackThrows, bool newForce, long version) { await using var rig = new TestRig(2); - rig.OnDispatch = attempt => attempt.Number != 1; + var failure = new InvalidOperationException("The parent dispatcher failed."); + rig.OnDispatch = attempt => attempt.Number != 1 || (callbackThrows ? throw failure : false); var first = await rig.StartPublicationAsync(0, force: true); var scheduled = rig.Clock.WhenScheduled(); rig.Clock.Advance(Period); @@ -501,10 +503,21 @@ public async Task PartialAdmissionRejectsReceiptButRetriesNewestHintAndForce(boo Assert.Single(rig.Attempts); rig.Clock.Advance(Milliseconds(1)); await Phase(rig.Batcher.FlushAsync(TestToken), "retained hint admitted on retry"); + Assert.Equal(2, rig.Attempts.Count); Assert.Equal(new[] { rig.Notification(0, 1, true), rig.Notification(0, version, true) }, rig.Attempts.SelectMany(attempt => attempt.Notifications)); // A later hint retry is not another publication receipt or another producer contribution. Assert.Equal(new(false, TimeSpan.Zero), await rig.Publish(0)); + if (callbackThrows) + { + var entry = Assert.Single(rig.Logger.Entries); + Assert.Equal(LogLevel.Warning, entry.Level); + Assert.Same(failure, entry.Exception); + } + else + { + Assert.Empty(rig.Logger.Entries); + } } [Fact] @@ -527,26 +540,6 @@ public async Task PartialChunkAdmissionResolvesOnlyItsOwnProducerReceipts() rig.Attempts.SelectMany(attempt => attempt.Notifications)); } - [Fact] - public async Task CallbackExceptionRejectsReceiptsRetriesHintsAndReportsFailure() - { - await using var rig = new TestRig(); - var failure = new InvalidOperationException("The parent dispatcher failed."); - rig.OnDispatch = attempt => attempt.Number == 1 ? throw failure : true; - var publication = await rig.StartPublicationAsync(0, force: true); - var scheduled = rig.Clock.WhenScheduled(); - rig.Clock.Advance(Period); - Assert.Equal(new(false, TimeSpan.Zero), await Phase(publication, "exception rejects the producer receipt")); - Assert.Equal(Period, await Phase(scheduled, "exception retry armed")); - var entry = Assert.Single(rig.Logger.Entries); - Assert.Equal(LogLevel.Warning, entry.Level); - Assert.Same(failure, entry.Exception); - rig.Clock.Advance(Period); - await Phase(rig.Batcher.FlushAsync(TestToken), "exception hint recovered"); - Assert.Equal(rig.Notification(0, 1, true), Assert.Single(rig.Attempts.Last().Notifications)); - Assert.Equal(2, rig.Attempts.Count); - } - [Fact] public async Task CallbackLoggingFailureIsVisibleToReceiptAdmissionAndDrain() { @@ -984,37 +977,6 @@ await Phase(Task.Run(() => rig.Batcher.Prune(new HashSet()), T Assert.Equal(new KeyNotification("a", 1, true), Assert.Single(Assert.Single(rig.Attempts).Notifications)); } - [Fact] - public async Task HighPriorityHintsBypassCollectionButFailedRetriesUsePublicationPeriod() - { - await using var rig = new TestRig(); - rig.Namespace.Options.Priority = DisseminationPriority.High; - rig.OnDispatch = attempt => attempt.Number != 1; - var started = rig.Clock.GetTimestamp(); - var scheduled = rig.Clock.WhenScheduled(); - Assert.True(rig.Batcher.Notify([new("a", 1, true)])); - Assert.Equal(started, (await rig.NextAttemptAsync()).Timestamp); - Assert.Equal(Period, await Phase(scheduled, "high priority retry bounded")); - rig.Clock.Advance(Period); - await Phase(rig.Batcher.FlushAsync(TestToken), "high priority retry accepted"); - Assert.Equal(2, rig.Attempts.Count); - } - - [Fact] - public async Task HighPriorityPublicationsStillRequireEveryProducerOrDeadline() - { - await using var rig = new TestRig(2); - rig.Namespace.Options.Priority = DisseminationPriority.High; - var first = await rig.StartPublicationAsync(0); - rig.Clock.Advance(Milliseconds(25)); - Assert.False(first.IsCompleted); - Assert.Empty(rig.Attempts); - var second = rig.Publish(1); - Assert.All(await Phase(Task.WhenAll(first, second), "priority cannot bypass the publication cohort"), - receipt => Assert.Equal(new(true, Milliseconds(975)), receipt)); - Assert.Equal(2, Assert.Single(rig.Attempts).Notifications.Length); - } - [Fact] public async Task StopSealsPartialCohortDrainsBoundedChunksAndRejectsFurtherAdmission() { @@ -1244,7 +1206,6 @@ private sealed class TestNamespace : IDisseminationNamespace private readonly DisseminationNamespaceOptions _options = new() { Enabled = true, - MaxCoalescingDelay = TimeSpan.FromMilliseconds(25), MaxPendingItemCount = 8192, }; @@ -1291,12 +1252,12 @@ public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest requ return DisseminationRepairResult.Current(1); } - if (request.MaxBatchBytes < sizeof(long) || request.MaxPayloadBytes < sizeof(long) || request.MaxItemCount < 1) + if (request.MaxBatchBytes < sizeof(long) || request.MaxPayloadBytes < sizeof(long)) { return DisseminationRepairResult.InsufficientCapacity(1); } - return DisseminationRepairResult.Produced(1, [new(request.Key, 0, 1, BitConverter.GetBytes(1L))]); + return DisseminationRepairResult.Produced(new(request.Key, 0, 1, BitConverter.GetBytes(1L))); } public ValueTask ApplyValueAsync(DisseminationValue value, CancellationToken cancellationToken) => diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs index c91e33a4d07..7b4321777b2 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs @@ -12,14 +12,22 @@ namespace UnitTests.Dissemination; [TestArea("Dissemination")] public class WakeTimerTests { - [Fact] - public async Task ChangeCompletesWaitWhenDueTimeElapses() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ChangeCompletesWaitAtLatestDueTime(bool rearm) { var timeProvider = new FakeTimeProvider(); using var timer = new WakeTimer(timeProvider); var wait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); timer.Change(TimeSpan.FromSeconds(1)); + if (rearm) + { + timeProvider.Advance(TimeSpan.FromMilliseconds(500)); + timer.Change(TimeSpan.FromSeconds(1)); + } + timeProvider.Advance(TimeSpan.FromMilliseconds(999)); Assert.False(wait.IsCompleted); timeProvider.Advance(TimeSpan.FromMilliseconds(1)); @@ -37,23 +45,6 @@ public async Task WakeBeforeWaitCompletesNextWait() Assert.True(await timer.WaitAsync(TestContext.Current.CancellationToken)); } - [Fact] - public async Task ChangeRearmsCurrentWait() - { - var timeProvider = new FakeTimeProvider(); - using var timer = new WakeTimer(timeProvider); - var wait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); - - timer.Change(TimeSpan.FromSeconds(1)); - timeProvider.Advance(TimeSpan.FromMilliseconds(500)); - timer.Change(TimeSpan.FromSeconds(1)); - timeProvider.Advance(TimeSpan.FromMilliseconds(999)); - Assert.False(wait.IsCompleted); - timeProvider.Advance(TimeSpan.FromMilliseconds(1)); - - Assert.True(await wait); - } - [Fact] public async Task StaleCallbackDoesNotWakeRearmedTimer() { @@ -120,29 +111,23 @@ public async Task WakeRacingCancelledWaitIsObservedByNextWaiter() Assert.True(await nextWait); } - [Fact] - public async Task DisposeCompletesCurrentAndFutureWaitsWithFalse() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DisposeCompletesCurrentAndFutureWaitsWithoutChangingTimer(bool changeThrows) { - var timer = new WakeTimer(TimeProvider.System); - var wait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); - - timer.Dispose(); - - Assert.False(await wait); - Assert.False(await timer.WaitAsync(TestContext.Current.CancellationToken)); - } - - [Fact] - public async Task DisposeCompletesWaitAndDisposesTimerWithoutChangingIt() - { - var timeProvider = new ThrowingChangeTimeProvider(); + TimeProvider timeProvider = changeThrows ? new ThrowingChangeTimeProvider() : TimeProvider.System; var timer = new WakeTimer(timeProvider); var wait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); timer.Dispose(); Assert.False(await wait); - Assert.True(timeProvider.Timer.IsDisposed); + Assert.False(await timer.WaitAsync(TestContext.Current.CancellationToken)); + if (timeProvider is ThrowingChangeTimeProvider throwingProvider) + { + Assert.True(throwingProvider.Timer.IsDisposed); + } } private sealed class QueuedSynchronizationContext : SynchronizationContext From 4096c5a0973dc29838f98efa72c3e42db32f77dd Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 17 Sep 2026 02:00:12 -0700 Subject: [PATCH 43/50] fix(dissemination): consume wake signals with captured work --- .../DisseminationBroadcastQueue.cs | 19 +- .../Dissemination/WakeTimer.cs | 18 ++ ...onProtocolTests.OwnerCancellationReview.cs | 17 ++ .../DisseminationProtocolTests.QueueReview.cs | 104 ++++++++- .../DisseminationProtocolTests.cs | 206 +++++++++++------- .../Dissemination/WakeTimerTests.cs | 54 +++++ 6 files changed, 321 insertions(+), 97 deletions(-) diff --git a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs index 6f5f66d0c23..7e1ff920d70 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs @@ -545,7 +545,6 @@ public async ValueTask FlushAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); Task? flushCompletion; - var wake = false; lock (_lock) { if (_pumpFailure is { } pumpFailure) @@ -556,7 +555,7 @@ public async ValueTask FlushAsync(CancellationToken cancellationToken) if (DirtyCount > 0) { flushCompletion = _nextFlushCompletion.Task; - wake = true; + _flushTimer.Wake(); } else { @@ -569,11 +568,6 @@ public async ValueTask FlushAsync(CancellationToken cancellationToken) return; } - if (wake) - { - _flushTimer.Wake(); - } - flushCompletion.Ignore(); await flushCompletion.WaitAsync(cancellationToken); } @@ -582,7 +576,6 @@ public async ValueTask StopAsync(bool drain, CancellationToken cancellationToken { Task? flushCompletion; TaskCompletionSource? droppedFlushCompletion = null; - var wake = false; var alreadyStopping = false; lock (_lock) { @@ -604,7 +597,6 @@ public async ValueTask StopAsync(bool drain, CancellationToken cancellationToken else if (DirtyCount > 0) { flushCompletion = _nextFlushCompletion.Task; - wake = true; } else { @@ -630,9 +622,12 @@ public async ValueTask StopAsync(bool drain, CancellationToken cancellationToken droppedFlushCompletion?.TrySetResult(); try { - if (wake) + lock (_lock) { - _flushTimer.Wake(); + if (ReferenceEquals(flushCompletion, _nextFlushCompletion.Task)) + { + _flushTimer.Wake(); + } } while (flushCompletion is not null) @@ -707,6 +702,8 @@ private async Task RunScheduledFlushIteration(CancellationToken cancellationToke // Move one dirty generation to in-flight atomically; a concurrent notification can mark it dirty again. lock (_lock) { + // All wakes issued before this drain refer to the work being consumed now. + _flushTimer.Reset(); if (DirtyCount == 0) { return; diff --git a/src/Orleans.Runtime/Dissemination/WakeTimer.cs b/src/Orleans.Runtime/Dissemination/WakeTimer.cs index 3f6b89989ed..51912cb4fd3 100644 --- a/src/Orleans.Runtime/Dissemination/WakeTimer.cs +++ b/src/Orleans.Runtime/Dissemination/WakeTimer.cs @@ -61,6 +61,24 @@ public void Change(TimeSpan dueTime) } } + /// + /// Disarms the timer and consumes any pending wake, leaving an active waiter in place. + /// + public void Reset() + { + lock (_lock) + { + if (_disposed) + { + return; + } + + _armed = false; + _signaled = false; + _timer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + } + } + /// /// Immediately completes the current wait, or causes the next wait to complete immediately. /// diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs index 112169db208..e2733ff75ff 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs @@ -63,6 +63,23 @@ private sealed class MembershipReviewContinuationContext : SynchronizationContex public override void Post(SendOrPostCallback callback, object? state) => _continuations.Writer.TryWrite((callback, state)); + public void RunAll() + { + var previous = SynchronizationContext.Current; + SynchronizationContext.SetSynchronizationContext(this); + try + { + while (_continuations.Reader.TryRead(out var continuation)) + { + continuation.Callback(continuation.State); + } + } + finally + { + SynchronizationContext.SetSynchronizationContext(previous); + } + } + public async Task<(SendOrPostCallback Callback, object? State)> TakeContinuation(CancellationToken cancellationToken) { using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs index 5fcf23fc33a..755885b0133 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs @@ -133,6 +133,77 @@ public void DisseminationKeysSortAcrossSupportedKinds() } } + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task RedundantNotificationAndFlushWakesRespectRetryBoundary(bool transportFailure, bool publishNewVersion) + { + var cancellationToken = TestContext.Current.CancellationToken; + var (_, peer, transport, ns) = CreatePeerFixture(41601, 41602); + var clock = new FakeTimeProvider(); + var context = new MembershipReviewContinuationContext(); + var versions = new List(); + ns.SetValue("value", 1); + transport.SendBroadcastResponseHandler = (_, batch, _) => + { + versions.Add(Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); + return versions.Count == 1 + ? transportFailure + ? Task.FromException(new InvalidOperationException("Initial send fails.")) + : Task.FromResult(new DisseminationBroadcastResponse + { + Acknowledgments = new() { [ns.Name] = [new("value", 0)] }, + }) + : Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); + try + { + Task initialFlush; + var previous = SynchronizationContext.Current; + SynchronizationContext.SetSynchronizationContext(context); + try + { + Assert.True(queue.Notify(peer, ns, "value")); + initialFlush = queue.FlushPendingBroadcast(cancellationToken); + Assert.False(initialFlush.IsCompleted); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previous); + } + + context.RunAll(); + await initialFlush.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(new long[] { 1 }, versions); + clock.Advance(TimeSpan.FromMilliseconds(99)); + context.RunAll(); + Assert.Equal(new long[] { 1 }, versions); + if (publishNewVersion) + { + ns.SetValue("value", 2); + Assert.True(queue.Notify(peer, ns, "value")); + } + else + { + clock.Advance(TimeSpan.FromMilliseconds(1)); + } + + context.RunAll(); + Assert.Equal(new long[] { 1, publishNewVersion ? 2 : 1 }, versions); + Assert.Equal(2, ns.RepairRequestCount); + } + finally + { + ns.Options.Enabled = false; + var stop = queue.StopAsync(cancellationToken); + context.RunAll(); + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -145,14 +216,24 @@ public async Task ShutdownDrainRetriesUntilAcceptedWorkIsAcknowledged(bool trans var ns = new FakeNamespace(local); ns.SetValue(FakeNamespace.DefaultKey, 2); var sentVersions = new List(); - transport.SendBroadcastResponseHandler = (_, batch, _) => + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = async (_, batch, token) => { sentVersions.Add(Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); - return sentVersions.Count == 1 - ? transportFailure - ? Task.FromException(new InvalidOperationException("Transient send failure.")) - : Task.FromResult(new DisseminationBroadcastResponse()) - : Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + if (sentVersions.Count == 1) + { + firstStarted.TrySetResult(); + await releaseFirst.Task.WaitAsync(token); + if (transportFailure) + { + throw new InvalidOperationException("Transient send failure."); + } + + return new DisseminationBroadcastResponse(); + } + + return FakeTransport.CreateAcknowledgment(batch); }; var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); @@ -163,10 +244,13 @@ public async Task ShutdownDrainRetriesUntilAcceptedWorkIsAcknowledged(bool trans TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - Assert.True(queue.Notify(peer, ns, FakeNamespace.DefaultKey)); - var stop = queue.StopAsync(cancellation.Token); + Task? stop = null; try { + Assert.True(queue.Notify(peer, ns, FakeNamespace.DefaultKey)); + await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + stop = queue.StopAsync(cancellation.Token); + releaseFirst.TrySetResult(); var scheduled = await retry; Assert.False(stop.IsCompleted); Assert.False(queue.Notify(peer, ns, "after-stop")); @@ -179,12 +263,14 @@ public async Task ShutdownDrainRetriesUntilAcceptedWorkIsAcknowledged(bool trans } finally { + releaseFirst.TrySetResult(); cancellation.Cancel(); + stop ??= queue.StopAsync(cancellation.Token); try { await stop.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); } - catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + catch (OperationCanceledException exception) when (exception.CancellationToken == cancellation.Token) { } } diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index 076cc52486d..2a8782e666d 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -264,6 +264,7 @@ public async Task NewPublicationWakesFailedPeerWithoutWaitingForBackoff() var peer = CreateSilo(11112); var transport = new FakeTransport(local, peer); var timeProvider = new FakeTimeProvider(); + var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var sendCount = 0; transport.SendBroadcastHandler = (target, batch, cancellationToken) => { @@ -273,22 +274,38 @@ public async Task NewPublicationWakesFailedPeerWithoutWaitingForBackoff() } transport.BroadcastBatches.Add((target, batch)); + sent.TrySetResult(); return Task.CompletedTask; }; var ns = new FakeNamespace(local); var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); - - var firstResult = await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken); - await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - var secondResult = await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), TestContext.Current.CancellationToken); - await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - - Assert.True(firstResult); - Assert.True(secondResult); - Assert.Equal(2, sendCount); - var batch = Assert.Single(transport.BroadcastBatches); - Assert.Equal(2, GetBroadcastValues(batch.Batch).Single().Value.ToVersion); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + scheduled => scheduled.LocalSilo.Equals(local) && scheduled.Peer.Equals(peer) + && scheduled.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + try + { + var firstResult = await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken); + await retry; + Assert.Equal(1, sendCount); + var beforeNotification = timeProvider.GetTimestamp(); + var secondResult = await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), TestContext.Current.CancellationToken); + await sent.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(firstResult); + Assert.True(secondResult); + Assert.Equal(TimeSpan.Zero, timeProvider.GetElapsedTime(beforeNotification)); + Assert.Equal(2, sendCount); + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(2, GetBroadcastValues(batch.Batch).Single().Value.ToVersion); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(TestContext.Current.CancellationToken); + } } [Theory] @@ -489,9 +506,8 @@ public async Task PeerVersionAdvancesOnlyFromExplicitAcknowledgment() ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken)); - await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); var retry = await schedule.WaitAsync( - e => e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry, + e => e.LocalSilo.Equals(local) && e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry, TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(1, sendCount); @@ -503,6 +519,7 @@ public async Task PeerVersionAdvancesOnlyFromExplicitAcknowledgment() Assert.Equal( new[] { 1L, 1L }, transport.BroadcastBatches.Select(batch => Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion)); + await protocol.StopAsync(TestContext.Current.CancellationToken); } [Fact] @@ -616,13 +633,13 @@ public async Task RemovedPendingKeyIsDroppedWithoutRetry() [Fact] public async Task BroadcastBatchingStopsPeerFlushAfterSendFailure() { - var local = CreateSilo(11111); - var peer = CreateSilo(11112); - var transport = new FakeTransport(local, peer); - var timeProvider = new AutoAdvancingTimeProvider(TimeSpan.FromSeconds(2)); + var (local, peer, transport, ns) = CreatePeerFixture(); + var timeProvider = new FakeTimeProvider(); var sendCount = 0; + var attempted = new List(); transport.SendBroadcastHandler = (target, batch, cancellationToken) => { + attempted.Add(Assert.Single(GetBroadcastValues(batch)).Value.Key); if (Interlocked.Increment(ref sendCount) == 1) { throw new InvalidOperationException("transient send failure"); @@ -632,33 +649,29 @@ public async Task BroadcastBatchingStopsPeerFlushAfterSendFailure() return Task.CompletedTask; }; - var ns = new FakeNamespace(local); - DisseminationOptions? optionsRef = null; - var protocol = CreateProtocol(transport, ns, options => - { - optionsRef = options; - options.MaxBatchItems = 10; - }, timeProvider); - - var publications = BeforeBroadcastPumpsRun(() => + ns.SetValue("first", 1); + ns.SetValue("second", 1); + ns.SetValue("third", 1); + var queue = CreateBroadcastQueue(transport, [ns], options => options.MaxBatchItems = 1, timeProvider); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + scheduled => scheduled.LocalSilo.Equals(local) && scheduled.Peer.Equals(peer) + && scheduled.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + try { - var pending = new[] - { - PublishValue(protocol, ns, ns.CreateValue("first", sequence: 1), TestContext.Current.CancellationToken), - PublishValue(protocol, ns, ns.CreateValue("second", sequence: 1), TestContext.Current.CancellationToken), - PublishValue(protocol, ns, ns.CreateValue("third", sequence: 1), TestContext.Current.CancellationToken), - }; - optionsRef!.MaxBatchItems = 1; - return pending; - }); - foreach (var publication in publications) + Assert.True(queue.NotifyBatch(peer, ns, [new("first", 1, true), new("second", 1, true), new("third", 1, true)])); + await retry; + Assert.Equal(1, sendCount); + Assert.Equal(new DisseminationKey[] { "first" }, attempted); + Assert.Equal(1, ns.RepairRequestCount); + Assert.Empty(transport.BroadcastBatches); + } + finally { - Assert.True(await publication); + ns.Options.Enabled = false; + await queue.StopAsync(TestContext.Current.CancellationToken); } - await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); - - Assert.Equal(1, sendCount); - Assert.Empty(transport.BroadcastBatches); } [Fact] @@ -670,6 +683,7 @@ public async Task BroadcastPeerFailureDoesNotBlockOtherPeerPumps() var transport = new FakeTransport(local, failedPeer, healthyPeer); var timeProvider = new FakeTimeProvider(); var failedPeerAttempts = 0; + var healthySent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); transport.SendBroadcastHandler = (target, batch, cancellationToken) => { if (Equals(target, failedPeer) && Interlocked.Increment(ref failedPeerAttempts) == 1) @@ -682,6 +696,11 @@ public async Task BroadcastPeerFailureDoesNotBlockOtherPeerPumps() transport.BroadcastBatches.Add((target, batch)); } + if (target.Equals(healthyPeer)) + { + healthySent.TrySetResult(); + } + return Task.CompletedTask; }; @@ -690,9 +709,14 @@ public async Task BroadcastPeerFailureDoesNotBlockOtherPeerPumps() { options.Overlay.FanOutFactor = static _ => 10; }, timeProvider); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + scheduled => scheduled.LocalSilo.Equals(local) && scheduled.Peer.Equals(failedPeer) + && scheduled.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.True(await PublishValue(protocol, ns, ns.CreateValue("first", sequence: 1), TestContext.Current.CancellationToken)); - await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await Task.WhenAll(retry, healthySent.Task).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.Equal(1, failedPeerAttempts); Assert.Equal(new[] { healthyPeer }, GetSentBroadcastPeers(transport)); @@ -701,6 +725,7 @@ public async Task BroadcastPeerFailureDoesNotBlockOtherPeerPumps() await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); Assert.Equal(2, failedPeerAttempts); Assert.Equal(new[] { failedPeer, healthyPeer }.OrderBy(static peer => peer), GetSentBroadcastPeers(transport).OrderBy(static peer => peer)); + await protocol.StopAsync(TestContext.Current.CancellationToken); } [Fact] @@ -4030,22 +4055,6 @@ public void Advance(TimeSpan value) } } - private sealed class AutoAdvancingTimeProvider(TimeSpan step) : TimeProvider - { - private readonly object _lock = new(); - private DateTimeOffset _utcNow = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); - - public override DateTimeOffset GetUtcNow() - { - lock (_lock) - { - var result = _utcNow; - _utcNow += step; - return result; - } - } - } - private sealed class RecordingFakeTimeProvider : TimeProvider { private readonly FakeTimeProvider _inner = new(); @@ -4878,10 +4887,16 @@ public async Task PeerQueueAdvancesKnownVersionOnlyFromResponseAcknowledgment() }; ns.Options.MaxPendingItemCount = 1; var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + scheduled => scheduled.LocalSilo.Equals(local) && scheduled.Peer.Equals(peer) + && scheduled.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); ns.Inner.SetValue("ack", 1); queue.Notify(peer, ns, "ack"); - await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await retry; + Assert.Equal(1, sendCount); ns.Inner.SetValue("ack", 2); queue.Notify(peer, ns, "ack"); await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); @@ -6633,8 +6648,8 @@ public async Task BroadcastDeadlineCompletesFlushBeforeRemoteCancellationAcknowl try { queue.Notify(peer, ns, "value"); - var flush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); var sendCancellation = await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var flush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); clock.Advance(ns.Options.StaleItemTtl); await flush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); @@ -6678,8 +6693,8 @@ public async Task TimedOutBroadcastReleasesLocalSlotBeforeRpcCompletes(int maxCo try { Assert.True(queue.Notify(peer, ns, "value")); - var firstFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); var sendCancellation = await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var firstFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); clock.Advance(ns.Options.StaleItemTtl); await firstFlush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); Assert.True(sendCancellation.IsCancellationRequested); @@ -6812,12 +6827,12 @@ public async Task BroadcastLocalCompletionReleasesCapacityAcrossPeerRemovalAndRe try { Assert.True(queue.Notify(blocked, ns, "value")); - var initialFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); var remoteCancellation = await blockedStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var initialFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); var healthyQueued = admission.Wait(healthy, "queued", TestContext.Current.CancellationToken); Assert.True(queue.Notify(healthy, ns, "value")); - var healthyFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); await healthyQueued.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var healthyFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); Assert.Equal(1, sends[blocked]); Assert.False(sends.ContainsKey(healthy)); Assert.Equal(1, ns.RepairRequestCount); @@ -7169,7 +7184,7 @@ public async Task AntiEntropyConvergesAfterPartitionWithEveryTreeBroadcastDroppe Assert.True(await PublishValue(node.Protocol, ns, ns.CreateValue(keys[i], i + 1), network.Cancellation)); } - await network.Flush(network.Cancellation); + await network.WaitForBroadcastRetries(members, network.Cancellation); Assert.Equal(12, network.Broadcasts.Count); Assert.All(network.Broadcasts, static send => Assert.False(send.Delivered)); await network.RunSweep(network.Cancellation); @@ -7218,10 +7233,11 @@ public async Task LostBroadcastAcknowledgmentsAndReorderedRepairRepliesConvergeA var receiverNamespace = Assert.IsType(receiver.Namespace); Assert.True(await PublishValue( source.Protocol, sourceNamespace, sourceNamespace.CreateValue(FakeNamespace.DefaultKey, 1), network.Cancellation)); - await network.Flush(network.Cancellation); + await network.WaitForBroadcastRetries([source.Address], network.Cancellation); Assert.Equal(2, network.Broadcasts.Count); Assert.All(network.Broadcasts, static send => Assert.True(send.Delivered)); AssertFakeState(receiverNamespace, (FakeNamespace.DefaultKey, 1)); + network.DropTreeBroadcasts = true; sourceNamespace.SetValue(FakeNamespace.DefaultKey, 3); olderNamespace.SetValue(FakeNamespace.DefaultKey, 2); @@ -7251,7 +7267,7 @@ public async Task LostBroadcastAcknowledgmentsAndReorderedRepairRepliesConvergeA try { - network.Clock.Advance(TimeSpan.FromSeconds(1)); + network.Clock.Advance(RecoveryNetwork.RepairCadence); var round = receiver.Protocol.RunAntiEntropyRound(network.Cancellation); await bothRepliesCaptured.Task.WaitAsync(TimeSpan.FromSeconds(5), network.Cancellation); Assert.Equal(3, Assert.Single(GetAntiEntropyResponseValues(replies[source.Address])).Value.ToVersion); @@ -7265,7 +7281,7 @@ public async Task LostBroadcastAcknowledgmentsAndReorderedRepairRepliesConvergeA AssertFakeState(receiverNamespace, (FakeNamespace.DefaultKey, 3)); Assert.Equal(2, receiverNamespace.ApplyCounts[FakeNamespace.DefaultKey]); receiver.Transport.ExchangeAntiEntropyHandler = (peer, _, _) => ValueTask.FromResult(replies[peer]); - network.Clock.Advance(TimeSpan.FromSeconds(1)); + network.Clock.Advance(RecoveryNetwork.RepairCadence); await receiver.Protocol.RunAntiEntropyRound(network.Cancellation); AssertFakeState(receiverNamespace, (FakeNamespace.DefaultKey, 3)); Assert.Equal(2, receiverNamespace.ApplyCounts[FakeNamespace.DefaultKey]); @@ -7393,7 +7409,7 @@ public async Task MembershipAntiEntropyConvergesSameVersionLivenessAndTerminates managers[source.Address].CurrentSnapshot = advanced; Assert.NotEqual(oldFingerprint, source.Namespace.Digests.Single().Fingerprint); Assert.True(await source.Protocol.Publish(source.Namespace, DisseminationKey.Default, 7, network.Cancellation)); - await network.Flush(network.Cancellation); + await network.WaitForBroadcastRetries([source.Address], network.Cancellation); await network.RunSweep(network.Cancellation); foreach (var member in members) { @@ -7402,6 +7418,7 @@ public async Task MembershipAntiEntropyConvergesSameVersionLivenessAndTerminates var repairCount = network.Replies.Sum(static response => GetAntiEntropyResponseValues(response).Count()); await network.RunSweep(network.Cancellation); + var broadcastsBeforeDuplicates = network.Broadcasts.ToArray(); var duplicate = source.Namespace.CreateRepair(new DisseminationRepairRequest( DisseminationKey.Default, null, 1024 * 1024, 1024 * 1024)).Value; foreach (var member in members.Skip(1)) @@ -7425,6 +7442,7 @@ public async Task MembershipAntiEntropyConvergesSameVersionLivenessAndTerminates Assert.Equal(0, applied[source.Address]); Assert.Equal(18, network.RequestCount); Assert.Equal(repairCount, network.Replies.Sum(static response => GetAntiEntropyResponseValues(response).Count())); + Assert.Equal(broadcastsBeforeDuplicates, network.Broadcasts.ToArray()); Assert.Equal(2, network.Broadcasts.Count); Assert.All(network.Broadcasts, static send => Assert.False(send.Delivered)); } @@ -7469,7 +7487,7 @@ public async Task RejectedQueueKeysRecoverThroughTruncatedDigestsDespiteAdvancin for (var round = 1; round <= 4; round++) { sourceNamespace.SetValue(keys[0], round); - network.Clock.Advance(TimeSpan.FromSeconds(1)); + network.Clock.Advance(RecoveryNetwork.RepairCadence); await receiver.Protocol.RunAntiEntropyRound(network.Cancellation); } @@ -7485,7 +7503,7 @@ public async Task RejectedQueueKeysRecoverThroughTruncatedDigestsDespiteAdvancin for (var round = 0; round < 2; round++) { - network.Clock.Advance(TimeSpan.FromSeconds(1)); + network.Clock.Advance(RecoveryNetwork.RepairCadence); await receiver.Protocol.RunAntiEntropyRound(network.Cancellation); } @@ -7527,8 +7545,10 @@ private static void AssertMembershipState(MembershipTableSnapshot expected, Memb // Every delivery enters a real protocol. There is deliberately no legacy transport or direct namespace-copy path. private sealed class RecoveryNetwork : IAsyncDisposable { + public static readonly TimeSpan RepairCadence = TimeSpan.FromMilliseconds(1); private readonly Dictionary _nodes = []; private readonly CancellationTokenSource _cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + private readonly BroadcastScheduleObserver _schedules = new(); private SiloAddress[] _participants; public RecoveryNetwork( @@ -7540,7 +7560,7 @@ public RecoveryNetwork( foreach (var member in members) { var ns = createNamespace(member); - ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(1); + ns.Options.ExpectedUpdateCadence = RepairCadence; var transport = new FakeTransport(member, members.Where(peer => !peer.Equals(member)).ToArray()); var protocol = CreateProtocol(transport, [ns], options => { @@ -7609,7 +7629,8 @@ public void SetTopology(params (SiloAddress Silo, SiloStatus Status)[] members) public async Task RunSweep(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - Clock.Advance(TimeSpan.FromSeconds(1)); + // Repair progresses while the fixed 100 ms broadcast retry timer remains dormant. + Clock.Advance(RepairCadence); foreach (var member in _participants) { await _nodes[member].Protocol.RunAntiEntropyRound(cancellationToken) @@ -7619,6 +7640,26 @@ await _nodes[member].Protocol.RunAntiEntropyRound(cancellationToken) Sweeps++; } + public async Task WaitForBroadcastRetries(SiloAddress[] publishers, CancellationToken cancellationToken) + { + var waits = publishers.SelectMany(publisher => + GetOriginatorTreeTargets(publisher, _nodes[publisher].Transport.Peers, fanout: 8) + .Select(peer => _schedules.WaitAsync( + scheduled => scheduled.LocalSilo.Equals(publisher) && scheduled.Peer.Equals(peer) + && scheduled.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), cancellationToken))).ToArray(); + try + { + await Task.WhenAll(waits); + } + catch (TimeoutException exception) + { + throw new TimeoutException( + $"Recovery broadcasts did not enter backoff for {string.Join(", ", publishers.Select(static silo => silo.ToString()))}; " + + $"broadcasts={Broadcasts.Count}, repair requests={RequestCount}, sweeps={Sweeps}.", exception); + } + } + public async Task Flush(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -7631,6 +7672,7 @@ public async Task Flush(CancellationToken cancellationToken) public async ValueTask DisposeAsync() { _cancellation.Cancel(); + var cleanupToken = _cancellation.Token; foreach (var node in _nodes.Values) { node.Namespace.Options.Enabled = false; @@ -7638,16 +7680,26 @@ public async ValueTask DisposeAsync() try { - foreach (var node in _nodes.Values) - { - await node.Protocol.StopAsync(TestContext.Current.CancellationToken) - .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - } + // Teardown aborts accepted retries; scenarios assert graceful drainage before disposal. + await Task.WhenAll(_nodes.Values.Select(StopNode)) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); } finally { + _schedules.Dispose(); _cancellation.Dispose(); } + + async Task StopNode(RecoveryNode node) + { + try + { + await node.Protocol.StopAsync(cleanupToken); + } + catch (OperationCanceledException exception) when (exception.CancellationToken == cleanupToken) + { + } + } } } diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs index 7b4321777b2..cba2cc6cd52 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs @@ -45,6 +45,60 @@ public async Task WakeBeforeWaitCompletesNextWait() Assert.True(await timer.WaitAsync(TestContext.Current.CancellationToken)); } + [Fact] + public async Task ResetConsumesRedundantWakeBeforeRetryIsArmed() + { + var timeProvider = new FakeTimeProvider(); + using var timer = new WakeTimer(timeProvider); + var first = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + timer.Wake(); + timer.Wake(); + Assert.True(await first); + + timer.Reset(); + timer.Change(TimeSpan.FromMilliseconds(100)); + var retry = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(retry.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(99)); + Assert.False(retry.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + Assert.True(await retry.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ResetDisarmsOldDeadlineAndPreservesActiveWaiter(bool cancelWait) + { + var timeProvider = new FakeTimeProvider(); + using var timer = new WakeTimer(timeProvider); + using var cancellation = new CancellationTokenSource(); + var wait = timer.WaitAsync(cancellation.Token).AsTask(); + timer.Change(TimeSpan.FromMilliseconds(100)); + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + timer.Reset(); + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + Assert.False(wait.IsCompleted); + + if (cancelWait) + { + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + () => wait.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + timer.Wake(); + Assert.True(await timer.WaitAsync(TestContext.Current.CancellationToken)); + } + else + { + timer.Change(TimeSpan.FromMilliseconds(100)); + timeProvider.Advance(TimeSpan.FromMilliseconds(99)); + Assert.False(wait.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + Assert.True(await wait.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + } + [Fact] public async Task StaleCallbackDoesNotWakeRearmedTimer() { From 3435701f79cca42a7d607b4d9f25565cdeeea2cf Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 17 Sep 2026 02:30:32 -0700 Subject: [PATCH 44/50] test(dissemination): release deferred regression continuations --- ...onProtocolTests.OwnerCancellationReview.cs | 17 -------- .../DisseminationProtocolTests.QueueReview.cs | 40 ++++++++++--------- 2 files changed, 21 insertions(+), 36 deletions(-) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs index e2733ff75ff..112169db208 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs @@ -63,23 +63,6 @@ private sealed class MembershipReviewContinuationContext : SynchronizationContex public override void Post(SendOrPostCallback callback, object? state) => _continuations.Writer.TryWrite((callback, state)); - public void RunAll() - { - var previous = SynchronizationContext.Current; - SynchronizationContext.SetSynchronizationContext(this); - try - { - while (_continuations.Reader.TryRead(out var continuation)) - { - continuation.Callback(continuation.State); - } - } - finally - { - SynchronizationContext.SetSynchronizationContext(previous); - } - } - public async Task<(SendOrPostCallback Callback, object? State)> TakeContinuation(CancellationToken cancellationToken) { using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs index 755885b0133..c74934c79dd 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs @@ -141,14 +141,23 @@ public void DisseminationKeysSortAcrossSupportedKinds() public async Task RedundantNotificationAndFlushWakesRespectRetryBoundary(bool transportFailure, bool publishNewVersion) { var cancellationToken = TestContext.Current.CancellationToken; - var (_, peer, transport, ns) = CreatePeerFixture(41601, 41602); + var (local, peer, transport, ns) = CreatePeerFixture(41601, 41602); var clock = new FakeTimeProvider(); - var context = new MembershipReviewContinuationContext(); var versions = new List(); + var secondAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + value => value.LocalSilo.Equals(local) && value.Peer.Equals(peer) + && value.Reason == DisseminationBroadcastScheduleReason.Retry); ns.SetValue("value", 1); transport.SendBroadcastResponseHandler = (_, batch, _) => { versions.Add(Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); + if (versions.Count == 2) + { + secondAttempt.TrySetResult(); + } + return versions.Count == 1 ? transportFailure ? Task.FromException(new InvalidOperationException("Initial send fails.")) @@ -161,26 +170,21 @@ public async Task RedundantNotificationAndFlushWakesRespectRetryBoundary(bool tr var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); try { - Task initialFlush; - var previous = SynchronizationContext.Current; - SynchronizationContext.SetSynchronizationContext(context); - try + var initialFlush = BeforeBroadcastPumpsRun(() => { Assert.True(queue.Notify(peer, ns, "value")); - initialFlush = queue.FlushPendingBroadcast(cancellationToken); - Assert.False(initialFlush.IsCompleted); - } - finally - { - SynchronizationContext.SetSynchronizationContext(previous); - } + var flush = queue.FlushPendingBroadcast(cancellationToken); + Assert.False(flush.IsCompleted); + return flush; + }); - context.RunAll(); await initialFlush.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + var scheduled = await retry.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(TimeSpan.FromMilliseconds(100), scheduled.DueTime); Assert.Equal(new long[] { 1 }, versions); clock.Advance(TimeSpan.FromMilliseconds(99)); - context.RunAll(); Assert.Equal(new long[] { 1 }, versions); + Assert.False(secondAttempt.Task.IsCompleted); if (publishNewVersion) { ns.SetValue("value", 2); @@ -191,16 +195,14 @@ public async Task RedundantNotificationAndFlushWakesRespectRetryBoundary(bool tr clock.Advance(TimeSpan.FromMilliseconds(1)); } - context.RunAll(); + await secondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); Assert.Equal(new long[] { 1, publishNewVersion ? 2 : 1 }, versions); Assert.Equal(2, ns.RepairRequestCount); } finally { ns.Options.Enabled = false; - var stop = queue.StopAsync(cancellationToken); - context.RunAll(); - await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await queue.StopAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); } } From 728523a11c4331de7c71fbc5d36fc68fb8b7844e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 17 Sep 2026 02:38:15 -0700 Subject: [PATCH 45/50] test(dissemination): bound the retry observer wait --- .../Dissemination/DisseminationProtocolTests.QueueReview.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs index c74934c79dd..1d2e7c2b8d6 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs @@ -148,7 +148,9 @@ public async Task RedundantNotificationAndFlushWakesRespectRetryBoundary(bool tr using var schedules = new BroadcastScheduleObserver(); var retry = schedules.WaitAsync( value => value.LocalSilo.Equals(local) && value.Peer.Equals(peer) - && value.Reason == DisseminationBroadcastScheduleReason.Retry); + && value.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), + cancellationToken); ns.SetValue("value", 1); transport.SendBroadcastResponseHandler = (_, batch, _) => { From 02e4326fd68f80a95e67fb78a3f751104181120a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 17 Sep 2026 14:10:50 -0700 Subject: [PATCH 46/50] feat(dissemination): broadcast sparse membership deltas --- .../implementation/runtime-dissemination.md | 21 +- .../IDisseminationSystemTarget.cs | 5 +- .../DisseminationBroadcastQueue.cs | 67 ++- .../DisseminationProtocol.Cohorts.cs | 3 +- .../Dissemination/DisseminationProtocol.cs | 35 +- .../Dissemination/IDisseminationNamespace.cs | 27 +- .../MembershipDisseminationNamespace.cs | 315 ++++++++++- ...eminationProtocolTests.MembershipDeltas.cs | 523 ++++++++++++++++++ .../DisseminationProtocolTests.cs | 20 +- 9 files changed, 941 insertions(+), 75 deletions(-) create mode 100644 test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipDeltas.cs diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index c153d8b8eba..f5fca428a11 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -36,7 +36,7 @@ flowchart LR `DisseminationProtocol` coordinates routing, peer capability evidence, anti-entropy, and application isolation. `DisseminationRootBatcher` owns the aggregation root's bounded latest-key cohorts, producer contributions, and seal receipts. `DisseminationBroadcastQueue` owns per-peer scheduling, retry, drain, and acknowledged-version ledgers. `DisseminationMembership` projects one membership snapshot into topology-specific member sets. Each `IDisseminationNamespace` owns serialization, current versions, payload limits, repair construction, and application semantics. -Queue entries contain `(namespace, key)` identities. A pump acquires both destination ownership and a global broadcast slot before materializing a repair. Coalesced notifications therefore use the latest namespace state and acknowledged baseline after admission, without retaining serialized payloads for waiting peers. +Queue entries contain `(namespace, key)` identities. A pump acquires both destination ownership and a global broadcast slot before materializing an update. Membership peer ledgers also retain one immutable comparison snapshot from the last accepted, exactly acknowledged broadcast. Coalesced notifications compare that snapshot with current owner state after admission, preserving cumulative changes and reversions while waiting peers retain shared snapshot references rather than serialized messages. Single-key publication and received-batch forwarding share one synchronous queue-admission path. A borrowed notification span is consumed under the pump lock before scheduling once, and diagnostic callbacks run after releasing the lock. Peer capability records store confirmed namespace identities for the lifetime of that silo generation; explicit rejection or membership removal retires the confirmation. @@ -44,24 +44,27 @@ Applied state wakes each forwarding child, including same-version membership liv ## Version and application invariants -A `DisseminationValue` carries one key's full current state and version: +A `DisseminationValue` carries one key's update and version: - its zero `FromVersion` identifies a full value which can establish state at any receiver baseline; +- a positive `FromVersion` identifies a membership broadcast delta from that canonical view to `ToVersion`; - acknowledged peer versions advance monotonically from explicit receiver evidence; - duplicate and obsolete values leave authoritative state unchanged; and - one rejected value does not prevent later values in the batch from being considered. -The namespace reports whether the peer is current, a full value was produced, the key is unavailable, or the value exceeds the supplied byte budgets. Batch assembly reserves an item slot before requesting a value and retains unsent keys for subsequent bounded batches. +Namespaces construct broadcasts separately from full anti-entropy repairs. Each result contains one value; batch assembly reserves an item slot before construction and retains unsent keys for subsequent bounded batches. Deployment-load broadcasts remain full latest samples. -Broadcast senders advertise support for compact acknowledgments. Receive selection explicitly reports whether the whole batch passed validation. When it does and each key's current version equals the maximum version transmitted for that key, one response flag certifies those versions. Namespace entries retain capability evidence with empty per-key lists. The sender advances each ledger to its captured transmitted version. Receivers report explicit versions for older callers, truncated batches, and differing receiver versions. +Broadcast senders advertise support for compact acknowledgments. The compact flag requires complete receive admission, accepted or duplicate application results, and exact agreement with each key's transmitted version. Namespace entries retain capability evidence with empty per-key lists. An exact compact acknowledgment also installs the membership comparison snapshot captured when constructing that send. Explicit higher versions and passive peer knowledge advance version evidence; a comparison snapshot is usable only while its version matches that evidence. -Membership uses the membership-table version plus a liveness fingerprint in its digest because `IAmAliveTime` can advance without a table-version change. Repair sends the current full snapshot and caches its serialized payload until the owner changes state. The complete inventory reconciles table cleanup across peers with different same-version baselines. Same-version peer snapshots can advance liveness or remove inactive entries while preserving every Active entry and every retained versioned field, including status, suspect votes, endpoint metadata, and start time. Authoritative same-version table reads preserve previously accepted versioned fields while merging heartbeat progress, accommodating provider timestamp rounding. Dissemination reports whether the membership manager actually published changed state. +Membership broadcasts contain changed entries and removed silo identities, together with base and target canonical view versions. A receiver at the base applies the transition. A receiver already at the target preserves its canonical fields, merges maximum heartbeat timestamps, and applies inactive-entry removals. Other versions require full repair. A rejected delta makes the key eligible for the next anti-entropy round, even if its local version advanced recently. Relays construct their own sparse update against each child's comparison snapshot, so an ahead relay can distribute its newer accepted view rather than regress to an older incoming update. -Development clustering recreates its volatile membership table when the primary restarts. Its manager treats received membership snapshots as notifications to perform a fresh authoritative read, so retained peer state from the prior table cannot replace the recreated table's view. Durable membership providers retain snapshot-based application. Repair re-reads the current owner under the payload-cache lock, including after a delayed publication or an owner-authorized version replacement. +A peer without a usable comparison snapshot receives an empty delta at the sender's current version. A peer already at that version acknowledges the probe; a lagging peer obtains a full snapshot through repair or ordinary membership bootstrap. The retained snapshot establishes a canonical comparison baseline, while independent initial heartbeat and inactive-cleanup differences remain repair work. -If an existing development silo observes a table counter below the version it knew when it started the read, it requests local shutdown and reports that it must restart against the recreated table. This preserves the monotonic membership-version contract shared with directory and other runtime consumers. A read which began before a newer local observation can finish normally; its older result is ignored. Restart the affected silos after recreating development membership. +Membership anti-entropy uses the view version and a liveness/inventory fingerprint. Responses always contain full current snapshots, whose serialized payloads are cached until owner state changes. Same-version repair preserves canonical fields, merges maximum heartbeats, and intersects inactive inventories so independently cleaned entries stay removed. Application checks the requested effects against resulting owner state before acknowledging acceptance, including when an authority refresh completes concurrently. -The membership namespace requests authoritative validation for older full values. Broadcast and anti-entropy let those values reach the owner, and repair selection consults the namespace for numerically greater peer baselines. The owner decides the resulting state. Deployment-load samples retain their timestamp ordering. +Comparison snapshots are shared immutable objects, with one retained reference per existing outbound membership peer/key and constant-size namespace caches. They are pruned with peer and key knowledge. Healthy peers share the same snapshot; lagging peers can retain different snapshots, giving a worst-case memory cost proportional to outbound peers times membership size. Sparse payload caching uses snapshot identity/content, including same-version changes. + +Canonical membership versions are monotonic within a cluster lifetime; hosts can skip observed versions. Development clustering's primary table defines that lifetime. Loss of that authority invalidates the continuing cluster, and an observed table rollback requests fatal silo termination. Full repair follows the membership owner's lifetime and validation rules. Deployment-load samples retain their timestamp ordering. Deployment-load versions are sample timestamps. Each repair is a full latest value. Application runs on the deployment-load publisher's scheduler and accepts samples only for Active silo generations in the membership oracle. Terminal membership transitions remove placement statistics, and the oracle's monotonic membership state rejects later samples for departed generations. @@ -167,7 +170,7 @@ Digests identify namespace-owned keys, including missing state at version zero, Requests advertise the receiver's item and payload-byte limits. Responders use the smaller local and advertised limits, allowing their response cursor to provide fair service even when peers use different batch sizes. Older requests use the responder's own limits. Receivers independently cap incoming broadcasts and repair responses before applying values. Broadcast acknowledgments cover only the admitted keys at their actual local versions. Receive cursors advance through bounded ordered intervals for oversized deliveries. Cursor state is scoped to peer and direction, with at most 64 retained non-member cursors. Request digest inventory remains separate from these response admission budgets. -Each selected full value is offered to its namespace. This preserves complementary same-version membership heartbeat advances from different peers; the owner merges them and classifies duplicates or obsolete state. Values with a nonzero baseline or nonpositive destination version are rejected before application. Responses which completed successfully within their local attempt are still applied when another peer exceeds the round deadline. Late RPC faults are observed, and subsequent rounds repair responses which missed the application window. During a partition, local attempts remain bounded and retries follow the configured cadence; after connectivity and membership views recover, rotating peer selection and cursors continue convergence. +Each selected full repair value is offered to its namespace. This preserves complementary same-version membership heartbeat advances from different peers; the owner merges them and classifies duplicates or obsolete state. Anti-entropy responses require a zero baseline and a positive destination version; membership deltas are admitted through broadcast instead. Responses which completed successfully within their local attempt are still applied when another peer exceeds the round deadline. Late RPC faults are observed, and subsequent rounds repair responses which missed the application window. During a partition, local attempts remain bounded and retries follow the configured cadence; after connectivity and membership views recover, rotating peer selection and cursors continue convergence. ## Mixed-version behavior diff --git a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs index a4ff1aba8a0..bae02f50690 100644 --- a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs +++ b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs @@ -215,7 +215,7 @@ public DigestEntry(DisseminationKey key, long version, long fingerprint = 0) public long Fingerprint { get; } } -// Values carry full state. FromVersion remains zero on the wire. +// FromVersion zero carries full state; a positive baseline identifies a namespace-specific broadcast delta. [GenerateSerializer, Immutable] internal readonly struct DisseminationValue { @@ -275,7 +275,8 @@ internal sealed class DisseminationBroadcastResponse [Id(1)] public List UnsupportedNamespaces { get; init; } = []; - // Certifies the maximum transmitted version of every key. Namespace entries retain capability evidence. + // Certifies accepted values and the exact maximum transmitted version of every key. + // Namespace entries retain capability evidence. [Id(2)] public bool AllVersionsAcknowledged { get; init; } } diff --git a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs index 7e1ff920d70..1545e88628b 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs @@ -369,7 +369,7 @@ private bool NotifyKeyUnsafe( var knownVersion = keyState?.KnownVersion; if (knownVersion is null && namespaceState.KnownVersions.TryGetValue(key, out var recordedVersion)) { - knownVersion = recordedVersion; + knownVersion = recordedVersion.Version; } if (knownVersion >= version) @@ -845,7 +845,7 @@ private async ValueTask SendValues( List initialWork, CancellationToken cancellationToken) { - // Every pass re-materializes repairs from the latest acknowledged version instead of retaining serialized messages. + // Materialize current broadcasts after admission using the peer's acknowledged baseline. var pending = new Queue(initialWork); var requiresBackoff = false; var madeProgress = false; @@ -905,13 +905,13 @@ private async ValueTask SendValues( continue; } - var knownVersion = GetKnownVersion(work); + var (knownVersion, baseline) = GetBroadcastBaseline(work); var request = new DisseminationRepairRequest( work.Key, knownVersion, currentOptions.MaxBatchBytes - byteCount, work.Namespace.Options.MaxPayloadBytes); - var repair = work.Namespace.CreateRepair(request); + var repair = work.Namespace.CreateBroadcast(request, baseline); if (repair.Status is DisseminationRepairStatus.Current) { // The namespace confirms that the peer is already current, so no RPC is needed. @@ -944,7 +944,7 @@ private async ValueTask SendValues( } if (repair.Status is not DisseminationRepairStatus.Produced - || !ValidateRepair(request, repair)) + || !ValidateBroadcast(work.Namespace, request, repair)) { // Invalid or temporarily unavailable repairs retain the dirty key and enter backoff. pending.Dequeue(); @@ -968,7 +968,7 @@ private async ValueTask SendValues( itemCount++; byteCount += value.Payload.Length; - sentKeys.Add(new(work, knownVersion, value.ToVersion)); + sentKeys.Add(new(work, knownVersion, value.ToVersion, repair.BroadcastState)); if (itemCount >= currentOptions.MaxBatchItems || byteCount >= currentOptions.MaxBatchBytes) { break; @@ -1018,7 +1018,7 @@ private async ValueTask SendValues( continue; } - var completion = CompleteAcknowledged(sent, acknowledgedVersion); + var completion = CompleteAcknowledged(sent, acknowledgedVersion, response.AllVersionsAcknowledged); madeProgress |= completion.MadeProgress; requiresBackoff |= completion.RequiresBackoff; } @@ -1167,13 +1167,17 @@ private List DrainDirtyUnsafe() return result; } - private long? GetKnownVersion(PendingKeyWork work) + private (long? Version, DisseminationBroadcastState? State) GetBroadcastBaseline(PendingKeyWork work) { lock (_lock) { - return TryGetKeyStateUnsafe(work, out _, out var keyState) - ? keyState.KnownVersion - : work.KnownVersion; + if (!TryGetKeyStateUnsafe(work, out _, out var keyState)) + { + return (work.KnownVersion, null); + } + + var state = keyState.BroadcastState; + return (keyState.KnownVersion, state?.Version == keyState.KnownVersion ? state : null); } } @@ -1207,7 +1211,7 @@ private void CompleteCurrent(PendingKeyWork work, long version) } } - private SendWorkResult CompleteAcknowledged(SentKey sent, long acknowledgedVersion) + private SendWorkResult CompleteAcknowledged(SentKey sent, long acknowledgedVersion, bool accepted) { lock (_lock) { @@ -1222,6 +1226,11 @@ private SendWorkResult CompleteAcknowledged(SentKey sent, long acknowledgedVersi keyState.KnownVersion = acknowledgedVersion; } + if (accepted && keyState.KnownVersion == sent.SentVersion) + { + keyState.BroadcastState = sent.BroadcastState; + } + var madeProgress = sent.FromVersion is null ? acknowledgedVersion > 0 : acknowledgedVersion > sent.FromVersion; @@ -1320,6 +1329,7 @@ private void CompleteUnsupported(DisseminationNamespace namespaceName, List disseminationNamespace.Digests.Any(entry => entry.Key == key); - private static bool ValidateRepair( + private static bool ValidateBroadcast( + IDisseminationNamespace disseminationNamespace, in DisseminationRepairRequest request, in DisseminationRepairResult repair) { @@ -1482,8 +1493,11 @@ private static bool ValidateRepair( return repair.Status is DisseminationRepairStatus.Produced && repair.Version > 0 && value.Key == request.Key - && value.FromVersion == 0 + && (disseminationNamespace.BroadcastsAreDeltas + ? value.FromVersion > 0 && value.FromVersion <= value.ToVersion + : value.FromVersion == 0) && value.ToVersion == repair.Version + && (repair.BroadcastState is null || repair.BroadcastState.Version == value.ToVersion) && value.Payload.Length <= request.MaxPayloadBytes && value.Payload.Length <= request.MaxBatchBytes; } @@ -1494,25 +1508,31 @@ private sealed class PeerNamespaceState(IDisseminationNamespace disseminationNam public Dictionary Keys { get; } = []; - public Dictionary KnownVersions { get; } = []; + public Dictionary KnownVersions { get; } = []; public PeerKeyState AddKey(DisseminationKey key) { var result = new PeerKeyState(); if (KnownVersions.TryGetValue(key, out var knownVersion)) { - result.KnownVersion = knownVersion; + result.KnownVersion = knownVersion.Version; + result.BroadcastState = knownVersion.State; } Keys.Add(key, result); return result; } - public void ObserveKnownVersion(DisseminationKey key, long version) + public void ObserveKnownVersion( + DisseminationKey key, + long version, + DisseminationBroadcastState? state = null) { - if (!KnownVersions.TryGetValue(key, out var knownVersion) || version > knownVersion) + state = state?.Version == version ? state : null; + if (!KnownVersions.TryGetValue(key, out var knownVersion) || version > knownVersion.Version + || version == knownVersion.Version && state is not null) { - KnownVersions[key] = version; + KnownVersions[key] = new(version, state); } } @@ -1520,7 +1540,7 @@ public void RetireKey(DisseminationKey key, PeerKeyState keyState) { if (keyState.KnownVersion is { } knownVersion) { - ObserveKnownVersion(key, knownVersion); + ObserveKnownVersion(key, knownVersion, keyState.BroadcastState); } Keys.Remove(key); @@ -1543,6 +1563,8 @@ private sealed class PeerKeyState // A null version means no known baseline. Dirty can coexist with InFlight when a newer notification arrives mid-send. public long? KnownVersion { get; set; } + public DisseminationBroadcastState? BroadcastState { get; set; } + public long NotificationGeneration { get; set; } public long NotificationVersion { get; set; } @@ -1562,7 +1584,10 @@ private readonly record struct PendingKeyWork( private readonly record struct SentKey( PendingKeyWork Work, long? FromVersion, - long SentVersion); + long SentVersion, + DisseminationBroadcastState? BroadcastState); + + private readonly record struct PeerKnowledge(long Version, DisseminationBroadcastState? State); private readonly record struct DigestKey( DisseminationNamespace Namespace, diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs index 990bcbd4bd5..55f585ae847 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs @@ -136,7 +136,8 @@ public async Task ReceivePublication( return Reject("root-changed"); } - var result = await ApplyReceivedValue(ns, request.Value, request.Sender, options, receivedTimestamp, cancellationToken); + var result = await ApplyReceivedValue( + ns, request.Value, request.Sender, options, receivedTimestamp, allowDelta: false, cancellationToken); if (result is not (DisseminationApplyResult.Applied or DisseminationApplyResult.Duplicate)) { return Reject("application-rejected"); diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index f9b81eed418..32ff0b101d3 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -215,8 +215,23 @@ public async Task ReceiveBroadcast( batch.Sender, options, receivedTimestamp, + allowDelta: disseminationNamespace.BroadcastsAreDeltas, cancellationToken); - namespaceKeys[item.Value.Key] = new(sentVersion, keyState.Applied || result is DisseminationApplyResult.Applied); + var accepted = result is DisseminationApplyResult.Applied or DisseminationApplyResult.Duplicate; + namespaceKeys[item.Value.Key] = new( + sentVersion, + keyState.Applied || result is DisseminationApplyResult.Applied, + accepted && (!existing || keyState.Accepted)); + if (disseminationNamespace.BroadcastsAreDeltas + && item.Value.FromVersion > 0 + && result == DisseminationApplyResult.Rejected) + { + // A missing broadcast baseline should participate in the next repair round. + lock (_valueUpdateLock) + { + _lastValueUpdates.Remove(new(namespaceName, item.Value.Key)); + } + } } } @@ -289,7 +304,7 @@ private static bool CanAcknowledgeTransmittedVersions( foreach (var (key, state) in keys) { // A different version needs an explicit acknowledgment to preserve the peer's exact repair baseline. - if (disseminationNamespace.GetVersion(key) != state.SentVersion) + if (!state.Accepted || disseminationNamespace.GetVersion(key) != state.SentVersion) { return false; } @@ -299,7 +314,7 @@ private static bool CanAcknowledgeTransmittedVersions( return true; } - private readonly record struct ReceivedKeyState(long SentVersion, bool Applied); + private readonly record struct ReceivedKeyState(long SentVersion, bool Applied, bool Accepted); public async Task RunAntiEntropyRound(CancellationToken cancellationToken) { @@ -627,6 +642,7 @@ await ApplyReceivedValue( response.Sender, options, receivedTimestamp, + allowDelta: false, cancellationToken); } } @@ -1075,6 +1091,7 @@ private async ValueTask ApplyReceivedValue( SiloAddress sender, DisseminationOptions options, long receivedTimestamp, + bool allowDelta, CancellationToken cancellationToken) { try @@ -1085,6 +1102,7 @@ private async ValueTask ApplyReceivedValue( sender, options, receivedTimestamp, + allowDelta, cancellationToken); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -1112,6 +1130,7 @@ private async ValueTask ApplyReceivedValueCore( SiloAddress sender, DisseminationOptions options, long receivedTimestamp, + bool allowDelta, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -1131,8 +1150,7 @@ private async ValueTask ApplyReceivedValueCore( return DisseminationApplyResult.Obsolete; } - // Only full values are supported; obsolete values need no deserialization unless the owner validates authority. - if (TryGetTerminalApplyResult(disseminationNamespace, item.Value, out var terminalResult)) + if (TryGetTerminalApplyResult(disseminationNamespace, item.Value, allowDelta, out var terminalResult)) { EmitApplyResult(namespaceName, item, sender, terminalResult); return terminalResult; @@ -1588,16 +1606,19 @@ private bool ValidateRepair( private static bool TryGetTerminalApplyResult( IDisseminationNamespace disseminationNamespace, DisseminationValue value, + bool allowDelta, out DisseminationApplyResult result) { - if (value.FromVersion != 0 || value.ToVersion <= 0) + if (value.ToVersion <= 0 || value.FromVersion < 0 + || value.FromVersion > 0 && (!allowDelta || value.FromVersion > value.ToVersion)) { result = DisseminationApplyResult.Rejected; return true; } var localVersion = disseminationNamespace.GetVersion(value.Key); - if (value.ToVersion < localVersion && !disseminationNamespace.ValidateOlderFullValues) + if (value.ToVersion < localVersion + && (value.FromVersion > 0 || !disseminationNamespace.ValidateOlderFullValues)) { result = DisseminationApplyResult.Obsolete; return true; diff --git a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs index 72ef3b3072c..00f11dd92ad 100644 --- a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs @@ -11,6 +11,8 @@ internal interface IDisseminationNamespace DisseminationRoutingMode RoutingMode => DisseminationRoutingMode.BroadcastTree; + bool BroadcastsAreDeltas => false; + TimeSpan AggregationPeriod => TimeSpan.FromSeconds(1); // Full values can be authority-refresh hints even when their numeric version is older. @@ -36,11 +38,21 @@ IEnumerable Keys DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request); + DisseminationRepairResult CreateBroadcast( + in DisseminationRepairRequest request, + DisseminationBroadcastState? baseline) => CreateRepair(request); + ValueTask ApplyValueAsync( DisseminationValue value, CancellationToken cancellationToken); } +// Owned by the namespace and retained with an acknowledged peer/key until that knowledge is pruned. +internal abstract class DisseminationBroadcastState(long version) +{ + public long Version { get; } = version; +} + internal enum DisseminationMembershipScope { ActiveMembers, @@ -69,11 +81,12 @@ internal readonly struct DisseminationRepairRequest( public int MaxPayloadBytes { get; } = maxPayloadBytes; } -// Version reports the current namespace version; a Produced result carries its full value. +// A produced result carries one full repair or broadcast update at Version. internal readonly struct DisseminationRepairResult( DisseminationRepairStatus status, long version, - DisseminationValue value) + DisseminationValue value, + DisseminationBroadcastState? broadcastState = null) { public DisseminationRepairStatus Status { get; } = status; @@ -81,11 +94,15 @@ internal readonly struct DisseminationRepairResult( public DisseminationValue Value { get; } = value; + public DisseminationBroadcastState? BroadcastState { get; } = broadcastState; + public static DisseminationRepairResult Current(long version) => new(DisseminationRepairStatus.Current, version, default); - public static DisseminationRepairResult Produced(DisseminationValue value) => - new(DisseminationRepairStatus.Produced, value.ToVersion, value); + public static DisseminationRepairResult Produced( + DisseminationValue value, + DisseminationBroadcastState? broadcastState = null) => + new(DisseminationRepairStatus.Produced, value.ToVersion, value, broadcastState); public static DisseminationRepairResult Unavailable(long version) => new(DisseminationRepairStatus.Unavailable, version, default); @@ -98,7 +115,7 @@ internal enum DisseminationRepairStatus { // The peer is already at or beyond the resolved version. Current, - // Value contains the current full value. + // Value contains the current repair or broadcast update. Produced, // The key has no current value. Unavailable, diff --git a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs index 7ba4cf935fa..6f4c480d2a7 100644 --- a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs @@ -1,3 +1,4 @@ +using System.Collections.Immutable; using Microsoft.Extensions.Options; using Orleans.Configuration; using Orleans.Runtime.MembershipService; @@ -5,7 +6,7 @@ namespace Orleans.Runtime.Dissemination; -// Full snapshots preserve the complete inventory and same-version heartbeat advances. +// Broadcasts compare immutable snapshots; repair carries the complete current inventory. internal sealed class MembershipDisseminationNamespace( IMembershipManager membershipManager, IOptionsMonitor options, @@ -13,13 +14,17 @@ internal sealed class MembershipDisseminationNamespace( { private static readonly DisseminationKey[] MembershipKeys = [DisseminationKey.Default]; private readonly object _cacheLock = new(); - private MembershipTableSnapshot? _cachedSnapshot; + private BroadcastState? _cachedState; private byte[]? _cachedPayload; + private BroadcastState? _cachedBroadcastBase; + private DisseminationValue _cachedBroadcast; public DisseminationNamespace Name => DisseminationNamespaceNames.Membership; public DisseminationMembershipScope MembershipScope => DisseminationMembershipScope.AllMembers; + public bool BroadcastsAreDeltas => true; + public bool ValidateOlderFullValues => true; public DisseminationNamespaceOptions Options => options.CurrentValue.Dissemination; @@ -67,21 +72,95 @@ public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest requ lock (_cacheLock) { // Re-read the owner under the cache lock: publication arguments can outlive the state they describe. - var snapshot = membershipManager.CurrentSnapshot; - var version = snapshot.Version.Value; - if (_cachedSnapshot is null || !MembershipSnapshotsEqual(_cachedSnapshot, snapshot)) - { - _cachedPayload = null; - } - - _cachedSnapshot = snapshot; - _cachedPayload ??= serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = snapshot }); + var state = GetCurrentState(); + var version = state.Version; + _cachedPayload ??= serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = state.Snapshot }); return _cachedPayload.Length <= request.MaxPayloadBytes && _cachedPayload.Length <= request.MaxBatchBytes ? DisseminationRepairResult.Produced(new DisseminationValue(DisseminationKey.Default, 0, version, _cachedPayload)) : DisseminationRepairResult.InsufficientCapacity(version); } } + public DisseminationRepairResult CreateBroadcast( + in DisseminationRepairRequest request, + DisseminationBroadcastState? baseline) + { + if (request.Key != DisseminationKey.Default) + { + return DisseminationRepairResult.Unavailable(version: 0); + } + + lock (_cacheLock) + { + var current = GetCurrentState(); + if (current.Version <= 0) + { + return DisseminationRepairResult.Unavailable(current.Version); + } + + if (request.FromVersion > current.Version) + { + return DisseminationRepairResult.Current(current.Version); + } + + // An empty current-view delta establishes a comparison baseline. Missing views use full repair. + var previous = baseline is null ? current : (BroadcastState)baseline; + if (previous.Version > current.Version) + { + return DisseminationRepairResult.Current(current.Version); + } + + if (!ReferenceEquals(_cachedBroadcastBase, previous)) + { + var updated = ImmutableArray.CreateBuilder(); + foreach (var (silo, entry) in current.Snapshot.Entries) + { + if (!previous.Snapshot.Entries.TryGetValue(silo, out var old) + || !MembershipEntriesEqual(old, entry)) + { + updated.Add(entry); + } + } + + var removed = ImmutableArray.CreateBuilder(); + foreach (var silo in previous.Snapshot.Entries.Keys) + { + if (!current.Snapshot.Entries.ContainsKey(silo)) + { + removed.Add(silo); + } + } + + var delta = new MembershipTableSnapshotDelta( + previous.Snapshot.Version, current.Snapshot.Version, + updated.ToImmutable(), removed.ToImmutable()); + _cachedBroadcast = new( + DisseminationKey.Default, previous.Version, current.Version, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Delta = delta })); + _cachedBroadcastBase = previous; + } + + return _cachedBroadcast.Payload.Length <= request.MaxPayloadBytes + && _cachedBroadcast.Payload.Length <= request.MaxBatchBytes + ? DisseminationRepairResult.Produced(_cachedBroadcast, current) + : DisseminationRepairResult.InsufficientCapacity(current.Version); + } + } + + private BroadcastState GetCurrentState() + { + var snapshot = membershipManager.CurrentSnapshot; + if (_cachedState is null || !MembershipSnapshotsEqual(_cachedState.Snapshot, snapshot)) + { + _cachedState = new(snapshot); + _cachedPayload = null; + _cachedBroadcastBase = null; + _cachedBroadcast = default; + } + + return _cachedState; + } + public async ValueTask ApplyValueAsync( DisseminationValue value, CancellationToken cancellationToken) @@ -92,31 +171,196 @@ public async ValueTask ApplyValueAsync( return DisseminationApplyResult.Rejected; } - if (value.FromVersion != 0 - || serializer.Deserialize(value.Payload)?.Snapshot is not { } snapshot - || value.ToVersion != snapshot.Version.Value) + if (serializer.Deserialize(value.Payload) is not { } update) { return DisseminationApplyResult.Rejected; } - // The membership manager merges maximum per-entry IAmAliveTime before publishing a full snapshot. - var currentSnapshot = membershipManager.CurrentSnapshot; - if (snapshot.Version == currentSnapshot.Version) + var previous = membershipManager.CurrentSnapshot; + MembershipTableSnapshot? snapshot; + IEnumerable heartbeats; + IEnumerable removed; + if (update.Delta is { } delta) { - // Same-version snapshots can advance liveness or remove inactive entries. - if (!snapshot.IsSuccessorTo(currentSnapshot)) + if (update.Snapshot is not null + || value.FromVersion != delta.BaseVersion.Value + || value.ToVersion != delta.Version.Value + || delta.BaseVersion.Value <= 0 + || delta.Version < delta.BaseVersion + || delta.UpdatedEntries.IsDefault || delta.RemovedSilos.IsDefault) + { + return DisseminationApplyResult.Rejected; + } + + if (delta.Version < previous.Version) { - return DisseminationApplyResult.Duplicate; + return DisseminationApplyResult.Obsolete; } + + if (previous.Version != delta.BaseVersion && previous.Version != delta.Version) + { + return DisseminationApplyResult.Rejected; + } + + snapshot = ApplyDelta(previous, delta); + heartbeats = delta.UpdatedEntries; + removed = delta.RemovedSilos; + } + else if (update.Snapshot is { } full + && value.FromVersion == 0 + && value.ToVersion == full.Version.Value) + { + snapshot = full.Version == previous.Version ? MergeSameVersion(previous, full) : full; + heartbeats = full.Entries.Values; + removed = full.Version == previous.Version + ? previous.Entries.Keys.Where(silo => !full.Entries.ContainsKey(silo)) + : []; + } + else + { + return DisseminationApplyResult.Rejected; + } + + if (snapshot is null) + { + return DisseminationApplyResult.Rejected; + } + + if (MembershipSnapshotsEqual(previous, snapshot) && CoversChanges(previous, heartbeats, removed)) + { + return DisseminationApplyResult.Duplicate; } await membershipManager.ProcessGossipSnapshot(snapshot, cancellationToken); - var result = MembershipSnapshotsEqual(currentSnapshot, membershipManager.CurrentSnapshot) + var current = membershipManager.CurrentSnapshot; + if (snapshot.Version < current.Version) + { + return DisseminationApplyResult.Obsolete; + } + + // A concurrent authority refresh can publish while the gossip call waits. Confirm the requested + // effects, rather than treating any unrelated owner change as acceptance of this update. + if (current.Version != snapshot.Version || !CoversChanges(current, heartbeats, removed)) + { + return DisseminationApplyResult.Rejected; + } + + return MembershipSnapshotsEqual(previous, current) ? DisseminationApplyResult.Duplicate : DisseminationApplyResult.Applied; - return snapshot.Version < currentSnapshot.Version && result == DisseminationApplyResult.Duplicate - ? DisseminationApplyResult.Obsolete - : result; + } + + private static MembershipTableSnapshot? ApplyDelta( + MembershipTableSnapshot current, + MembershipTableSnapshotDelta delta) + { + var entries = current.Entries.ToBuilder(); + var touched = new HashSet(); + var sameVersion = current.Version == delta.Version; + foreach (var entry in delta.UpdatedEntries) + { + if (entry?.SiloAddress is not { } silo || !touched.Add(silo)) + { + return null; + } + + if (sameVersion) + { + if (entries.TryGetValue(silo, out var existing)) + { + entries[silo] = MergeHeartbeat(existing, entry); + } + else if (entry.Status == SiloStatus.Active) + { + return null; + } + } + else + { + entries[silo] = entries.TryGetValue(silo, out var existing) + ? MergeHeartbeat(entry, existing) + : entry; + } + } + + foreach (var silo in delta.RemovedSilos) + { + if (silo is null || !touched.Add(silo) + || sameVersion && entries.TryGetValue(silo, out var existing) && existing.Status == SiloStatus.Active) + { + return null; + } + + entries.Remove(silo); + } + + return new(delta.Version, entries.ToImmutable()); + } + + private static MembershipTableSnapshot? MergeSameVersion( + MembershipTableSnapshot current, + MembershipTableSnapshot incoming) + { + var entries = current.Entries.ToBuilder(); + foreach (var (silo, entry) in current.Entries) + { + if (incoming.Entries.TryGetValue(silo, out var update)) + { + entries[silo] = MergeHeartbeat(entry, update); + } + else if (entry.Status == SiloStatus.Active) + { + return null; + } + else + { + entries.Remove(silo); + } + } + + foreach (var (silo, entry) in incoming.Entries) + { + if (entry.Status == SiloStatus.Active && !current.Entries.ContainsKey(silo)) + { + return null; + } + } + + return new(current.Version, entries.ToImmutable()); + } + + private static MembershipEntry MergeHeartbeat(MembershipEntry current, MembershipEntry incoming) => + incoming.IAmAliveTime > current.IAmAliveTime + ? current.WithIAmAliveTime(incoming.IAmAliveTime) + : current; + + private static bool CoversChanges( + MembershipTableSnapshot current, + IEnumerable updated, + IEnumerable removed) + { + foreach (var entry in updated) + { + if (current.Entries.TryGetValue(entry.SiloAddress, out var existing)) + { + if (existing.EffectiveIAmAliveTime < entry.IAmAliveTime) + { + return false; + } + } + else if (entry.Status == SiloStatus.Active) + { + return false; + } + } + + return !removed.Any(current.Entries.ContainsKey); + } + + private sealed class BroadcastState(MembershipTableSnapshot snapshot) + : DisseminationBroadcastState(snapshot.Version.Value) + { + public MembershipTableSnapshot Snapshot { get; } = snapshot; } private static long GetFingerprint(MembershipTableSnapshot snapshot) @@ -170,4 +414,27 @@ internal sealed class MembershipTableSnapshotUpdate { [Id(0)] public MembershipTableSnapshot? Snapshot { get; init; } + + [Id(1)] + public MembershipTableSnapshotDelta? Delta { get; init; } +} + +[GenerateSerializer, Immutable] +internal sealed class MembershipTableSnapshotDelta( + MembershipVersion baseVersion, + MembershipVersion version, + ImmutableArray updatedEntries, + ImmutableArray removedSilos) +{ + [Id(0)] + public MembershipVersion BaseVersion { get; } = baseVersion; + + [Id(1)] + public MembershipVersion Version { get; } = version; + + [Id(2)] + public ImmutableArray UpdatedEntries { get; } = updatedEntries; + + [Id(3)] + public ImmutableArray RemovedSilos { get; } = removedSilos; } diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipDeltas.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipDeltas.cs new file mode 100644 index 00000000000..2cdc6cf7952 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipDeltas.cs @@ -0,0 +1,523 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public void MembershipBroadcastDeltaIsSparseWhileRepairRemainsFull() + { + var members = Enumerable.Range(43000, 2000).Select(CreateSilo).ToArray(); + var initial = CreateMembershipSnapshot(1, members.Select(silo => + CreateMembershipEntry(silo, SiloStatus.Active, DateTime.UnixEpoch)).ToArray()); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(initial); + var ns = CreateMembershipNamespace(manager, serializer); + var request = new DisseminationRepairRequest(DisseminationKey.Default, 1, 1024 * 1024, 1024 * 1024); + var probe = ns.CreateBroadcast(request, baseline: null); + Assert.Equal(DisseminationRepairStatus.Produced, probe.Status); + Assert.Empty(ReadMembershipDelta(serializer, probe.Value).UpdatedEntries); + var baseline = Assert.IsAssignableFrom(probe.BroadcastState); + manager.CurrentSnapshot = new(new MembershipVersion(2), + initial.Entries.SetItem(members[999], initial.Entries[members[999]].WithStatus(SiloStatus.ShuttingDown))); + + var broadcast = ns.CreateBroadcast(request, baseline); + var full = ns.CreateRepair(request); + + Assert.Equal(DisseminationRepairStatus.Produced, broadcast.Status); + Assert.Equal(DisseminationRepairStatus.Produced, full.Status); + var delta = ReadMembershipDelta(serializer, broadcast.Value); + Assert.Equal((1L, 2L), (broadcast.Value.FromVersion, broadcast.Value.ToVersion)); + var changed = Assert.Single(delta.UpdatedEntries); + Assert.Equal(members[999], changed.SiloAddress); + Assert.Equal(SiloStatus.ShuttingDown, changed.Status); + Assert.Empty(delta.RemovedSilos); + Assert.Equal(0, full.Value.FromVersion); + var repair = Assert.IsType(serializer.Deserialize(full.Value.Payload)); + Assert.Null(repair.Delta); + Assert.Equal(2000, Assert.IsType(repair.Snapshot).Entries.Count); + Assert.True(broadcast.Value.Payload.Length * 10 < full.Value.Payload.Length, + $"Sparse broadcast {broadcast.Value.Payload.Length} bytes, full repair {full.Value.Payload.Length} bytes."); + Assert.Equal(broadcast.Value.Payload, ns.CreateBroadcast(request, baseline).Value.Payload); + } + + [Fact] + public async Task MembershipBroadcastCoalescesFromAcknowledgedSnapshot() + { + var local = CreateSilo(45001); + var peer = CreateSilo(45002); + var removed = CreateSilo(45003); + var added = CreateSilo(45004); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(removed, SiloStatus.Dead, DateTime.UnixEpoch)); + await using var link = new MembershipDeltaLink(local, peer, initial, initial); + await link.Send(); + // Metadata can return to its prior value across skipped observations; compare with the actual baseline. + var temporaryEntry = CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch); + temporaryEntry.HostName = "temporary"; + var temporary = new MembershipTableSnapshot(new MembershipVersion(2), + initial.Entries.SetItem(local, temporaryEntry)); + var final = new MembershipTableSnapshot(new MembershipVersion(3), + initial.Entries.Remove(removed) + .SetItem(peer, initial.Entries[peer].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))) + .Add(added, CreateMembershipEntry(added, SiloStatus.Joining, DateTime.UnixEpoch))); + var flush = BeforeBroadcastPumpsRun(() => + { + link.SourceManager.CurrentSnapshot = temporary; + Assert.True(link.Queue.Notify(peer, link.Source, DisseminationKey.Default)); + link.SourceManager.CurrentSnapshot = final; + Assert.True(link.Queue.Notify(peer, link.Source, DisseminationKey.Default)); + return link.Queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + }); + + await flush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(2, link.Batches.Count); + var value = Assert.Single(GetBroadcastValues(link.Batches[1])).Value; + var delta = ReadMembershipDelta(link.Serializer, value); + Assert.Equal((1L, 3L), (value.FromVersion, value.ToVersion)); + Assert.Equal(new[] { peer, added }, delta.UpdatedEntries.Select(entry => entry.SiloAddress).Order()); + Assert.Equal(removed, Assert.Single(delta.RemovedSilos)); + AssertMembershipState(final, link.TargetManager.CurrentSnapshot); + } + + [Fact] + public async Task MembershipBroadcastCapturesBaselineBeforeAcknowledgment() + { + var local = CreateSilo(45101); + var peer = CreateSilo(45102); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + await using var link = new MembershipDeltaLink(local, peer, initial, initial); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + link.BeforeResponse = async (_, index, token) => + { + if (index == 1) + { + entered.TrySetResult(); + await release.Task.WaitAsync(token); + } + }; + try + { + Assert.True(link.Queue.Notify(peer, link.Source, DisseminationKey.Default)); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + link.SourceManager.CurrentSnapshot = new(initial.Version, initial.Entries.SetItem( + local, initial.Entries[local].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10)))); + Assert.True(link.Queue.Notify(peer, link.Source, DisseminationKey.Default)); + var flush = link.Queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + release.TrySetResult(); + await flush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(2, link.Batches.Count); + var delta = ReadMembershipDelta(link.Serializer, Assert.Single(GetBroadcastValues(link.Batches[1])).Value); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(10), Assert.Single(delta.UpdatedEntries).IAmAliveTime); + AssertMembershipState(link.SourceManager.CurrentSnapshot, link.TargetManager.CurrentSnapshot); + } + finally + { + release.TrySetResult(); + } + } + + [Fact] + public async Task MembershipBroadcastGapUsesFullAntiEntropyRepair() + { + var local = CreateSilo(45201); + var peer = CreateSilo(45202); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + var source = new MembershipTableSnapshot(new MembershipVersion(3), + initial.Entries.SetItem(local, initial.Entries[local].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10)))); + await using var link = new MembershipDeltaLink(local, peer, source, initial); + await link.Send(); + var probe = Assert.Single(GetBroadcastValues(Assert.Single(link.Batches))).Value; + Assert.Equal((3L, 3L), (probe.FromVersion, probe.ToVersion)); + Assert.Equal(1, link.TargetManager.CurrentSnapshot.Version.Value); + Assert.False(Assert.Single(link.Responses).AllVersionsAcknowledged); + + var repair = await link.SourceProtocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + SupportedNamespaces = [link.Source.Name], + Digests = new() { [link.Source.Name] = [.. link.Target.Digests] }, + }, TestContext.Current.CancellationToken); + var full = Assert.Single(repair.Values[link.Source.Name]).Value; + Assert.Equal(0, full.FromVersion); + Assert.Null(Assert.IsType( + link.Serializer.Deserialize(full.Payload)).Delta); + Assert.Equal(DisseminationApplyResult.Applied, + await link.Target.ApplyValueAsync(full, TestContext.Current.CancellationToken)); + await link.Send(); + + AssertMembershipState(source, link.TargetManager.CurrentSnapshot); + Assert.All(link.Batches.SelectMany(GetBroadcastValues), item => Assert.True(item.Value.FromVersion > 0)); + Assert.True(link.Responses[^1].AllVersionsAcknowledged); + } + + [Fact] + public async Task MembershipDeltaRetransmitsCumulativeChangesAfterLostAcknowledgment() + { + var local = CreateSilo(45601); + var peer = CreateSilo(45602); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + await using var link = new MembershipDeltaLink(local, peer, initial, initial); + await link.Send(); + link.BeforeResponse = (_, index, _) => index == 2 + ? Task.FromException(new InvalidOperationException("Acknowledgment lost.")) + : Task.CompletedTask; + link.SourceManager.CurrentSnapshot = new(initial.Version, initial.Entries.SetItem( + local, initial.Entries[local].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10)))); + await link.Send(); + link.SourceManager.CurrentSnapshot = new(initial.Version, link.SourceManager.CurrentSnapshot.Entries.SetItem( + peer, initial.Entries[peer].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(20)))); + + await link.Send(); + + Assert.Equal(3, link.Batches.Count); + var delta = ReadMembershipDelta(link.Serializer, Assert.Single(GetBroadcastValues(link.Batches[2])).Value); + Assert.Equal(new[] { local, peer }, delta.UpdatedEntries.Select(entry => entry.SiloAddress).Order()); + AssertMembershipState(link.SourceManager.CurrentSnapshot, link.TargetManager.CurrentSnapshot); + } + + [Fact] + public async Task AheadMembershipRelayBroadcastsLatestDeltaFromChildBaseline() + { + var root = CreateSilo(45801); + var relay = CreateSilo(45802); + var child = CreateSilo(45803); + var members = new[] { root, relay, child }; + var initial = CreateMembershipSnapshot(1, members.Select(silo => + CreateMembershipEntry(silo, SiloStatus.Active, DateTime.UnixEpoch)).ToArray()); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var managers = members.ToDictionary(silo => silo, _ => new FakeMembershipManager(initial)); + var namespaces = members.ToDictionary(silo => silo, silo => CreateMembershipNamespace(managers[silo], serializer)); + var transports = members.ToDictionary(silo => silo, silo => new FakeTransport(silo, members.Where(other => !other.Equals(silo)).ToArray())); + var protocols = members.ToDictionary(silo => silo, silo => CreateProtocol( + transports[silo], [namespaces[silo]], + options => options.Overlay.FanOutFactor = static _ => 1)); + var sent = new Dictionary> + { + [root] = [], + [relay] = [], + [child] = [], + }; + foreach (var silo in members) + { + transports[silo].SendBroadcastResponseHandler = (peer, batch, token) => + { + sent[silo].Add(batch); + return protocols[peer].ReceiveBroadcast(batch, token); + }; + } + + var token = TestContext.Current.CancellationToken; + try + { + Assert.True(await protocols[root].Publish(namespaces[root], DisseminationKey.Default, 1, token)); + await protocols[root].FlushPendingBroadcast(token); + await protocols[relay].FlushPendingBroadcast(token); + Assert.Single(sent[root]); + Assert.Single(sent[relay]); + + var second = new MembershipTableSnapshot(new MembershipVersion(2), + initial.Entries.SetItem(root, initial.Entries[root].WithStatus(SiloStatus.ShuttingDown))); + var third = new MembershipTableSnapshot(new MembershipVersion(3), + initial.Entries.SetItem(root, initial.Entries[root].WithStatus(SiloStatus.Stopping))); + managers[relay].CurrentSnapshot = third; + managers[root].CurrentSnapshot = second; + Assert.True(await protocols[root].Publish(namespaces[root], DisseminationKey.Default, 2, token)); + await protocols[root].FlushPendingBroadcast(token); + await protocols[relay].FlushPendingBroadcast(token); + + Assert.Equal(2, sent[root].Count); + Assert.Equal(2, sent[relay].Count); + var rootDelta = ReadMembershipDelta(serializer, Assert.Single(GetBroadcastValues(sent[root][1])).Value); + var relayDelta = ReadMembershipDelta(serializer, Assert.Single(GetBroadcastValues(sent[relay][1])).Value); + Assert.Equal((1L, 2L), (rootDelta.BaseVersion.Value, rootDelta.Version.Value)); + Assert.Equal((1L, 3L), (relayDelta.BaseVersion.Value, relayDelta.Version.Value)); + Assert.Equal(SiloStatus.Stopping, Assert.Single(relayDelta.UpdatedEntries).Status); + AssertMembershipState(third, managers[child].CurrentSnapshot); + Assert.Empty(sent[child]); + Assert.All(transports.Values, transport => Assert.Empty(transport.AntiEntropyRequests)); + } + finally + { + var canceled = new CancellationToken(canceled: true); + try + { + await Task.WhenAll(protocols.Values.Select(protocol => protocol.StopAsync(canceled))) + .WaitAsync(TimeSpan.FromSeconds(5)); + } + catch (OperationCanceledException) when (canceled.IsCancellationRequested) + { + } + } + } + + [Fact] + public async Task MembershipDeltaRejectsIntermediateViewAndMergesTargetReplay() + { + var local = CreateSilo(45301); + var inactive = CreateSilo(45302); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(inactive, SiloStatus.Dead, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(new MembershipTableSnapshot(new MembershipVersion(2), initial.Entries)); + var ns = CreateMembershipNamespace(manager, serializer); + var updated = initial.Entries[local].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10)); + var value = CreateMembershipDelta(serializer, 1, 3, [updated], [inactive]); + + Assert.Equal(DisseminationApplyResult.Rejected, + await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken)); + Assert.Equal(2, manager.CurrentSnapshot.Version.Value); + Assert.Contains(inactive, manager.CurrentSnapshot.Entries.Keys); + + var canonical = initial.Entries[local].WithStatus(SiloStatus.ShuttingDown); + canonical.HostName = "canonical"; + manager.CurrentSnapshot = CreateMembershipSnapshot(3, canonical); + Assert.Equal(DisseminationApplyResult.Applied, + await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken)); + Assert.Equal(SiloStatus.ShuttingDown, manager.CurrentSnapshot.Entries[local].Status); + Assert.Equal("canonical", manager.CurrentSnapshot.Entries[local].HostName); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(10), manager.CurrentSnapshot.Entries[local].IAmAliveTime); + Assert.DoesNotContain(inactive, manager.CurrentSnapshot.Entries.Keys); + Assert.Equal(DisseminationApplyResult.Duplicate, + await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task MembershipFullRepairMergesIncomparableCleanupAndHeartbeatState() + { + var local = CreateSilo(45401); + var first = CreateSilo(45402); + var second = CreateSilo(45403); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var sourceManager = new FakeMembershipManager(CreateMembershipSnapshot(7, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(30)), + CreateMembershipEntry(first, SiloStatus.Dead, DateTime.UnixEpoch))); + var receiverManager = new FakeMembershipManager(CreateMembershipSnapshot(7, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(40)), + CreateMembershipEntry(second, SiloStatus.Dead, DateTime.UnixEpoch))); + var source = CreateMembershipNamespace(sourceManager, serializer); + var receiver = CreateMembershipNamespace(receiverManager, serializer); + + Assert.Equal(DisseminationApplyResult.Applied, + await receiver.ApplyValueAsync(GetMembershipRepair(source, 7), TestContext.Current.CancellationToken)); + Assert.Equal(local, Assert.Single(receiverManager.CurrentSnapshot.Entries).Key); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(40), receiverManager.CurrentSnapshot.Entries[local].IAmAliveTime); + Assert.Equal(DisseminationApplyResult.Applied, + await source.ApplyValueAsync(GetMembershipRepair(receiver, 7), TestContext.Current.CancellationToken)); + AssertMembershipState(receiverManager.CurrentSnapshot, sourceManager.CurrentSnapshot); + } + + [Fact] + public async Task MembershipDeltaDoesNotAcknowledgeUnrelatedOwnerRefresh() + { + var local = CreateSilo(45501); + var initial = CreateMembershipSnapshot(1, CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(initial); + manager.ProcessGossipSnapshotHandler = (_, _) => + { + manager.CurrentSnapshot = new(new MembershipVersion(2), initial.Entries); + return Task.CompletedTask; + }; + var ns = CreateMembershipNamespace(manager, serializer); + var value = CreateMembershipDelta(serializer, 1, 2, + [initial.Entries[local].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))], []); + + Assert.Equal(DisseminationApplyResult.Rejected, + await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken)); + Assert.Equal(DateTime.UnixEpoch, manager.CurrentSnapshot.Entries[local].IAmAliveTime); + + var peer = CreateSilo(45502); + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns]); + try + { + var response = await protocol.ReceiveBroadcast(new() + { + Sender = peer, + SupportsCompactAcknowledgments = true, + Values = new() { [ns.Name] = [new() { Value = value, TimeToLive = TimeSpan.FromSeconds(30) }] }, + }, TestContext.Current.CancellationToken); + Assert.False(response.AllVersionsAcknowledged); + Assert.Equal(2, Assert.Single(response.Acknowledgments[ns.Name]).Version); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task MembershipAntiEntropyRejectsDeltaResponses() + { + var local = CreateSilo(45701); + var peer = CreateSilo(45702); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(initial); + var applied = 0; + manager.ProcessGossipSnapshotHandler = (_, _) => + { + applied++; + return Task.CompletedTask; + }; + var ns = CreateMembershipNamespace(manager, serializer); + var delta = CreateMembershipDelta(serializer, 1, 2, + [initial.Entries[peer].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))], []); + var transport = new FakeTransport(local, peer); + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = new() { [ns.Name] = [new() { Value = delta, TimeToLive = TimeSpan.FromSeconds(30) }] }, + }); + var protocol = CreateProtocol(transport, [ns]); + try + { + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Single(transport.AntiEntropyRequests); + Assert.Equal(0, applied); + Assert.Same(initial, manager.CurrentSnapshot); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + private static MembershipTableSnapshotDelta ReadMembershipDelta(Serializer serializer, DisseminationValue value) + { + var update = Assert.IsType(serializer.Deserialize(value.Payload)); + Assert.Null(update.Snapshot); + var delta = Assert.IsType(update.Delta); + Assert.Equal(value.FromVersion, delta.BaseVersion.Value); + Assert.Equal(value.ToVersion, delta.Version.Value); + return delta; + } + + private static DisseminationValue CreateMembershipDelta( + Serializer serializer, + long from, + long to, + ImmutableArray entries, + ImmutableArray removed) => + new(DisseminationKey.Default, from, to, serializer.SerializeToArray(new MembershipTableSnapshotUpdate + { + Delta = new(new MembershipVersion(from), new MembershipVersion(to), entries, removed), + })); + + private sealed class MembershipDeltaLink : IAsyncDisposable + { + private readonly ServiceProvider _services; + private readonly SiloAddress _peer; + private readonly DisseminationProtocol _targetProtocol; + + public MembershipDeltaLink( + SiloAddress local, + SiloAddress peer, + MembershipTableSnapshot source, + MembershipTableSnapshot target) + { + _peer = peer; + _services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + Serializer = _services.GetRequiredService(); + SourceManager = new(source); + TargetManager = new(target); + Source = CreateMembershipNamespace(SourceManager, Serializer); + Target = CreateMembershipNamespace(TargetManager, Serializer); + var transport = new FakeTransport(local, peer); + SourceProtocol = CreateProtocol(transport, [Source], timeProvider: Clock); + _targetProtocol = CreateProtocol(new FakeTransport(peer, local), [Target], timeProvider: Clock); + transport.SendBroadcastResponseHandler = async (_, batch, token) => + { + Batches.Add(batch); + var response = await _targetProtocol.ReceiveBroadcast(batch, token); + Responses.Add(response); + if (BeforeResponse is { } before) + { + await before(batch, Batches.Count, token); + } + + return response; + }; + Queue = CreateBroadcastQueue(transport, [Source], timeProvider: Clock); + } + + public Serializer Serializer { get; } + public FakeTimeProvider Clock { get; } = new(); + public FakeMembershipManager SourceManager { get; } + public FakeMembershipManager TargetManager { get; } + public MembershipDisseminationNamespace Source { get; } + public MembershipDisseminationNamespace Target { get; } + public DisseminationProtocol SourceProtocol { get; } + public DisseminationBroadcastQueue Queue { get; } + public List Batches { get; } = []; + public List Responses { get; } = []; + public Func? BeforeResponse { get; set; } + + public async Task Send() + { + var token = TestContext.Current.CancellationToken; + var flush = BeforeBroadcastPumpsRun(() => + { + Assert.True(Queue.Notify(_peer, Source, DisseminationKey.Default)); + return Queue.FlushPendingBroadcast(token); + }); + await flush.WaitAsync(TimeSpan.FromSeconds(5), token); + } + + public async ValueTask DisposeAsync() + { + var token = new CancellationToken(canceled: true); + var stops = new[] + { + Queue.StopAsync(token), + SourceProtocol.StopAsync(token), + _targetProtocol.StopAsync(token), + }; + try + { + await Task.WhenAll(stops).WaitAsync(TimeSpan.FromSeconds(5)); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + } + finally + { + _services.Dispose(); + } + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs index 2a8782e666d..b2709379711 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -2254,7 +2254,7 @@ public async Task MembershipAntiEntropyRepairsSameVersionSuccessor() } [Fact] - public async Task MembershipNamespacePublishesIndependentlyApplicableFullSnapshots() + public async Task MembershipPublicationValidationProducesIndependentlyApplicableFullRepairs() { var local = CreateSilo(11111); var peer = CreateSilo(11112); @@ -2324,7 +2324,7 @@ public async Task MembershipNamespacePublishesIndependentlyApplicableFullSnapsho [Theory] [InlineData(false, 2)] [InlineData(true, 3)] - public async Task MembershipNamespacePublishesCurrentSnapshotWhenNotificationIsSuperseded(bool baselineAccepted, long nextVersion) + public async Task MembershipPublicationValidationUsesCurrentSnapshotWhenNotificationIsSuperseded(bool baselineAccepted, long nextVersion) { var local = CreateSilo(11111); var peer = CreateSilo(11112); @@ -2360,7 +2360,7 @@ public async Task MembershipNamespacePublishesCurrentSnapshotWhenNotificationIsS } [Fact] - public async Task MembershipNamespacePublishesFullInventoryWhenTopologyChanges() + public async Task MembershipPublicationValidationIncludesFullInventoryWhenTopologyChanges() { var local = CreateSilo(11111); var firstPeer = CreateSilo(11112); @@ -6458,7 +6458,7 @@ public async Task MembershipDuplicatesStopForwardingWhileNewLivenessStillPropaga var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var sends = 0; - transport.SendBroadcastHandler = async (peer, batch, cancellationToken) => + transport.SendBroadcastResponseHandler = async (peer, batch, cancellationToken) => { lock (transport.BroadcastBatches) { @@ -6473,6 +6473,12 @@ public async Task MembershipDuplicatesStopForwardingWhileNewLivenessStillPropaga await releaseSend.Task.WaitAsync(cancellationToken); } } + + return new DisseminationBroadcastResponse + { + Acknowledgments = FakeTransport.CreateAcknowledgment(batch).Acknowledgments, + AllVersionsAcknowledged = true, + }; }; var protocol = CreateProtocol( transport, @@ -6509,8 +6515,10 @@ public async Task MembershipDuplicatesStopForwardingWhileNewLivenessStillPropaga var forwarded = Assert.Single(GetBroadcastValues(transport.BroadcastBatches[1].Batch)); var update = Assert.IsType( serializer.Deserialize(forwarded.Value.Payload)); - Assert.NotNull(update.Snapshot); - Assert.Equal(DateTime.UnixEpoch.AddSeconds(2), update.Snapshot.Entries[sender].IAmAliveTime); + Assert.Null(update.Snapshot); + var delta = Assert.IsType(update.Delta); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(2), + Assert.Single(delta.UpdatedEntries, entry => entry.SiloAddress.Equals(sender)).IAmAliveTime); Assert.Equal(1, forwarded.Value.ToVersion); } finally From 4f77c61a4169225d8c2b5d0ff602c226e9c8df35 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 17 Sep 2026 14:30:12 -0700 Subject: [PATCH 47/50] fix(dissemination): preserve canonical views and Dead-row pruning --- .../implementation/runtime-dissemination.md | 10 +- .../IMembershipTable.cs | 4 +- .../MembershipDisseminationNamespace.cs | 33 ++-- ...isseminationProtocolTests.FinalComments.cs | 19 ++- ...eminationProtocolTests.MembershipDeltas.cs | 155 +++++++++++++++++- ...eminationProtocolTests.MembershipReview.cs | 91 +++++----- 6 files changed, 230 insertions(+), 82 deletions(-) diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md index f5fca428a11..f9df8b461dc 100644 --- a/docs/site/src/content/docs/implementation/runtime-dissemination.md +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -56,15 +56,15 @@ Namespaces construct broadcasts separately from full anti-entropy repairs. Each Broadcast senders advertise support for compact acknowledgments. The compact flag requires complete receive admission, accepted or duplicate application results, and exact agreement with each key's transmitted version. Namespace entries retain capability evidence with empty per-key lists. An exact compact acknowledgment also installs the membership comparison snapshot captured when constructing that send. Explicit higher versions and passive peer knowledge advance version evidence; a comparison snapshot is usable only while its version matches that evidence. -Membership broadcasts contain changed entries and removed silo identities, together with base and target canonical view versions. A receiver at the base applies the transition. A receiver already at the target preserves its canonical fields, merges maximum heartbeat timestamps, and applies inactive-entry removals. Other versions require full repair. A rejected delta makes the key eligible for the next anti-entropy round, even if its local version advanced recently. Relays construct their own sparse update against each child's comparison snapshot, so an ahead relay can distribute its newer accepted view rather than regress to an older incoming update. +Membership broadcasts contain changed entries and removed silo identities, together with base and target canonical view versions. A receiver at the base applies the transition. A receiver already at the target preserves canonical fields and every non-Dead row, merges maximum heartbeat timestamps, and prunes explicitly removed Dead rows. A coalesced delta across versions can span an intervening Dead transition and cleanup. Other receiver versions require full repair. A rejected delta makes the key eligible for the next anti-entropy round, even if its local version advanced recently. Relays construct their own sparse update against each child's comparison snapshot, so an ahead relay can distribute its newer accepted view rather than regress to an older incoming update. -A peer without a usable comparison snapshot receives an empty delta at the sender's current version. A peer already at that version acknowledges the probe; a lagging peer obtains a full snapshot through repair or ordinary membership bootstrap. The retained snapshot establishes a canonical comparison baseline, while independent initial heartbeat and inactive-cleanup differences remain repair work. +A peer without a usable comparison snapshot receives an empty delta at the sender's current version. A peer already at that version acknowledges the probe; a lagging peer obtains a full snapshot through repair or ordinary membership bootstrap. The retained snapshot establishes a canonical comparison baseline, while initial heartbeat and retained-Dead differences remain repair work. -Membership anti-entropy uses the view version and a liveness/inventory fingerprint. Responses always contain full current snapshots, whose serialized payloads are cached until owner state changes. Same-version repair preserves canonical fields, merges maximum heartbeats, and intersects inactive inventories so independently cleaned entries stay removed. Application checks the requested effects against resulting owner state before acknowledging acceptance, including when an authority refresh completes concurrently. +Membership anti-entropy uses the view version and a liveness/inventory fingerprint. Responses always contain full current snapshots, whose serialized payloads are cached until owner state changes. Same-version repair preserves canonical fields and non-Dead entries, merges maximum heartbeats, and reconciles independently pruned Dead rows without resurrecting them. Application checks the requested effects against resulting owner state before acknowledging acceptance, including when an authority refresh completes concurrently. Comparison snapshots are shared immutable objects, with one retained reference per existing outbound membership peer/key and constant-size namespace caches. They are pruned with peer and key knowledge. Healthy peers share the same snapshot; lagging peers can retain different snapshots, giving a worst-case memory cost proportional to outbound peers times membership size. Sparse payload caching uses snapshot identity/content, including same-version changes. -Canonical membership versions are monotonic within a cluster lifetime; hosts can skip observed versions. Development clustering's primary table defines that lifetime. Loss of that authority invalidates the continuing cluster, and an observed table rollback requests fatal silo termination. Full repair follows the membership owner's lifetime and validation rules. Deployment-load samples retain their timestamp ordering. +Canonical membership versions are monotonic within a cluster lifetime; hosts can skip observed versions. Same-version snapshots can differ in maximum heartbeat progress and retention of Dead rows. Removing any other status requires a canonical view advance. Provider versioned compaction is a stronger storage contract and remains independently applicable. Development clustering's primary table defines that lifetime. Loss of that authority invalidates the continuing cluster, and an observed table rollback requests fatal silo termination. Full repair follows the membership owner's lifetime and validation rules. Deployment-load samples retain their timestamp ordering. Deployment-load versions are sample timestamps. Each repair is a full latest value. Application runs on the deployment-load publisher's scheduler and accepts samples only for Active silo generations in the membership oracle. Terminal membership transitions remove placement statistics, and the oracle's monotonic membership state rejects later samples for departed generations. @@ -99,7 +99,7 @@ Namespaces select a membership scope before topology construction: Membership fanout is derived from the target hop count and configured bounds, or selected by the code-configured callback. Aggregation uses its independent fanout setting. At 2,000 silos, fanout eight gives at most four distribution hops. A membership change creates a new topology from the next snapshot; acknowledged ledgers and anti-entropy repair convergence across the transition. -The projection cache tracks its authoritative source snapshot. On a cache miss it re-reads the owner under the cache lock, so a delayed reader follows the current view, including an owner-authorized version replacement. Eligible membership and status changes refresh routing and peer selection. Heartbeat-only updates and cleanup of non-participants reuse the existing topology, while rebuilt projections retain rotation progress. +The projection cache tracks its authoritative source snapshot. On a cache miss it re-reads the owner under the cache lock, so a delayed reader follows the current view. Canonical view changes refresh routing and peer selection. Heartbeat-only updates reuse the existing topology, while rebuilt projections retain rotation progress. ## Configuration and defaults diff --git a/src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs b/src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs index 7b2681ae9ae..3e8b898d776 100644 --- a/src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs +++ b/src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs @@ -71,8 +71,8 @@ async Task DeleteMembershipTableEntriesAsync(string clusterId, CancellationToken /// Cleans up defunct silo entries older than . /// /// - /// Providers can remove expired inactive entries without advancing the table version. - /// Active entries are retained. + /// Cleanup can remove retained Dead entries at the same membership version. + /// All other entries remain part of that canonical view. /// /// The exclusive upper bound for the last known update time of entries to delete. /// A task representing the cleanup operation. diff --git a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs index 6f4c480d2a7..aa482646053 100644 --- a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs @@ -270,7 +270,7 @@ public async ValueTask ApplyValueAsync( { entries[silo] = MergeHeartbeat(existing, entry); } - else if (entry.Status == SiloStatus.Active) + else if (entry.Status != SiloStatus.Dead) { return null; } @@ -286,7 +286,7 @@ public async ValueTask ApplyValueAsync( foreach (var silo in delta.RemovedSilos) { if (silo is null || !touched.Add(silo) - || sameVersion && entries.TryGetValue(silo, out var existing) && existing.Status == SiloStatus.Active) + || sameVersion && entries.TryGetValue(silo, out var existing) && existing.Status != SiloStatus.Dead) { return null; } @@ -308,7 +308,7 @@ public async ValueTask ApplyValueAsync( { entries[silo] = MergeHeartbeat(entry, update); } - else if (entry.Status == SiloStatus.Active) + else if (entry.Status != SiloStatus.Dead) { return null; } @@ -320,7 +320,7 @@ public async ValueTask ApplyValueAsync( foreach (var (silo, entry) in incoming.Entries) { - if (entry.Status == SiloStatus.Active && !current.Entries.ContainsKey(silo)) + if (entry.Status != SiloStatus.Dead && !current.Entries.ContainsKey(silo)) { return null; } @@ -343,12 +343,12 @@ private static bool CoversChanges( { if (current.Entries.TryGetValue(entry.SiloAddress, out var existing)) { - if (existing.EffectiveIAmAliveTime < entry.IAmAliveTime) + if (existing.IAmAliveTime < entry.IAmAliveTime) { return false; } } - else if (entry.Status == SiloStatus.Active) + else if (entry.Status != SiloStatus.Dead) { return false; } @@ -372,15 +372,28 @@ private static long GetFingerprint(MembershipTableSnapshot snapshot) foreach (var entry in snapshot.Entries.OrderBy(static entry => entry.Key)) { hash = unchecked((hash ^ (uint)entry.Key.GetConsistentHashCode()) * prime); - hash = unchecked((hash ^ (ulong)entry.Value.EffectiveIAmAliveTime.Ticks) * prime); + hash = unchecked((hash ^ (ulong)entry.Value.IAmAliveTime.Ticks) * prime); } return unchecked((long)hash); } + // Comparing cross-version snapshots identifies the entries to include in a sparse broadcast. private static bool MembershipEntriesEqual(MembershipEntry left, MembershipEntry right) => - MembershipTableSnapshot.AreVersionedFieldsEqual(left, right) - && left.IAmAliveTime == right.IAmAliveTime; + ReferenceEquals(left, right) + || left.SiloAddress.Equals(right.SiloAddress) + && left.Status == right.Status + && left.ProxyPort == right.ProxyPort + && string.Equals(left.HostName, right.HostName, StringComparison.Ordinal) + && string.Equals(left.SiloName, right.SiloName, StringComparison.Ordinal) + && string.Equals(left.RoleName, right.RoleName, StringComparison.Ordinal) + && left.UpdateZone == right.UpdateZone + && left.FaultZone == right.FaultZone + && left.StartTime == right.StartTime + && left.IAmAliveTime == right.IAmAliveTime + && (ReferenceEquals(left.SuspectTimes, right.SuspectTimes) + || left.SuspectTimes is not null && right.SuspectTimes is not null + && left.SuspectTimes.SequenceEqual(right.SuspectTimes)); private static bool MembershipSnapshotsEqual( MembershipTableSnapshot left, @@ -399,7 +412,7 @@ private static bool MembershipSnapshotsEqual( foreach (var (siloAddress, entry) in left.Entries) { if (!right.Entries.TryGetValue(siloAddress, out var other) - || !MembershipEntriesEqual(entry, other)) + || entry.IAmAliveTime != other.IAmAliveTime) { return false; } diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs index aef884ce0c2..ed92e3b2e85 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs @@ -20,8 +20,10 @@ namespace UnitTests.Dissemination; public partial class DisseminationProtocolTests { - [Fact] - public async Task MembershipInventoryOnlyRepairConvergesWithoutHeartbeat() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MembershipInventoryOnlyRepairConvergesWithoutHeartbeat(bool advanceVersion) { var cancellationToken = TestContext.Current.CancellationToken; var local = CreateSilo(40301); @@ -30,8 +32,9 @@ public async Task MembershipInventoryOnlyRepairConvergesWithoutHeartbeat() CreateSilo(40303), SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(20)); var retired = CreateMembershipEntry(CreateSilo(40304), SiloStatus.Dead, DateTime.UnixEpoch); var previous = CreateMembershipSnapshot(2, live, retired); - var incoming = CreateMembershipSnapshot(2, live.WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))); - var expected = CreateMembershipSnapshot(2, live); + var version = previous.Version.Value + (advanceVersion ? 1 : 0); + var incoming = CreateMembershipSnapshot(version, live.WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))); + var expected = CreateMembershipSnapshot(version, live); using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); var serializer = services.GetRequiredService(); using var receiverManager = CreateMembershipReviewManager(local, out _); @@ -42,17 +45,18 @@ public async Task MembershipInventoryOnlyRepairConvergesWithoutHeartbeat() var source = CreateMembershipNamespace((IMembershipManager)sourceManager, serializer); var before = Assert.Single(receiver.Digests); var update = new MembershipTableSnapshotUpdate { Snapshot = incoming }; - var value = new DisseminationValue(DisseminationKey.Default, 0, 2, serializer.SerializeToArray(update)); + var value = new DisseminationValue(DisseminationKey.Default, 0, version, serializer.SerializeToArray(update)); Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(value, cancellationToken)); AssertMembershipState(expected, receiverManager.MembershipTableSnapshot); Assert.NotEqual(before.Fingerprint, Assert.Single(receiver.Digests).Fingerprint); - Assert.Equal(before.Version, Assert.Single(receiver.Digests).Version); + Assert.Equal(version, Assert.Single(receiver.Digests).Version); Assert.Equal(Assert.Single(source.Digests), Assert.Single(receiver.Digests)); Assert.Equal(DisseminationApplyResult.Duplicate, await receiver.ApplyValueAsync(value, cancellationToken)); var staleInventory = new DisseminationValue( DisseminationKey.Default, 0, 2, serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = previous })); - Assert.Equal(DisseminationApplyResult.Duplicate, await receiver.ApplyValueAsync(staleInventory, cancellationToken)); + Assert.Equal(advanceVersion ? DisseminationApplyResult.Obsolete : DisseminationApplyResult.Duplicate, + await receiver.ApplyValueAsync(staleInventory, cancellationToken)); AssertMembershipState(expected, receiverManager.MembershipTableSnapshot); Assert.Equal(DateTime.UnixEpoch.AddSeconds(10), incoming.Entries[live.SiloAddress].IAmAliveTime); @@ -95,6 +99,7 @@ public async Task MembershipInventoryCleanupRespectsManagerLocalEntryRetention() Assert.Equal(DisseminationApplyResult.Duplicate, await ns.ApplyValueAsync(value, cancellationToken)); Assert.Same(current, manager.MembershipTableSnapshot); Assert.Equal(SiloStatus.Dead, manager.MembershipTableSnapshot.Entries[local].Status); + Assert.Equal(new[] { local, peer }, manager.MembershipTableSnapshot.Entries.Keys.Order()); } [Theory] diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipDeltas.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipDeltas.cs index 2cdc6cf7952..7a9aa2843ed 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipDeltas.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipDeltas.cs @@ -269,7 +269,7 @@ public async Task AheadMembershipRelayBroadcastsLatestDeltaFromChildBaseline() try { await Task.WhenAll(protocols.Values.Select(protocol => protocol.StopAsync(canceled))) - .WaitAsync(TimeSpan.FromSeconds(5)); + .WaitAsync(TimeSpan.FromSeconds(5), token); } catch (OperationCanceledException) when (canceled.IsCancellationRequested) { @@ -310,8 +310,60 @@ public async Task MembershipDeltaRejectsIntermediateViewAndMergesTargetReplay() await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken)); } - [Fact] - public async Task MembershipFullRepairMergesIncomparableCleanupAndHeartbeatState() + [Theory] + [InlineData("duplicate-upsert")] + [InlineData("upsert-and-remove")] + [InlineData("same-view-removal")] + [InlineData("same-view-addition")] + [InlineData("wire-version-mismatch")] + public async Task MembershipDeltaRejectsInvalidShapeBeforeOwnerMutation(string scenario) + { + var local = CreateSilo(45911); + var added = CreateSilo(45912); + var initial = CreateMembershipSnapshot(1, CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(initial); + var ns = CreateMembershipNamespace(manager, serializer); + var entry = initial.Entries[local].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10)); + var value = scenario switch + { + "duplicate-upsert" => CreateMembershipDelta(serializer, 1, 2, [entry, entry], []), + "upsert-and-remove" => CreateMembershipDelta(serializer, 1, 2, [entry], [local]), + "same-view-removal" => CreateMembershipDelta(serializer, 1, 1, [], [local]), + "same-view-addition" => CreateMembershipDelta(serializer, 1, 1, + [CreateMembershipEntry(added, SiloStatus.Joining, DateTime.UnixEpoch)], []), + "wire-version-mismatch" => new DisseminationValue(DisseminationKey.Default, 1, 3, + CreateMembershipDelta(serializer, 1, 2, [entry], []).Payload), + _ => throw new InvalidOperationException(scenario), + }; + var ownerCalls = 0; + manager.ProcessGossipSnapshotHandler = (_, _) => + { + ownerCalls++; + return Task.CompletedTask; + }; + + Assert.Equal(DisseminationApplyResult.Rejected, + await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken)); + Assert.Equal(0, ownerCalls); + Assert.Same(initial, manager.CurrentSnapshot); + } + + [Theory] + [InlineData(false, SiloStatus.Dead)] + [InlineData(true, SiloStatus.Dead)] + [InlineData(false, SiloStatus.Created)] + [InlineData(true, SiloStatus.Created)] + [InlineData(false, SiloStatus.Joining)] + [InlineData(true, SiloStatus.Joining)] + [InlineData(false, SiloStatus.Active)] + [InlineData(true, SiloStatus.Active)] + [InlineData(false, SiloStatus.ShuttingDown)] + [InlineData(true, SiloStatus.ShuttingDown)] + [InlineData(false, SiloStatus.Stopping)] + [InlineData(true, SiloStatus.Stopping)] + public async Task MembershipSameVersionPruningRemovesOnlyDeadRows(bool delta, SiloStatus removedStatus) { var local = CreateSilo(45401); var first = CreateSilo(45402); @@ -323,17 +375,104 @@ public async Task MembershipFullRepairMergesIncomparableCleanupAndHeartbeatState CreateMembershipEntry(first, SiloStatus.Dead, DateTime.UnixEpoch))); var receiverManager = new FakeMembershipManager(CreateMembershipSnapshot(7, CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(40)), - CreateMembershipEntry(second, SiloStatus.Dead, DateTime.UnixEpoch))); + CreateMembershipEntry(second, removedStatus, DateTime.UnixEpoch))); var source = CreateMembershipNamespace(sourceManager, serializer); var receiver = CreateMembershipNamespace(receiverManager, serializer); + var sameVersion = delta + ? CreateMembershipDelta(serializer, 7, 7, [], [second]) + : GetMembershipRepair(source, 7); + Assert.Equal(removedStatus == SiloStatus.Dead ? DisseminationApplyResult.Applied : DisseminationApplyResult.Rejected, + await receiver.ApplyValueAsync(sameVersion, TestContext.Current.CancellationToken)); + Assert.Equal(removedStatus != SiloStatus.Dead, receiverManager.CurrentSnapshot.Entries.ContainsKey(second)); + Assert.DoesNotContain(first, receiverManager.CurrentSnapshot.Entries.Keys); + + sourceManager.CurrentSnapshot = CreateMembershipSnapshot(8, + sourceManager.CurrentSnapshot.Entries[local]); + var removal = delta + ? CreateMembershipDelta(serializer, 7, 8, [], [second]) + : GetMembershipRepair(source, 7); Assert.Equal(DisseminationApplyResult.Applied, - await receiver.ApplyValueAsync(GetMembershipRepair(source, 7), TestContext.Current.CancellationToken)); + await receiver.ApplyValueAsync(removal, TestContext.Current.CancellationToken)); Assert.Equal(local, Assert.Single(receiverManager.CurrentSnapshot.Entries).Key); Assert.Equal(DateTime.UnixEpoch.AddSeconds(40), receiverManager.CurrentSnapshot.Entries[local].IAmAliveTime); + Assert.Equal(8, receiverManager.CurrentSnapshot.Version.Value); + } + + [Fact] + public async Task MembershipDeltaCanPruneAllDeadRowsWithoutResurrection() + { + var first = CreateSilo(45921); + var second = CreateSilo(45922); + var initial = CreateMembershipSnapshot(7, + CreateMembershipEntry(first, SiloStatus.Dead, DateTime.UnixEpoch), + CreateMembershipEntry(second, SiloStatus.Dead, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var sourceManager = new FakeMembershipManager(initial); + var source = CreateMembershipNamespace(sourceManager, serializer); + var receiverManager = new FakeMembershipManager(initial); + var receiver = CreateMembershipNamespace(receiverManager, serializer); + var request = new DisseminationRepairRequest(DisseminationKey.Default, 7, 1024 * 1024, 1024 * 1024); + var baseline = source.CreateBroadcast(request, baseline: null).BroadcastState; + var staleFull = source.CreateRepair(request).Value; + sourceManager.CurrentSnapshot = CreateMembershipSnapshot(7); + var delta = source.CreateBroadcast(request, baseline).Value; + + Assert.Equal(new[] { first, second }, ReadMembershipDelta(serializer, delta).RemovedSilos.Order()); Assert.Equal(DisseminationApplyResult.Applied, - await source.ApplyValueAsync(GetMembershipRepair(receiver, 7), TestContext.Current.CancellationToken)); - AssertMembershipState(receiverManager.CurrentSnapshot, sourceManager.CurrentSnapshot); + await receiver.ApplyValueAsync(delta, TestContext.Current.CancellationToken)); + Assert.Empty(receiverManager.CurrentSnapshot.Entries); + Assert.Equal(7, receiverManager.CurrentSnapshot.Version.Value); + Assert.Equal(DisseminationApplyResult.Duplicate, + await receiver.ApplyValueAsync(delta, TestContext.Current.CancellationToken)); + Assert.Equal(DisseminationApplyResult.Duplicate, + await receiver.ApplyValueAsync(staleFull, TestContext.Current.CancellationToken)); + Assert.Empty(receiverManager.CurrentSnapshot.Entries); + } + + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task MembershipHeartbeatAcceptanceIsIndependentOfStartTime(bool delta, bool rejectOwnerUpdate) + { + var local = CreateSilo(45901); + var start = DateTime.UnixEpoch.AddYears(50); + var heartbeat = DateTime.UnixEpoch.AddSeconds(10); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, start, DateTime.MinValue)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(initial); + if (rejectOwnerUpdate) + { + manager.ProcessGossipSnapshotHandler = (_, _) => Task.CompletedTask; + } + + var ns = CreateMembershipNamespace(manager, serializer); + var fingerprint = Assert.Single(ns.Digests).Fingerprint; + var entry = initial.Entries[local].WithIAmAliveTime(heartbeat); + var value = delta + ? CreateMembershipDelta(serializer, 1, 1, [entry], []) + : new DisseminationValue(DisseminationKey.Default, 0, 1, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = CreateMembershipSnapshot(1, entry) })); + + var result = await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken); + + Assert.Equal(rejectOwnerUpdate ? DisseminationApplyResult.Rejected : DisseminationApplyResult.Applied, result); + Assert.Equal(start, manager.CurrentSnapshot.Entries[local].StartTime); + Assert.Equal(rejectOwnerUpdate ? DateTime.MinValue : heartbeat, manager.CurrentSnapshot.Entries[local].IAmAliveTime); + if (rejectOwnerUpdate) + { + Assert.Equal(fingerprint, Assert.Single(ns.Digests).Fingerprint); + } + else + { + Assert.NotEqual(fingerprint, Assert.Single(ns.Digests).Fingerprint); + Assert.True(manager.CurrentSnapshot.IsSuccessorTo(initial)); + } } [Fact] @@ -509,7 +648,7 @@ public async ValueTask DisposeAsync() }; try { - await Task.WhenAll(stops).WaitAsync(TimeSpan.FromSeconds(5)); + await Task.WhenAll(stops).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); } catch (OperationCanceledException) when (token.IsCancellationRequested) { diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs index 20cf5500359..c069eaf3f90 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs @@ -98,62 +98,51 @@ public async Task MembershipNamespaceFullSnapshotPreservesMixedLivenessInRealMan } [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task MembershipNamespaceFullSnapshotPrunesDivergentInventoryAndRepairsDownstream(bool receiverAlreadyAtTargetVersion) + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task MembershipNamespaceFullSnapshotPrunesDeadRowsAndRepairsDownstream( + bool advanceVersion, bool receiverAlreadyAtTargetVersion) { var cancellationToken = TestContext.Current.CancellationToken; var retired = CreateSilo(39511); var successor = SiloAddress.New(retired.Endpoint, retired.Generation + 1); using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); var serializer = services.GetRequiredService(); - var table = new InMemoryMembershipTable(services.GetRequiredService()); - Assert.True(table.Insert( - CreateMembershipEntry(retired, SiloStatus.Active, DateTime.UnixEpoch), - table.ReadTableVersion().Next())); - var staleGenerationSnapshot = MembershipTableSnapshot.Create(table.ReadAll()); - var retiredRow = Assert.Single(table.ReadAll().Members); - Assert.True(table.Update( - retiredRow.Item1.WithStatus(SiloStatus.Dead), - retiredRow.Item2, - table.ReadTableVersion().Next())); - Assert.True(table.Insert( - CreateMembershipEntry(successor, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(10)), - table.ReadTableVersion().Next())); - var beforeCleanup = MembershipTableSnapshot.Create(table.ReadAll()); + var staleGenerationSnapshot = CreateMembershipSnapshot(1, + CreateMembershipEntry(retired, SiloStatus.Active, DateTime.UnixEpoch)); + var beforeCleanup = CreateMembershipSnapshot(2, + staleGenerationSnapshot.Entries[retired].WithStatus(SiloStatus.Dead), + CreateMembershipEntry(successor, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(20))); using var sourceManager = CreateMembershipReviewManager(CreateSilo(39510), out _); using var receiverManager = CreateMembershipReviewManager(CreateSilo(39512), out _); await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(beforeCleanup, cancellationToken); await ((IMembershipManager)receiverManager).ProcessGossipSnapshot(beforeCleanup, cancellationToken); - var tableVersion = table.ReadTableVersion(); - table.CleanupDefunctSiloEntries(DateTimeOffset.UnixEpoch.AddSeconds(1)); - Assert.Equal(tableVersion, table.ReadTableVersion()); - Assert.DoesNotContain(table.ReadAll().Members, entry => entry.Item1.SiloAddress.Equals(retired)); - table.UpdateIAmAlive(beforeCleanup.Entries[successor].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(20))); - var afterCleanup = MembershipTableSnapshot.Update(beforeCleanup, table.ReadAll()); - Assert.Equal(beforeCleanup.Version, afterCleanup.Version); - Assert.True(afterCleanup.IsSuccessorTo(beforeCleanup)); - await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(afterCleanup, cancellationToken); var source = CreateMembershipNamespace((IMembershipManager)sourceManager, serializer); var receiver = CreateMembershipNamespace((IMembershipManager)receiverManager, serializer); var sourceBaseline = Assert.Single(source.Digests); var receiverBaseline = Assert.Single(receiver.Digests); - Assert.Equal(sourceBaseline.Version, receiverBaseline.Version); - Assert.NotEqual(sourceBaseline.Fingerprint, receiverBaseline.Fingerprint); + Assert.Equal(sourceBaseline, receiverBaseline); - var row = Assert.Single(table.ReadAll().Members); - Assert.True(table.Update( - row.Item1.WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(30)), - row.Item2, - table.ReadTableVersion().Next())); - var target = MembershipTableSnapshot.Update(afterCleanup, table.ReadAll()); + var afterCleanup = MembershipTableSnapshot.Update(beforeCleanup, CreateMembershipSnapshot( + beforeCleanup.Version.Value + (advanceVersion ? 1 : 0), + beforeCleanup.Entries[successor].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10)))); + Assert.Equal(beforeCleanup.Version.Value + (advanceVersion ? 1 : 0), afterCleanup.Version.Value); + Assert.Equal(successor, Assert.Single(afterCleanup.Entries).Key); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(20), afterCleanup.Entries[successor].IAmAliveTime); + Assert.True(afterCleanup.IsSuccessorTo(beforeCleanup)); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(afterCleanup, cancellationToken); + var target = CreateMembershipSnapshot(afterCleanup.Version.Value, + afterCleanup.Entries[successor].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(30))); await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(target, cancellationToken); if (receiverAlreadyAtTargetVersion) { - await ((IMembershipManager)receiverManager).ProcessGossipSnapshot( - new MembershipTableSnapshot(target.Version, beforeCleanup.Entries), - cancellationToken); + await ((IMembershipManager)receiverManager).ProcessGossipSnapshot(afterCleanup, cancellationToken); + Assert.Equal(target.Entries.Keys, receiverManager.MembershipTableSnapshot.Entries.Keys); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(20), + receiverManager.MembershipTableSnapshot.Entries[successor].IAmAliveTime); } var value = GetMembershipRepair(source, sourceBaseline.Version); @@ -225,8 +214,10 @@ public async Task MembershipNamespaceFullSnapshotPreservesRealManagerLocalDeathE AssertMembershipState(expected, Assert.IsType(repaired.Snapshot)); } - [Fact] - public async Task MembershipNamespaceInventoryDivergenceConvergesWithoutHeartbeatWithFullRepair() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MembershipNamespaceDeadRowCleanupConvergesWithoutHeartbeatWithFullRepair(bool advanceVersion) { var cancellationToken = TestContext.Current.CancellationToken; var retired = CreateSilo(39541); @@ -234,26 +225,25 @@ public async Task MembershipNamespaceInventoryDivergenceConvergesWithoutHeartbea var receiverBaseline = CreateMembershipSnapshot( 1, CreateMembershipEntry(retired, SiloStatus.Dead, DateTime.UnixEpoch), - CreateMembershipEntry(successor, SiloStatus.Active, DateTime.UnixEpoch)); - var sourceBaseline = CreateMembershipSnapshot(1, receiverBaseline.Entries[successor]); + CreateMembershipEntry(successor, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(10))); var target = CreateMembershipSnapshot( - 2, - sourceBaseline.Entries[successor].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))); + receiverBaseline.Version.Value + (advanceVersion ? 1 : 0), + receiverBaseline.Entries[successor]); using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); var serializer = services.GetRequiredService(); using var sourceManager = CreateMembershipReviewManager(CreateSilo(39540), out _); using var receiverManager = CreateMembershipReviewManager(CreateSilo(39542), out _); - await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(sourceBaseline, cancellationToken); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(receiverBaseline, cancellationToken); await ((IMembershipManager)receiverManager).ProcessGossipSnapshot(receiverBaseline, cancellationToken); var source = CreateMembershipNamespace((IMembershipManager)sourceManager, serializer); var receiver = CreateMembershipNamespace((IMembershipManager)receiverManager, serializer); _ = Assert.Single(source.Digests); await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(target, cancellationToken); - await ((IMembershipManager)receiverManager).ProcessGossipSnapshot( - CreateMembershipSnapshot(2, receiverBaseline.Entries[retired], target.Entries[successor]), cancellationToken); Assert.Equal(SiloStatus.Dead, receiverManager.MembershipTableSnapshot.GetSiloStatus(retired)); Assert.True(receiverManager.MembershipTableSnapshot.Entries.ContainsKey(retired)); - Assert.Equal(target.Version, receiverManager.MembershipTableSnapshot.Version); + Assert.Equal(receiverBaseline.Version, receiverManager.MembershipTableSnapshot.Version); + Assert.Equal(target.Version, sourceManager.MembershipTableSnapshot.Version); + Assert.Equal(receiverBaseline.Entries[successor].IAmAliveTime, target.Entries[successor].IAmAliveTime); var transport = new FakeTransport(CreateSilo(39540), CreateSilo(39542)); var protocol = CreateProtocol(transport, [source], timeProvider: new FakeTimeProvider()); @@ -265,9 +255,10 @@ public async Task MembershipNamespaceInventoryDivergenceConvergesWithoutHeartbea Digests = new() { [source.Name] = [Assert.Single(receiver.Digests)] }, }, cancellationToken); var full = Assert.Single(GetAntiEntropyResponseValues(response)).Value; - Assert.Equal((0L, 2L), (full.FromVersion, full.ToVersion)); + Assert.Equal((0L, target.Version.Value), (full.FromVersion, full.ToVersion)); Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(full, cancellationToken)); AssertMembershipState(target, receiverManager.MembershipTableSnapshot); + Assert.Equal(successor, Assert.Single(receiverManager.MembershipTableSnapshot.Entries).Key); Assert.Equal(Assert.Single(source.Digests), Assert.Single(receiver.Digests)); Assert.Equal(DisseminationApplyResult.Duplicate, await receiver.ApplyValueAsync(full, cancellationToken)); var converged = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest @@ -278,7 +269,7 @@ public async Task MembershipNamespaceInventoryDivergenceConvergesWithoutHeartbea Assert.Empty(GetAntiEntropyResponseValues(converged)); var heartbeatTarget = CreateMembershipSnapshot( - 2, + target.Version.Value, target.Entries[successor].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(20))); await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(heartbeatTarget, cancellationToken); response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest @@ -287,7 +278,7 @@ public async Task MembershipNamespaceInventoryDivergenceConvergesWithoutHeartbea Digests = new() { [source.Name] = [Assert.Single(receiver.Digests)] }, }, cancellationToken); full = Assert.Single(GetAntiEntropyResponseValues(response)).Value; - Assert.Equal((0L, 2L), (full.FromVersion, full.ToVersion)); + Assert.Equal((0L, target.Version.Value), (full.FromVersion, full.ToVersion)); Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(full, cancellationToken)); AssertMembershipState(heartbeatTarget, receiverManager.MembershipTableSnapshot); Assert.Equal(SiloStatus.Dead, receiverManager.MembershipTableSnapshot.GetSiloStatus(retired)); From 285bd5d4998bb3214a471c74ff850e01faf42e46 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 17 Sep 2026 15:04:12 -0700 Subject: [PATCH 48/50] test(dissemination): assert retained-row acknowledgment rejection --- ...isseminationProtocolTests.FinalComments.cs | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs index ed92e3b2e85..01a34835e6e 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs @@ -77,7 +77,7 @@ public async Task MembershipInventoryOnlyRepairConvergesWithoutHeartbeat(bool ad } [Fact] - public async Task MembershipInventoryCleanupRespectsManagerLocalEntryRetention() + public async Task MembershipInventoryCleanupDoesNotAcknowledgeRetainedLocalEntry() { var cancellationToken = TestContext.Current.CancellationToken; var local = CreateSilo(40311); @@ -96,10 +96,28 @@ public async Task MembershipInventoryCleanupRespectsManagerLocalEntryRetention() DisseminationKey.Default, 0, 2, serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = incoming })); Assert.True(incoming.IsSuccessorTo(current)); - Assert.Equal(DisseminationApplyResult.Duplicate, await ns.ApplyValueAsync(value, cancellationToken)); + Assert.Equal(DisseminationApplyResult.Rejected, await ns.ApplyValueAsync(value, cancellationToken)); Assert.Same(current, manager.MembershipTableSnapshot); Assert.Equal(SiloStatus.Dead, manager.MembershipTableSnapshot.Entries[local].Status); Assert.Equal(new[] { local, peer }, manager.MembershipTableSnapshot.Entries.Keys.Order()); + + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns]); + try + { + var response = await protocol.ReceiveBroadcast(new() + { + Sender = peer, + SupportsCompactAcknowledgments = true, + Values = new() { [ns.Name] = [new() { Value = value, TimeToLive = TimeSpan.FromSeconds(30) }] }, + }, cancellationToken); + Assert.False(response.AllVersionsAcknowledged); + Assert.Equal(current.Version.Value, Assert.Single(response.Acknowledgments[ns.Name]).Version); + Assert.Same(current, manager.MembershipTableSnapshot); + } + finally + { + await protocol.StopAsync(cancellationToken); + } } [Theory] From 3bf682c312a3bb1926dac98487642d99b460c56c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 17 Sep 2026 15:14:49 -0700 Subject: [PATCH 49/50] refactor(dissemination): rely on merged membership lifetime policy --- .../IMembershipTable.cs | 6 +- .../Dissemination/DisseminationProtocol.cs | 5 +- .../Dissemination/IDisseminationNamespace.cs | 3 - .../MembershipDisseminationNamespace.cs | 2 - .../MembershipTableManager.cs | 34 ++------- .../Membership/MembershipTableManagerTests.cs | 71 ------------------- .../DisseminationClusterTests.cs | 44 ------------ ...seminationProtocolTests.MembershipReset.cs | 45 ++++-------- 8 files changed, 24 insertions(+), 186 deletions(-) diff --git a/src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs b/src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs index 3e8b898d776..3acd979d042 100644 --- a/src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs +++ b/src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs @@ -68,12 +68,8 @@ async Task DeleteMembershipTableEntriesAsync(string clusterId, CancellationToken } /// - /// Cleans up defunct silo entries older than . + /// Delete all dead silo entries older than /// - /// - /// Cleanup can remove retained Dead entries at the same membership version. - /// All other entries remain part of that canonical view. - /// /// The exclusive upper bound for the last known update time of entries to delete. /// A task representing the cleanup operation. [Obsolete("Use CleanupDefunctSiloEntriesAsync instead.")] diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs index 32ff0b101d3..bf5cac0e1df 100644 --- a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -921,7 +921,7 @@ private DisseminationAntiEntropyResponse CreateAntiEntropyResponse( continue; } - if (localDigest.Version < peerDigest.Version && !requestedNamespace.ValidateOlderFullValues + if (localDigest.Version < peerDigest.Version || localDigest.Version == peerDigest.Version && localDigest.Fingerprint == peerDigest.Fingerprint) { @@ -1617,8 +1617,7 @@ private static bool TryGetTerminalApplyResult( } var localVersion = disseminationNamespace.GetVersion(value.Key); - if (value.ToVersion < localVersion - && (value.FromVersion > 0 || !disseminationNamespace.ValidateOlderFullValues)) + if (value.ToVersion < localVersion) { result = DisseminationApplyResult.Obsolete; return true; diff --git a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs index 00f11dd92ad..8d91dab466c 100644 --- a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs @@ -15,9 +15,6 @@ internal interface IDisseminationNamespace TimeSpan AggregationPeriod => TimeSpan.FromSeconds(1); - // Full values can be authority-refresh hints even when their numeric version is older. - bool ValidateOlderFullValues => false; - DisseminationNamespaceOptions Options { get; } IEnumerable Digests { get; } diff --git a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs index aa482646053..0b6700c6dcc 100644 --- a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs +++ b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs @@ -25,8 +25,6 @@ internal sealed class MembershipDisseminationNamespace( public bool BroadcastsAreDeltas => true; - public bool ValidateOlderFullValues => true; - public DisseminationNamespaceOptions Options => options.CurrentValue.Dissemination; public IEnumerable Keys => MembershipKeys; diff --git a/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs b/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs index 2d88346886d..9d08d66ae62 100644 --- a/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs +++ b/src/Orleans.Runtime/MembershipService/MembershipTableManager.cs @@ -193,7 +193,7 @@ public async Task RefreshFromSnapshot(MembershipTableSnapshot snapshot, Cancella private async Task RefreshInternal(bool requireCleanup, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var table = await this.ReadMembershipTable(cancellationToken); + var table = await this.membershipTableProvider.ReadAllAsync(cancellationToken); bool success; try @@ -215,28 +215,6 @@ private async Task RefreshInternal(bool requireCleanup, CancellationToken return !requireCleanup || success; } - private Task ReadMembershipTable(CancellationToken cancellationToken) => - this.membershipTableProvider is SystemTargetBasedMembershipTable - ? ReadDevelopmentMembershipTable(cancellationToken) - : this.membershipTableProvider.ReadAllAsync(cancellationToken); - - private async Task ReadDevelopmentMembershipTable(CancellationToken cancellationToken) - { - // Capture before issuing the read: an older read completing after a newer one is not a reset. - var observedVersion = this.MembershipTableSnapshot.Version; - var table = await this.membershipTableProvider.ReadAllAsync(cancellationToken); - cancellationToken.ThrowIfCancellationRequested(); - if (table.Version.Version < observedVersion.Value) - { - var reason = $"The development membership table version decreased from {observedVersion} to {table.Version.Version}. " - + "Its previous incarnation is no longer authoritative. Restart this silo against the recreated table."; - this.KillMyselfLocally(reason); - throw new OrleansException(reason); - } - - return table; - } - private async Task Start(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -477,7 +455,7 @@ async Task UpdateMyStatusTask(int counter, CancellationToken token) private async Task TryUpdateMyStatusGlobalOnce(SiloStatus newStatus, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var table = await ReadMembershipTable(cancellationToken); + var table = await membershipTableProvider.ReadAllAsync(cancellationToken); LogDebugTryUpdateMyStatusGlobalOnce(this.log, newStatus.Equals(SiloStatus.Active) ? "All" : " my entry from", table.ToString()); LogMissedIAmAlives(table); @@ -937,7 +915,7 @@ private void PublishSuspectOrKillRequestCompletion(SuspectOrKillRequest request, private async Task InnerTryKill(SiloAddress silo, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var table = await ReadMembershipTable(cancellationToken); + var table = await membershipTableProvider.ReadAllAsync(cancellationToken); LogDebugTryKillReadMembershipTable(this.log, table.ToString()); @@ -984,7 +962,7 @@ public async Task TryToSuspectOrKill(SiloAddress silo, SiloAddress? indire private async Task InnerTryToSuspectOrKill(SiloAddress silo, SiloAddress? indirectProbingSilo, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var table = await ReadMembershipTable(cancellationToken); + var table = await membershipTableProvider.ReadAllAsync(cancellationToken); var now = GetDateTimeUtcNow(); LogDebugTryToSuspectOrKillReadMembershipTable(this.log, table.ToString()); @@ -1059,7 +1037,7 @@ private async Task InnerTryToSuspectOrKill(SiloAddress silo, SiloAddress? var ok = await membershipTableProvider.UpdateRowAsync(entry, eTag, table.Version.Next(), cancellationToken); if (ok) { - table = await ReadMembershipTable(cancellationToken); + table = await membershipTableProvider.ReadAllAsync(cancellationToken); this.ProcessTableUpdate(table, "TrySuspectOrKill"); // Gossip using the local silo status, since this is just informational to propagate the suspicion vote. @@ -1091,7 +1069,7 @@ private async Task DeclareDead(MembershipEntry entry, string etag, TableVe { LogDebugSuccessfullyUpdatedStatusToDead(this.log, entry.SiloAddress); - var table = await ReadMembershipTable(cancellationToken); + var table = await membershipTableProvider.ReadAllAsync(cancellationToken); this.ProcessTableUpdate(table, "DeclareDead"); GossipToOthers(entry.SiloAddress, entry.Status, GossipTimeout, cancellationToken).Ignore(); return true; diff --git a/test/Orleans.Core.Tests/Membership/MembershipTableManagerTests.cs b/test/Orleans.Core.Tests/Membership/MembershipTableManagerTests.cs index 7eb724c314d..e4f884e0f92 100644 --- a/test/Orleans.Core.Tests/Membership/MembershipTableManagerTests.cs +++ b/test/Orleans.Core.Tests/Membership/MembershipTableManagerTests.cs @@ -1723,77 +1723,6 @@ public override void Post(SendOrPostCallback callback, object? state) => } } - [Theory] - [InlineData(true)] - [InlineData(false)] - public async Task MembershipTableResetStopsOnlyTheVolatileProviderIncarnation(bool developmentProvider) - { - var cancellationToken = TestContext.Current.CancellationToken; - var original = await new InMemoryMembershipTable(new TableVersion(100, "old"), - Entry(this.localSilo, SiloStatus.Active, DateTimeOffset.UtcNow)).ReadAllAsync(cancellationToken); - var reset = await new InMemoryMembershipTable(new TableVersion(1, "new")).ReadAllAsync(cancellationToken); - var backing = new LegacyMembershipTable(Substitute.For()); - backing.ConfigureReadAll(original); - IMembershipTable provider = developmentProvider ? CreateSystemTargetBasedMembershipTable(backing) : backing; - await provider.InitializeMembershipTableAsync(true, cancellationToken); - using var manager = CreateMembershipTableManager(provider); - await manager.Refresh(cancellationToken: cancellationToken, requireFresh: true); - var accepted = manager.MembershipTableSnapshot; - Assert.Equal(new MembershipVersion(100), accepted.Version); - backing.ConfigureReadAll(reset); - - if (developmentProvider) - { - var error = await Assert.ThrowsAsync( - () => manager.Refresh(cancellationToken: cancellationToken, requireFresh: true)); - Assert.Contains("version decreased from 100 to 1", error.Message, StringComparison.Ordinal); - this.fatalErrorHandler.Received(1).OnFatalException( - manager, Arg.Is(reason => reason.Contains("Restart this silo", StringComparison.Ordinal)), null); - } - else - { - await manager.Refresh(cancellationToken: cancellationToken, requireFresh: true); - this.fatalErrorHandler.DidNotReceiveWithAnyArgs().OnFatalException(default, default, default); - } - - Assert.Same(accepted, manager.MembershipTableSnapshot); - } - - [Fact] - public async Task OlderDevelopmentReadCompletingAfterNewerReadIsNotATableReset() - { - var cancellationToken = TestContext.Current.CancellationToken; - var original = await new InMemoryMembershipTable(new TableVersion(100, "old"), - Entry(this.localSilo, SiloStatus.Active, DateTimeOffset.UtcNow)).ReadAllAsync(cancellationToken); - var newer = new MembershipTableData(original.Members.ToList(), new TableVersion(101, "new")); - var backing = new LegacyMembershipTable(Substitute.For()); - backing.ConfigureReadAll(original); - var provider = CreateSystemTargetBasedMembershipTable(backing); - await provider.InitializeMembershipTableAsync(true, cancellationToken); - using var manager = CreateMembershipTableManager(provider); - await manager.Refresh(cancellationToken: cancellationToken, requireFresh: true); - var read = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - backing.ConfigureReadAll(read.Task); - var pending = manager.Refresh(cancellationToken: cancellationToken, requireFresh: true); - try - { - Assert.False(pending.IsCompleted); - backing.ConfigureReadAll(newer); - await manager.Refresh(cancellationToken: cancellationToken, requireFresh: true); - Assert.Equal(new MembershipVersion(101), manager.MembershipTableSnapshot.Version); - read.SetResult(original); - await pending.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - - Assert.Equal(new MembershipVersion(101), manager.MembershipTableSnapshot.Version); - this.fatalErrorHandler.DidNotReceiveWithAnyArgs().OnFatalException(default, default, default); - } - finally - { - read.TrySetResult(original); - await pending.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - } - } - private sealed class BackoffTimeProvider : FakeTimeProvider { public TaskCompletionSource TimerCreated { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs index 4fb1d90785a..968ae86ab1f 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs @@ -71,50 +71,6 @@ public async Task MembershipUpdatesAreDisseminatedAcrossRealClusterWithExplicitO + string.Join(", ", observer.AppliedSilos.Select(static silo => silo.ToString()))); } - [Fact] - public async Task DevelopmentPrimaryRestartRejectsRetainedPreResetMembership() - { - using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); - cancellation.CancelAfter(TimeSpan.FromMinutes(3)); - var cancellationToken = cancellation.Token; - var builder = new TestClusterBuilder(3); - builder.Options.InitializeClientOnDeploy = false; - builder.AddSiloBuilderConfigurator(); - await using var cluster = builder.Build(); - await cluster.DeployAsync(cancellationToken); - var primary = Assert.IsType(cluster.Primary); - Assert.IsType(primary.SiloHost.Services.GetRequiredService()); - var primaryManager = primary.SiloHost.Services.GetRequiredService(); - await primaryManager.Refresh(null, cancellationToken, requireFresh: true); - var retained = primaryManager.CurrentSnapshot; - Assert.Equal(3, retained.ActiveNodeCount); - var originalSilos = cluster.GetActiveSilos().ToArray(); - - var restarted = Assert.IsType( - await cluster.RestartSiloAsync(primary).WaitAsync(cancellationToken)); - Assert.NotEqual(primary.SiloAddress, restarted.SiloAddress); - var manager = restarted.SiloHost.Services.GetRequiredService(); - Assert.True(manager.CurrentSnapshot.Version < retained.Version); - await manager.ProcessGossipSnapshot(retained, cancellationToken); - Assert.Equal(SiloStatus.Active, manager.LocalSiloStatus); - Assert.Contains(restarted.SiloAddress, manager.CurrentSnapshot.Entries.Keys); - Assert.DoesNotContain(primary.SiloAddress, manager.CurrentSnapshot.Entries.Keys); - - foreach (var silo in originalSilos.Where(silo => !ReferenceEquals(silo, primary))) - { - await cluster.RestartSiloAsync(silo).WaitAsync(cancellationToken); - } - - foreach (var silo in cluster.GetActiveSilos().Cast()) - { - var current = silo.SiloHost.Services.GetRequiredService(); - await current.Refresh(null, cancellationToken, requireFresh: true); - Assert.Equal(SiloStatus.Active, current.LocalSiloStatus); - Assert.Contains(restarted.SiloAddress, current.CurrentSnapshot.Entries.Keys); - Assert.DoesNotContain(primary.SiloAddress, current.CurrentSnapshot.Entries.Keys); - } - } - public sealed class EnabledDisseminationConfigurator : ISiloConfigurator { public void Configure(ISiloBuilder builder) => builder.ConfigureServices(services => diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs index 17397795cc8..8a526c98c13 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs @@ -11,19 +11,13 @@ public partial class DisseminationProtocolTests [Theory] [InlineData(false)] [InlineData(true)] - public async Task OlderFullMembershipValueReachesAuthoritativeOwner(bool antiEntropy) + public async Task OlderFullMembershipValueDoesNotReachOwner(bool antiEntropy) { var cancellationToken = TestContext.Current.CancellationToken; var local = CreateSilo(41011); var peer = CreateSilo(41012); var current = CreateMembershipSnapshot(100, CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); var notification = CreateMembershipSnapshot(2, current.Entries.Values.ToArray()); - var authority = CreateMembershipSnapshot(101, current.Entries.Values.Select(entry => - { - var result = entry.Copy(); - result.HostName = "authoritative"; - return result; - }).ToArray()); using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); var serializer = services.GetRequiredService(); var manager = new FakeMembershipManager(current); @@ -31,9 +25,7 @@ public async Task OlderFullMembershipValueReachesAuthoritativeOwner(bool antiEnt manager.ProcessGossipSnapshotHandler = (snapshot, token) => { token.ThrowIfCancellationRequested(); - Assert.Equal(notification.Version, snapshot.Version); validations++; - manager.CurrentSnapshot = authority; return Task.CompletedTask; }; var ns = CreateMembershipNamespace(manager, serializer); @@ -53,12 +45,11 @@ public async Task OlderFullMembershipValueReachesAuthoritativeOwner(bool antiEnt else { var response = await protocol.ReceiveBroadcast(new() { Sender = peer, Values = values }, cancellationToken); - Assert.Equal(101, Assert.Single(response.Acknowledgments[ns.Name]).Version); + Assert.Equal(100, Assert.Single(response.Acknowledgments[ns.Name]).Version); } - Assert.Equal(1, validations); - Assert.Same(authority, manager.CurrentSnapshot); - Assert.Equal("authoritative", manager.CurrentSnapshot.Entries[local].HostName); + Assert.Equal(0, validations); + Assert.Same(current, manager.CurrentSnapshot); } finally { @@ -68,7 +59,7 @@ public async Task OlderFullMembershipValueReachesAuthoritativeOwner(bool antiEnt } [Fact] - public async Task AntiEntropyConsultsMembershipNamespaceWhenPeerVersionIsAhead() + public async Task AntiEntropyDoesNotSendOlderMembershipToAnAheadPeer() { var cancellationToken = TestContext.Current.CancellationToken; var local = CreateSilo(41013); @@ -87,11 +78,7 @@ public async Task AntiEntropyConsultsMembershipNamespaceWhenPeerVersionIsAhead() Digests = new() { [ns.Name] = [new(DisseminationKey.Default, 100)] }, }, cancellationToken); - var value = Assert.Single(GetAntiEntropyResponseValues(response)).Value; - Assert.Equal(0, value.FromVersion); - Assert.Equal(2, value.ToVersion); - Assert.NotNull(Assert.IsType( - serializer.Deserialize(value.Payload)).Snapshot); + Assert.Empty(GetAntiEntropyResponseValues(response)); } finally { @@ -100,7 +87,7 @@ public async Task AntiEntropyConsultsMembershipNamespaceWhenPeerVersionIsAhead() } [Fact] - public void MembershipRepairSendsFullStateWhenPeerVersionIsAhead() + public void MembershipRepairMaterializesTheCurrentOwnerSnapshot() { var local = CreateSilo(41001); var snapshot = CreateMembershipSnapshot(2, CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); @@ -124,7 +111,7 @@ public void MembershipRepairSendsFullStateWhenPeerVersionIsAhead() } [Fact] - public async Task MembershipResetSupersedesPriorIncarnationAndDelayedPublication() + public async Task MembershipCurrentViewSupersedesDelayedPublication() { var members = CreateSilos(20); var old = CreateMembershipSnapshot(1, members.Select( @@ -134,30 +121,28 @@ public async Task MembershipResetSupersedesPriorIncarnationAndDelayedPublication var manager = new FakeMembershipManager(old); var ns = CreateMembershipNamespace(manager, serializer); Assert.Equal(1, Assert.Single(ns.Digests).Version); - manager.CurrentSnapshot = CreateMembershipSnapshot(100, old.Entries.Values.ToArray()); - Assert.Equal(100, Assert.Single(ns.Digests).Version); - var reset = CreateMembershipSnapshot(2, old.Entries.Values.Select(entry => + var current = CreateMembershipSnapshot(100, old.Entries.Values.Select(entry => { var result = entry.Copy(); - result.HostName = "new-table"; + result.HostName = "current-view"; return result; }).ToArray()); - manager.CurrentSnapshot = reset; - Assert.Equal(2, Assert.Single(ns.Digests).Version); + manager.CurrentSnapshot = current; + Assert.Equal(100, Assert.Single(ns.Digests).Version); var publications = new FakeDisseminationService(); Assert.True(await ns.PublishAsync(publications, old, TestContext.Current.CancellationToken)); - Assert.Equal(2, Assert.Single(publications.Values).ToVersion); + Assert.Equal(100, Assert.Single(publications.Values).ToVersion); var repair = ns.CreateRepair(new( DisseminationKey.Default, 1, 1024 * 1024, 1024 * 1024)); Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); var value = repair.Value; Assert.Equal(0, value.FromVersion); - Assert.Equal(2, value.ToVersion); + Assert.Equal(100, value.ToVersion); var payload = Assert.IsType( serializer.Deserialize(value.Payload)); var repaired = Assert.IsType(payload.Snapshot); Assert.Equal(members.Length, repaired.Entries.Count); - Assert.All(repaired.Entries.Values, entry => Assert.Equal("new-table", entry.HostName)); + Assert.All(repaired.Entries.Values, entry => Assert.Equal("current-view", entry.HostName)); } } From b496bbb09af7772ded855f61080a15be499946ea Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 17 Sep 2026 15:46:35 -0700 Subject: [PATCH 50/50] test(dissemination): isolate broadcast failure metric injection --- ...eminationProtocolTests.MetricBoundaries.cs | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs index ea0d6618317..52922f8fbc1 100644 --- a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs @@ -36,6 +36,7 @@ public async Task MetricFailuresPreserveProtocolResults(string operation) }; var failure = new InvalidOperationException("Metric observer failure."); var failures = 0; + var broadcastFailureThread = 0; var listenerArmed = 1; using var listener = new MeterListener(); listener.InstrumentPublished = (instrument, owner) => @@ -53,6 +54,17 @@ public async Task MetricFailuresPreserveProtocolResults(string operation) return; } + // Failure counters have no namespace tag. This completed fake transport failure is observed + // synchronously on its throwing thread, so unrelated background failures cannot claim it. + if (operation == "broadcast-failure") + { + var thread = Environment.CurrentManagedThreadId; + if (Interlocked.CompareExchange(ref broadcastFailureThread, 0, thread) != thread) + { + return; + } + } + if (operation is "broadcast-receive" or "broadcast-send" or "publication") { var matches = false; @@ -122,12 +134,18 @@ public async Task MetricFailuresPreserveProtocolResults(string operation) break; case "broadcast-failure": ns.SetValue("value", 1); + DisseminationInstruments.OnBroadcastSendFailure(DisseminationFailureReason.Error); + Assert.Equal(0, Volatile.Read(ref failures)); transport.SendBroadcastResponseHandler = (target, batch, _) => { transport.BroadcastBatches.Add((target, batch)); - return transport.BroadcastBatches.Count == 1 - ? Task.FromException(new InvalidOperationException("Transport failure.")) - : Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + if (transport.BroadcastBatches.Count == 1) + { + Volatile.Write(ref broadcastFailureThread, Environment.CurrentManagedThreadId); + return Task.FromException(new InvalidOperationException("Transport failure.")); + } + + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); }; Assert.True(await protocol.Publish(ns, "value", 1, cancellationToken)); await protocol.FlushPendingBroadcast(cancellationToken); @@ -143,6 +161,7 @@ public async Task MetricFailuresPreserveProtocolResults(string operation) } Assert.Equal(1, Volatile.Read(ref failures)); + Assert.Equal(0, Volatile.Read(ref broadcastFailureThread)); if (operation is not ("broadcast-send" or "broadcast-failure")) { Assert.Contains(logger.Entries, entry => ReferenceEquals(entry.Exception, failure));