From d079563ca09be14cfae2ae59a55239bda341e4f3 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 14:40:54 -0700 Subject: [PATCH 01/21] perf(runtime): cache local grain message targets --- .testagent/plan.md | 377 +++++++++++++++++ .testagent/research.md | 147 +++++++ .testagent/status.md | 30 ++ .../Runtime/GrainReference.cs | 12 + .../Runtime/GrainReferenceRuntime.cs | 8 +- .../Core/InsideRuntimeClient.cs | 2 +- .../GrainDirectory/CachedGrainLocator.cs | 14 + .../GrainDirectory/DhtGrainLocator.cs | 11 + .../GrainDirectoryCacheEntry.cs | 55 +++ .../GrainDirectory/GrainLocator.cs | 20 + .../GrainDirectory/IGrainDirectoryCache.cs | 5 + .../GrainDirectory/LocalGrainDirectory.cs | 13 + .../GrainDirectory/LruGrainDirectoryCache.cs | 26 +- .../Messaging/MessageCenter.cs | 121 +++++- .../Placement/PlacementService.cs | 3 + .../GrainDirectoryCacheFactoryTests.cs | 391 ++++++++++++++++++ .../SharedEntryMessageTargetFastPathTests.cs | 247 +++++++++++ 17 files changed, 1459 insertions(+), 23 deletions(-) create mode 100644 .testagent/plan.md create mode 100644 .testagent/research.md create mode 100644 .testagent/status.md create mode 100644 src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs create mode 100644 test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs diff --git a/.testagent/plan.md b/.testagent/plan.md new file mode 100644 index 00000000000..3fca47f12ee --- /dev/null +++ b/.testagent/plan.md @@ -0,0 +1,377 @@ +# Test Implementation Plan + +> Final scope update: the implementation was narrowed to local `IGrainContext` targets. Remote connection and response-route cases below were superseded by explicit remote/client/stateless fallback tests. + +## Overview + +This is a targeted, three-phase plan for the forthcoming shared directory-entry message-target cache. Existing coverage is substantial for locator selection and partial for cache lifetime, placement, local activation, and networking, while the shared route-handle API is currently absent. + +All test source additions are gated on the production API landing. Once available, tests will use the repository's xUnit v3/Microsoft Testing Platform conventions, Arrange-Act-Assert structure, existing BVT/category/area attributes, deterministic synchronization, and supported internal test accessors rather than reflection. + +## Hard Boundary: Not in This Preparation Turn + +- Do not modify production code. +- Do not create or edit any test source file. +- Do not build, discover, or run tests while the production API is absent. +- Do not restore, clean, reset, stash, or delete tracked files. +- This turn creates only `.testagent/plan.md`; do not create `.testagent/status.md`. +- The phases below begin only after the route-handle/cache-entry API and required test seams land. + +## Commands + +Run from the workspace root only after the production API lands and dependencies are already available. + +- **Scoped build (runtime tests)**: `dotnet build test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj --framework net10.0 --no-restore` +- **Scoped build (core tests)**: `dotnet build test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore` +- **Full build**: `dotnet build Orleans.slnx --no-restore --no-incremental -bl` +- **Scoped test (directory cache lifetime)**: `dotnet test --project test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj --framework net10.0 --no-restore --filter-class "*GrainDirectoryCacheFactoryTests*" --minimum-expected-tests 1` +- **Scoped test (stale async binding)**: `dotnet test --project test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore --filter-class "*CachedGrainLocatorTests*" --minimum-expected-tests 1` +- **Scoped test (shared/local/remote routes and growth)**: `dotnet test --project test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore --filter-class "*MessageTargetCacheTests*" --minimum-expected-tests 1` +- **Scoped test (placement and compatibility)**: `dotnet test --project test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore --filter-class "*PlacementServiceTests*" --minimum-expected-tests 1` +- **Scoped regression (locator selection)**: `dotnet test --project test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore --filter-class "*GrainLocatorResolverTests*" --minimum-expected-tests 1` +- **Full test (all target frameworks with a fresh build)**: `dotnet test --solution Orleans.slnx --no-restore --minimum-expected-tests 1 --max-parallel-test-modules 1` +- **Full discovery check (all target frameworks)**: `dotnet test --solution Orleans.slnx --no-restore --list-tests --minimum-expected-tests 1 --max-parallel-test-modules 1` +- **Lint**: No separate command; the build enforces style and treats warnings, including nullable warnings, as errors. + +## Phase Summary + +| Phase | Focus | Test files | Est. tests | +|---|---|---:|---:| +| 1 | Directory-entry ownership and stale async binds | 2 existing | 8 | +| 2 | Shared tombstones plus local/remote route validation | 1 new | 9 | +| 3 | Header/compatibility regressions and bounded growth | 2 (1 existing, 1 from Phase 2) | 8 | + +--- + +## Phase 1: Directory Entry Lifetime and Race Safety + +### Overview + +Establish the leaf/mid-layer lifetime contract first. Every directory cache removal path must invalidate the owned shared route handle, and delayed directory lookups must not bind a disposed or superseded generation. + +### Production Scope + +Assigned to this phase: + +- `src/Orleans.Runtime/GrainDirectory/IGrainDirectoryCache.cs` +- `src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs` +- `src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheFactory.cs` +- `src/Orleans.Runtime/Configuration/Options/GrainDirectoryOptions.cs` +- `src/Orleans.Runtime/GrainDirectory/CachedGrainLocator.cs` + +### Files to Test + +#### 1. GrainDirectoryCacheFactoryTests.cs + +- **Project**: `test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj` +- **Test File**: `test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs` +- **Test Class**: `GrainDirectoryCacheFactoryTests` +- **Methods/contracts**: factory creation; cache `AddOrUpdate`, both `Remove` variants, `Clear`, expiration cleanup, and capacity eviction. +- **Current status**: Cache mechanics are regression-testable now, but route-handle assertions are blocked. Add all tests only after the API lands. +- **Future seam/blocker**: The cache value must be a disposable directory entry which owns an observable route handle/tombstone. Tests need supported validity/generation observations, not private-field reflection. + +**Tests**: + +1. `CreateGrainDirectoryCache_AddOrUpdateInvalidatesReplacedRouteHandle` + - Arrange: Create the LRU cache with the existing factory fixture. Add a directory entry for one `GrainId` with address A and handle H1, then prepare address B with H2. + - Act: `AddOrUpdate` the same `GrainId` with the replacement entry. + - Assert: H1 is invalid/disposed exactly once; H2 remains valid and lookup returns address B. + - Secondary observable: A consumer retaining H1 cannot obtain a usable local receiver or remote connection from it. + +2. `CreateGrainDirectoryCache_RemoveByGrainIdInvalidatesRouteHandle` + - Arrange: Add one route-owning entry and retain its handle separately. + - Act: Remove by `GrainId`. + - Assert: The retained handle is a tombstone and cache lookup misses. + - Secondary observable: A second removal is harmless and does not repeat disposal. + +3. `CreateGrainDirectoryCache_RemoveByAddressInvalidatesOnlyMatchingRouteHandle` + - Arrange: Add an entry for address A and retain H1; create a different address B for the same grain. + - Act: First remove using B, then remove using A. + - Assert: The mismatched removal leaves H1 valid and the entry present; the matching removal invalidates H1 and removes the entry. + - Secondary observable: No unrelated route handle is invalidated. + +4. `CreateGrainDirectoryCache_ClearInvalidatesAllRouteHandles` + - Arrange: Add multiple distinct grain entries and retain every handle. + - Act: Call `Clear`. + - Assert: Every retained handle is invalid exactly once and every lookup misses. + - Secondary observable: A subsequent `Clear` is idempotent. + +5. `CreateGrainDirectoryCache_ExpirationInvalidatesRouteHandle` + - Arrange: Use `FakeTimeProvider`, the configured TTL, and `ConcurrentLruCacheExpirationCleanupListener`; add an entry and retain its handle. + - Act: Advance fake time beyond TTL and deterministically invoke/wait for expiration cleanup without wall-clock delays. + - Assert: The handle is invalid and lookup misses. + - Secondary observable: An unexpired control entry remains valid until its own deadline. + +6. `CreateGrainDirectoryCache_EvictionInvalidatesRouteHandle` + - Arrange: Use deterministic capacity `3`; add three entries, establish the intended LRU order, retain all handles, then prepare a fourth. + - Act: Add the fourth entry. + - Assert: Only the least-recently-used entry's handle is invalidated and its lookup misses. + - Secondary observable: The other three entries/handles remain usable. + +#### 2. CachedGrainLocatorTests.cs + +- **Project**: `test/Orleans.Core.Tests/Orleans.Core.Tests.csproj` +- **Test File**: `test/Orleans.Core.Tests/Directory/CachedGrainLocatorTests.cs` +- **Test Class**: `CachedGrainLocatorTests` +- **Methods/contracts**: asynchronous `Lookup` cache binding and `Unregister` invalidation. +- **Current status**: Lookup/unregister races are regression-testable now; generation/tombstone assertions are blocked. Existing `UnregisterRacesWithLookupSameId` remains the nearest current regression. +- **Future seam/blocker**: Binding must accept/check an atomic generation or token so an await continuation cannot bind a disposed or superseded handle. + +**Tests**: + +1. `Lookup_WhenInvalidatedBeforeAsyncLookupCompletes_DoesNotBindStaleRoute` + - Arrange: Substitute the custom directory and gate its lookup with `TaskCompletionSource` instances created using `RunContinuationsAsynchronously`. Start `Lookup`, wait until the fake records entry, retain the pending route handle/generation, then invalidate or supersede that entry. + - Act: Complete the old directory lookup with a valid-looking address and await the original operation. + - Assert: The stale generation is not bound to the returned address/target. + - Secondary observable: A subsequent lookup takes the ordinary directory path (recorded call count) or uses only the newer generation; it never uses the stale receiver/connection. + +2. `Unregister_WhenLookupCompletesAfterRemoval_DoesNotResurrectRouteHandle` + - Arrange: Start a gated `Lookup` for a grain and wait until the directory call is in flight. + - Act: Call `Unregister`/remove first, then release the old lookup result. + - Assert: The removed handle remains an invalid tombstone and no live cache entry is resurrected. + - Secondary observable: The next lookup invokes the directory again and can bind only a newly issued generation. + - Cleanup: Release all gates in `finally` so failures cannot leave tasks blocked. + +### Phase 1 Success Criteria + +- [ ] Production API landing gate is satisfied. +- [ ] Six lifetime paths invalidate exactly the intended handles. +- [ ] Both race tests use deterministic gates, not delays. +- [ ] Scoped runtime and core builds pass. +- [ ] Both scoped test classes pass. + +--- + +## Phase 2: Shared Tombstones and Direct Route Validation + +### Overview + +Verify shared identity/lifetime first, then local and remote fast-path validity. These tests cover top-level route consumption while isolating fallback behavior with fakes and call counters. + +### Production Scope + +Assigned to this phase: + +- `src/Orleans.Core.Abstractions/Runtime/GrainReference.cs` +- `src/Orleans.Runtime/Catalog/ActivationDirectory.cs` +- `src/Orleans.Runtime/Catalog/Catalog.cs` +- `src/Orleans.Runtime/Catalog/ActivationData.cs` +- `src/Orleans.Core.Abstractions/Core/IGrainContext.cs` +- `src/Orleans.Core/Networking/Connection.cs` +- `src/Orleans.Core/Networking/ConnectionManager.cs` +- `src/Orleans.Runtime/Messaging/MessageCenter.cs` + +### File to Test + +#### MessageTargetCacheTests.cs + +- **Project**: `test/Orleans.Core.Tests/Orleans.Core.Tests.csproj` +- **Test File**: `test/Orleans.Core.Tests/Runtime/MessageTargetCacheTests.cs` (new, only after API landing) +- **Test Class**: `MessageTargetCacheTests` +- **Methods/contracts**: future shared route acquisition/invalidation, local receiver probe, remote connection-group probe, and fallback. +- **Fixtures/fakes**: NSubstitute for interfaces; a controllable collectible grain-context fake exposing exact `GrainAddress` and `IsValid`; a controllable connection/group fake exposing address, group identity, and `IsValid`; ordinary lookup/acquisition delegates with invocation counters. +- **Future seam/blocker**: An observable atomic route-handle/tombstone, weak local receiver retention after invalidation, remote group identity/address validation, and ordinary fallback hooks are all required. + +**Shared identity and lifetime tests**: + +1. `GrainReferences_WithSameGrainIdShareRouteHandle` + - Arrange: Create two grain references for the same `GrainId` through the normal activator/reference path, including different compatible interface views if supported. + - Act: Obtain the supported route-handle view and bind/invalidate through one reference. + - Assert: Both references expose the same handle instance and observe the same state transition. + - Secondary observable: The supported route-entry count increases once, not once per reference/interface view. + +2. `GrainReference_RetainsDisposedRouteHandleTombstone` + - Arrange: Create a grain reference and its directory entry; retain the reference, dispose/remove the entry, and drop other strong references to the live entry. + - Act: Force bounded GC only if needed to separate entry lifetime from reference lifetime. + - Assert: The grain reference still retains the same invalid tombstone, rather than silently acquiring a new live handle. + - Secondary observable: A route probe through that reference falls back and cannot use the former target. + +3. `DisposedRouteHandle_DoesNotRetainLocalReceiver` + - Arrange: In a `[MethodImpl(MethodImplOptions.NoInlining)]` helper, create a collectible local receiver, bind it to the handle, return a retained grain reference plus `WeakReference` to the receiver, invalidate the handle, and ensure the helper drops all receiver/context strong references. + - Act: Perform a bounded collect/finalize/collect loop, following the collectible-context test pattern; do not use timing sleeps. + - Assert: The receiver weak reference is no longer alive while the grain reference and tombstone remain alive. + - Secondary observable: The retained tombstone is invalid and its local probe takes fallback. + +**Local route tests**: + +4. `LocalRoute_WhenAddressMatchesAndActivationIsValidUsesReceiver` + - Arrange: Bind a local receiver whose `IGrainContext.Address` exactly equals the cached `GrainAddress` and whose collectible context reports `IsValid = true`; install an ordinary locator fallback counter. + - Act: Resolve/send through the local route probe. + - Assert: The exact receiver is selected. + - Secondary observable: Ordinary locator/fallback is not called and no different activation receives the message. + +5. `LocalRoute_WhenActivationAddressDiffersFallsBack` + - Arrange: Bind a receiver with the same grain identity but a different activation/address instance; configure fallback to return a known target. + - Act: Probe the cached local route. + - Assert: The cached receiver is rejected and the known fallback result is selected. + - Secondary observable: Fallback is called exactly once and the mismatched receiver is not sent to. + +6. `LocalRoute_WhenActivationIsInvalidFallsBack` + - Arrange: Bind a receiver with an exactly matching address but `IsValid = false`; configure a known fallback. + - Act: Probe the cached local route. + - Assert: The invalid receiver is rejected and fallback is selected. + - Secondary observable: Fallback is called once and no direct send occurs. + +**Remote route tests**: + +7. `RemoteRoute_WhenConnectionGroupMatchesAddressUsesConnection` + - Arrange: Bind a valid connection from the exact cached connection group associated with the destination `SiloAddress`; install an ordinary `ConnectionManager` acquisition counter. + - Act: Resolve/send through the remote route. + - Assert: The bound connection is selected. + - Secondary observable: Ordinary connection acquisition is not called and destination address remains exact. + +8. `RemoteRoute_WhenConnectionGroupAddressDiffersFallsBack` + - Arrange: Bind a valid connection/group for silo A while the message destination is silo B; configure ordinary acquisition to return a known connection for B. + - Act: Probe/send the cached route. + - Assert: The stale group/connection is rejected and the B fallback connection is selected. + - Secondary observable: Ordinary acquisition is called once and the A connection receives no send. + +9. `RemoteRoute_WhenConnectionIsInvalidFallsBack` + - Arrange: Bind the exact destination group but make the selected connection report `IsValid = false`; configure a valid ordinary fallback connection. + - Act: Probe/send the route. + - Assert: The invalid cached connection is rejected and fallback is selected. + - Secondary observable: Ordinary acquisition is called once and the invalid connection receives no send. + +### Phase 2 Success Criteria + +- [ ] Shared references demonstrably use one observable handle. +- [ ] Disposed handles remain tombstones but do not root receivers. +- [ ] Local probes require exact address and valid state. +- [ ] Remote probes require exact group/address and valid connection. +- [ ] Every rejection proves both non-use of the stale target and use of ordinary fallback. +- [ ] Scoped core build and `MessageTargetCacheTests` pass. + +--- + +## Phase 3: Placement Compatibility, Invalidation Headers, and Growth Guard + +### Overview + +Protect existing routing behavior around the new fast path, then add a durable supported-count guard against accidental per-client-`GrainId` growth. + +### Production Scope + +Assigned to this phase: + +- `src/Orleans.Runtime/Placement/PlacementService.cs` +- `src/Orleans.Runtime/Core/InsideRuntimeClient.cs` +- `src/Orleans.Runtime/GrainDirectory/ClientGrainLocator.cs` +- `src/Orleans.Runtime/GrainDirectory/GrainLocatorResolver.cs` +- `src/Orleans.Runtime/GrainDirectory/GrainLocator.cs` +- `src/Orleans.Core/GrainReferences/GrainReferenceActivator.cs` +- `src/Orleans.Core/Messaging/ClientMessageCenter.cs` +- `src/Orleans.Core/Runtime/OutsideRuntimeClient.cs` + +### Files to Test + +#### 1. PlacementServiceTests.cs + +- **Project**: `test/Orleans.Core.Tests/Orleans.Core.Tests.csproj` +- **Test File**: `test/Orleans.Core.Tests/Runtime/PlacementServiceTests.cs` +- **Test Class/fixture**: Existing `PlacementServiceTests` and `PlacementServiceFixture` +- **Methods/contracts**: `AddressMessage`, invalidation-header handling, and compatibility path selection. +- **Current status**: Header bypass and current locator/factory selection are regression-testable now, but all additions remain gated. Shared-route non-interference assertions require the future API. +- **Fixtures/fakes**: Reuse `PlacementServiceFixture`, `MockClusterMembershipService`, `SiloLifecycleSubject`, and NSubstitute. Explicitly stop lifecycle resources and dispose in `finally`. +- **Future seam/blocker**: Fixture-visible route binding/state and supported interaction points for the ordinary locator/placement path. + +**Tests**: + +1. `AddressMessage_CacheInvalidationHeaderBypassesSharedRouteFastPath` + - Arrange: Bind a shared fast route to stale address/receiver A; create a message carrying a matching cache-invalidation header; configure ordinary locator/worker resolution to return B. + - Act: Call `AddressMessage`. + - Assert: The message is addressed through B, not A, and the stale handle is invalidated/bypassed. + - Secondary observable: Ordinary locator/placement is called once and A receives no send. + +2. `AddressMessage_CacheInvalidationHeaderWithReplacementUsesReplacementRoute` + - Arrange: Bind stale route A and create an invalidation header containing valid replacement address B; configure locator cache update observation. + - Act: Call `AddressMessage`. + - Assert: The resulting destination/route is B and A is not reused. + - Secondary observable: The locator/cache records the replacement once and the retained A handle remains invalid. + +3. `AddressMessage_CustomDirectoryCacheUsesExistingPlacementPath` + - Arrange: Configure the custom directory-cache strategy and a substitute custom cache/locator with a known result; capture the supported route-entry count. + - Act: Address a message for the applicable directory grain. + - Assert: The established custom-cache/placement result is used. + - Secondary observable: The custom dependency receives the expected call and no unsupported shared fast-route entry is created. + +4. `AddressMessage_DisabledDirectoryCacheUsesExistingPlacementPath` + - Arrange: Configure caching as disabled/none and a known ordinary locator/placement result. + - Act: Address the message. + - Assert: The ordinary existing path supplies the destination. + - Secondary observable: No shared directory route is consulted or retained; supported count remains unchanged. + +5. `AddressMessage_DefaultDirectoryGrainUsesExistingPlacementPath` + - Arrange: Use a normal default-directory/DHT grain and configure its existing locator/worker result. + - Act: Address the message. + - Assert: The existing default-directory placement path produces the expected destination. + - Secondary observable: The expected locator interaction occurs once; no custom/client locator is called. + +6. `AddressMessage_ClientGrainUsesExistingPlacementPath` + - Arrange: Use a client `GrainId` and configure the existing client locator/routing result. + - Act: Address the message. + - Assert: The client-grain path produces the expected destination. + - Secondary observable: No directory-entry route is allocated and directory locator/cache calls remain absent. + +7. `SendMessage_SystemTargetUsesExistingFullyAddressedPath` + - Arrange: Create a fully addressed system-target message with a known local or remote target and record the supported directory-route count. + - Act: Send the message through the normal `MessageCenter` path. + - Assert: The existing system-target send path selects the exact addressed target without consulting or allocating a directory-entry route. + - Secondary observable: The expected direct local/remote send occurs once, directory locator/cache calls remain absent, and the supported directory-route count is unchanged. + +#### 2. MessageTargetCacheTests.cs — Growth Slice + +- **Project**: `test/Orleans.Core.Tests/Orleans.Core.Tests.csproj` +- **Test File**: `test/Orleans.Core.Tests/Runtime/MessageTargetCacheTests.cs` +- **Test Class**: `MessageTargetCacheTests` +- **Future seam/blocker**: A supported route-entry count/accessor partitioned by grain kind. Never inspect dictionary/private field names by reflection. + +1. `RouteCache_ManyClientGrainIdsDoesNotCreatePerGrainEntries` + - Arrange: Record the supported client-route and directory-route counts. Create/address a large deterministic set of unique client `GrainId` values through the public activator/client path. Include one supported directory grain as a control proving the accessor can observe an eligible entry. + - Act: Exercise route acquisition for all client IDs and read the supported counters. + - Assert: Client-grain route-entry count remains at baseline (normally zero), while the control changes only its expected supported count. + - Secondary observable: Client routing still succeeds through the existing fixed client sender-bucket/weak-connection path, demonstrating that bounded behavior is functional rather than skipped. + - Synchronization/GC: No private reflection and no GC-based size inference; use the supported accessor after all deterministic operations complete. + +### Phase 3 Success Criteria + +- [ ] Matching invalidation headers bypass stale fast routes. +- [ ] Replacement headers select and record the replacement route. +- [ ] Custom and disabled caches plus client and system-target non-directory types retain their existing behavior; the default-directory path remains a compatibility control. +- [ ] Many unique client IDs do not create per-grain route entries. +- [ ] Scoped placement, route-cache, and locator-resolver regressions pass. +- [ ] Full build, discovery, and full test commands pass. + +--- + +## Requirement-to-Test Traceability + +| Acceptance behavior | Exact project/file | Exact proposed test name(s) | Preparation status / production seam | +|---|---|---|---| +| Replacement invalidates shared handle | `Orleans.Runtime.Tests` / `Directories/GrainDirectoryCacheFactoryTests.cs` | `CreateGrainDirectoryCache_AddOrUpdateInvalidatesReplacedRouteHandle` | Handle assertion blocked; needs disposable route-owning entry | +| Remove by grain ID invalidates | Same | `CreateGrainDirectoryCache_RemoveByGrainIdInvalidatesRouteHandle` | Handle assertion blocked | +| Remove by address affects only exact match | Same | `CreateGrainDirectoryCache_RemoveByAddressInvalidatesOnlyMatchingRouteHandle` | Handle assertion blocked | +| Clear invalidates all | Same | `CreateGrainDirectoryCache_ClearInvalidatesAllRouteHandles` | Handle assertion blocked | +| TTL invalidates | Same | `CreateGrainDirectoryCache_ExpirationInvalidatesRouteHandle` | Mechanics possible now; handle blocked; use fake time/listener | +| Capacity eviction invalidates | Same | `CreateGrainDirectoryCache_EvictionInvalidatesRouteHandle` | Mechanics possible now; handle blocked; capacity 3 | +| Stale async bind cannot resurrect route | `Orleans.Core.Tests` / `Directory/CachedGrainLocatorTests.cs` | `Lookup_WhenInvalidatedBeforeAsyncLookupCompletes_DoesNotBindStaleRoute`; `Unregister_WhenLookupCompletesAfterRemoval_DoesNotResurrectRouteHandle` | Race regression possible now; generation/tombstone blocked | +| References share route state | `Orleans.Core.Tests` / `Runtime/MessageTargetCacheTests.cs` | `GrainReferences_WithSameGrainIdShareRouteHandle` | Fully blocked on shared handle API | +| Reference retains disposed tombstone | Same | `GrainReference_RetainsDisposedRouteHandleTombstone` | Fully blocked | +| Tombstone does not retain receiver | Same | `DisposedRouteHandle_DoesNotRetainLocalReceiver` | Fully blocked; needs weak retention and observable tombstone | +| Local route requires exact address and valid receiver | Same | `LocalRoute_WhenAddressMatchesAndActivationIsValidUsesReceiver`; `LocalRoute_WhenActivationAddressDiffersFallsBack`; `LocalRoute_WhenActivationIsInvalidFallsBack` | Existing primitives testable; direct route probe blocked | +| Remote route requires matching group/address and valid connection, with fallback | Same | `RemoteRoute_WhenConnectionGroupMatchesAddressUsesConnection`; `RemoteRoute_WhenConnectionGroupAddressDiffersFallsBack`; `RemoteRoute_WhenConnectionIsInvalidFallsBack` | Existing manager behavior testable; cached group route blocked | +| Invalidation header bypasses/replaces fast route | `Orleans.Core.Tests` / `Runtime/PlacementServiceTests.cs` | `AddressMessage_CacheInvalidationHeaderBypassesSharedRouteFastPath`; `AddressMessage_CacheInvalidationHeaderWithReplacementUsesReplacementRoute` | Header regression possible now; shared-route observation blocked | +| Custom/disabled caches and non-directory grain types use the existing path | `Orleans.Core.Tests` / `Runtime/PlacementServiceTests.cs` and `Runtime/MessageTargetCacheTests.cs` | `AddressMessage_CustomDirectoryCacheUsesExistingPlacementPath`; `AddressMessage_DisabledDirectoryCacheUsesExistingPlacementPath`; `AddressMessage_ClientGrainUsesExistingPlacementPath`; `SendMessage_SystemTargetUsesExistingFullyAddressedPath` | Existing selection/send behavior possible now; shared-route non-interference count blocked | +| Default-directory compatibility control retains existing path | `Orleans.Core.Tests` / `Runtime/PlacementServiceTests.cs` | `AddressMessage_DefaultDirectoryGrainUsesExistingPlacementPath` | Existing locator selection possible now; route interaction observation blocked | +| No per-client-`GrainId` route growth | `Orleans.Core.Tests` / `Runtime/MessageTargetCacheTests.cs` | `RouteCache_ManyClientGrainIdsDoesNotCreatePerGrainEntries` | Blocked on supported kind-aware count/accessor | + +## Implementation Order and Validation + +1. Confirm the landed API supplies all six required seams: observable atomic tombstone, disposable owning entry, generation-aware bind, exact local probe, group-aware remote probe, and kind-aware count/accessor. +2. Implement Phase 1 tests and run only its scoped builds/tests. +3. Implement Phase 2 tests, using the new cohesive test file, and run its scoped core build/test. +4. Implement Phase 3 tests and run placement, message-target-cache, and locator-resolver scopes. +5. Run full build, discovery, and full test only after all scoped checks pass. +6. Run the mandatory final pre-completion gate against the final test set: invoke `test-gap-analysis` for each tested source/test pair, fix every in-scope mutation gap, invoke `assertion-quality`, fix weak/trivial assertions, and manually map every prompt scenario to the exact tests in this plan. +7. If coverage-gap review adds or changes a test, repeat the complete pre-completion gate. +8. Write `.testagent/status.md` only after implementation, quality review, and validation, recording checklist evidence, commands/results, quality findings/fixes, and blockers. +9. Preserve repository xUnit v3/MTP metadata: `[Fact]`/`[Theory]`, class-level `[TestSuite("BVT")]`, `[TestProvider("None")]`, appropriate `[TestCategory]` and `[TestArea]`, async `Task`, and focused assertions. diff --git a/.testagent/research.md b/.testagent/research.md new file mode 100644 index 00000000000..9927eb16884 --- /dev/null +++ b/.testagent/research.md @@ -0,0 +1,147 @@ +# Test Generation Research + +> Final scope update: benchmarking showed material benefit only for in-process activation dispatch. The implemented cache target is therefore limited to `IGrainContext`; remote connection and response-route caching are deferred to separate experiments. + +## Project Overview +- **Path**: `C:\dev\copilot-worktrees\orleans\rb-scaling-guacamole` +- **Boundary**: Orleans runtime directory caching, silo/client message routing, local activation lookup, connection selection, and grain-reference creation. Samples, providers, streaming, persistence, and unrelated tests are out of scope. +- **Language / framework**: C# on SDK-style .NET; source targets `net8.0;net10.0`, and `global.json` selects SDK `10.0.400` with major roll-forward. +- **Test framework**: xUnit v3 `3.2.2` hosted by Microsoft Testing Platform (MTP) `2.3.3`. +- **Dependencies**: Central `PackageReference` versions in `Directory.Packages.props`; NSubstitute `5.3.0`, NSubstitute analyzer `1.0.17`, Microsoft.Extensions.TimeProvider.Testing `9.10.0`, and AwesomeAssertions `9.3.0`. +- **New-file registration**: SDK implicit `Compile` glob. A future `*.cs` test file needs no project edit. +- **Current state**: The shared directory-entry message-target/route-handle API does not exist. `GrainReference` currently stores only `GrainReferenceShared` and `IdSpan`; `IGrainDirectoryCache` stores `GrainAddress`; `PlacementService` resolves a silo for every not-fully-addressed message. + +## Static Pairing Heuristic +- The required Roslyn analyzer was executed **exactly once**, at the repository root. This is the narrowest common root because the bounded production files are under `src/` and their tests are under `test/`. +- Analyzer diagnostics: 3,844 C# files discovered; 2,837 source and 1,007 test files; indexing/scanning completed in 3.192 seconds. +- The file-based SDK emitted a leading `C...` build line into captured stdout, so `ConvertFrom-Json` rejected the combined stream before the relevant `source_to_tests` and `suggested_test_path` nodes could be extracted. It was not rerun, honoring the exactly-once constraint. +- Consequently, the pairings below are direct, bounded source/test references and naming conventions, **not recovered analyzer pairings**. The analyzer is a static identifier-pairing heuristic in any case, not line or branch coverage. + +## Dependency Graph and Production Seams +- **Leaf types**: + - `src/Orleans.Runtime/Catalog/ActivationDirectory.cs` — grain-id-to-context table; exact-instance conditional removal. + - `src/Orleans.Core.Abstractions/Runtime/GrainReference.cs` — reference identity and invocation surface. + - `src/Orleans.Runtime/GrainDirectory/IGrainDirectoryCache.cs` — current address-only cache contract. + - `src/Orleans.Core/Networking/Connection.cs` — exposes `IsValid`. +- **Mid-layer types**: + - `src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs` — wraps `ConcurrentLruCache`; replacement/removal/clear/expiry/eviction dispose cache values only when the value itself implements `IDisposable`. + - `src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheFactory.cs` and `Configuration/Options/GrainDirectoryOptions.cs` — LRU/custom/none selection and keyed `TimeProvider`. + - `src/Orleans.Runtime/GrainDirectory/CachedGrainLocator.cs` — async custom-directory lookup and cache population/invalidation. + - `src/Orleans.Runtime/GrainDirectory/ClientGrainLocator.cs`, `GrainLocatorResolver.cs`, and `GrainLocator.cs` — client/DHT/custom-directory path selection. + - `src/Orleans.Core/Networking/ConnectionManager.cs` — per-silo connection groups; `NextConnection()` validates the selected connection and the slow path removes defunct connections. + - `src/Orleans.Core/GrainReferences/GrainReferenceActivator.cs` — bounded today by `(GrainType, GrainInterfaceType)`, not `GrainId`. +- **Top-layer types**: + - `src/Orleans.Runtime/Placement/PlacementService.cs` — current cache fast path and cache-invalidation-header validation. + - `src/Orleans.Runtime/Messaging/MessageCenter.cs` — local loopback, remote send, fallback connection acquisition, and address/send sequencing. + - `src/Orleans.Runtime/Core/InsideRuntimeClient.cs` — applies invalidation headers before response handling. + - `src/Orleans.Core/Messaging/ClientMessageCenter.cs` and `Runtime/OutsideRuntimeClient.cs` — client path; routing is a fixed-size array of weak connection references, not a per-`GrainId` dictionary. + - `src/Orleans.Runtime/Catalog/Catalog.cs`, `ActivationData.cs`, and `src/Orleans.Core.Abstractions/Core/IGrainContext.cs` — local receiver address and validity. +- **Required future seams**: + 1. An observable shared route-handle/tombstone with atomic bind/invalidate state. + 2. A disposable directory-cache entry which owns that handle, so every LRU removal path invalidates it. + 3. Generation/token-aware binding so an await continuation cannot bind a disposed or superseded handle. + 4. A local-route probe which validates both exact `GrainAddress` and receiver validity without strongly retaining the receiver after invalidation. + 5. A remote-route probe which validates destination `SiloAddress`, connection-group identity, and connection validity, then exposes the ordinary fallback path. + 6. A test accessor/counter for route entries by grain kind; do not test private field names by reflection. + +## Build & Test Commands +Run from the repository root. `--no-restore` preserves the no-restore constraint once dependencies are already available. + +- **Build (scoped)**: + - `dotnet build test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore` + - `dotnet build test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj --framework net10.0 --no-restore` +- **Build (full)**: `dotnet build Orleans.slnx --no-restore --no-incremental -bl` +- **Test (scoped — future core route tests)**: `dotnet test --project test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore --filter-class "*MessageTargetCacheTests*" --minimum-expected-tests 1` +- **Test (scoped — existing cache file)**: `dotnet test --project test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj --framework net10.0 --no-restore --filter-class "*GrainDirectoryCacheFactoryTests*" --minimum-expected-tests 1` +- **Test (scoped — current regression files)**: substitute one of `*PlacementServiceTests*`, `*CachedGrainLocatorTests*`, or `*GrainLocatorResolverTests*` for `*MessageTargetCacheTests*` in the core command. +- **Test (full, all target frameworks with a fresh build)**: `dotnet test --solution Orleans.slnx --no-restore --minimum-expected-tests 1 --max-parallel-test-modules 1` +- **Harness-equivalent discovery (all target frameworks)**: `dotnet test --solution Orleans.slnx --no-restore --list-tests --minimum-expected-tests 1 --max-parallel-test-modules 1` +- **Lint**: no separate command; `dotnet build` enforces style and treats warnings/nullable warnings as errors. + +## Exact Test Targets + +### Projects +1. `test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj` + - Target: `src/Orleans.Runtime/Orleans.Runtime.csproj`. + - Canonical file: `test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs`. +2. `test/Orleans.Core.Tests/Orleans.Core.Tests.csproj` + - Targets: `src/Orleans.Core/Orleans.Core.csproj`, `src/Orleans.Core.Abstractions/Orleans.Core.Abstractions.csproj`, and `src/Orleans.Runtime/Orleans.Runtime.csproj`. + - Canonical files: `Directory/CachedGrainLocatorTests.cs`, `Directory/GrainLocatorResolverTests.cs`, and `Runtime/PlacementServiceTests.cs`. + - One cohesive **future** file, added only after the API exists: `test/Orleans.Core.Tests/Runtime/MessageTargetCacheTests.cs`. + +### Exact proposed test names + +`test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs`: +- `CreateGrainDirectoryCache_AddOrUpdateInvalidatesReplacedRouteHandle` +- `CreateGrainDirectoryCache_RemoveByGrainIdInvalidatesRouteHandle` +- `CreateGrainDirectoryCache_RemoveByAddressInvalidatesOnlyMatchingRouteHandle` +- `CreateGrainDirectoryCache_ClearInvalidatesAllRouteHandles` +- `CreateGrainDirectoryCache_ExpirationInvalidatesRouteHandle` +- `CreateGrainDirectoryCache_EvictionInvalidatesRouteHandle` + +`test/Orleans.Core.Tests/Directory/CachedGrainLocatorTests.cs`: +- `Lookup_WhenInvalidatedBeforeAsyncLookupCompletes_DoesNotBindStaleRoute` +- `Unregister_WhenLookupCompletesAfterRemoval_DoesNotResurrectRouteHandle` + +`test/Orleans.Core.Tests/Runtime/PlacementServiceTests.cs`: +- `AddressMessage_CacheInvalidationHeaderBypassesSharedRouteFastPath` +- `AddressMessage_CacheInvalidationHeaderWithReplacementUsesReplacementRoute` +- `AddressMessage_CustomDirectoryCacheUsesExistingPlacementPath` +- `AddressMessage_DisabledDirectoryCacheUsesExistingPlacementPath` +- `AddressMessage_DefaultDirectoryGrainUsesExistingPlacementPath` +- `AddressMessage_ClientGrainUsesExistingPlacementPath` +- `SendMessage_SystemTargetUsesExistingFullyAddressedPath` + +`test/Orleans.Core.Tests/Runtime/MessageTargetCacheTests.cs` (future file): +- `GrainReferences_WithSameGrainIdShareRouteHandle` +- `GrainReference_RetainsDisposedRouteHandleTombstone` +- `DisposedRouteHandle_DoesNotRetainLocalReceiver` +- `LocalRoute_WhenAddressMatchesAndActivationIsValidUsesReceiver` +- `LocalRoute_WhenActivationAddressDiffersFallsBack` +- `LocalRoute_WhenActivationIsInvalidFallsBack` +- `RemoteRoute_WhenConnectionGroupMatchesAddressUsesConnection` +- `RemoteRoute_WhenConnectionGroupAddressDiffersFallsBack` +- `RemoteRoute_WhenConnectionIsInvalidFallsBack` +- `RouteCache_ManyClientGrainIdsDoesNotCreatePerGrainEntries` + +## Acceptance Checklist and Blockers +1. **Replacement/removal/clear/TTL/eviction invalidates a shared route handle** — **blocked on future API** for the handle assertion. Cache mechanics are regression-testable now: factory TTL coverage exists, and `ConcurrentLruCache` already disposes `IDisposable` values on update/remove/clear/expiry/eviction. Use `FakeTimeProvider` plus `ConcurrentLruCacheExpirationCleanupListener`; capacity `3` is the minimum deterministic eviction fixture. +2. **Stale asynchronous binding cannot resurrect invalid entries** — **partly regression-testable now** via `CachedGrainLocator` lookup/unregister races; handle generation/tombstone assertions are blocked. Existing `UnregisterRacesWithLookupSameId` is the closest regression. +3. **Grain references retain disposed tombstones without retaining receivers** — **fully blocked**. There is no route field/tombstone today. Use a `[MethodImpl(NoInlining)]` setup helper, `WeakReference`, and bounded collect/finalize/collect attempts, following `ProviderRegistrationResolverTests.GeneratedRegistry_DoesNotRootCollectibleAssemblyLoadContext`. +4. **Local activation exact-address/state validation** — exact-instance removal and validity are regression-testable in `ActivationDirectory`/`LocalActivationStatusChecker`; direct-route validation against `IGrainContext.Address` and `ICollectibleGrainContext.IsValid` is **blocked**. +5. **Remote connection-group route validation and fallback** — current `ConnectionManager` valid/defunct selection and `MessageCenter` fallback are regression-testable; cached route/group identity is **blocked**. +6. **Cache invalidation headers bypass the fast path** — **regression-testable now**. `PlacementService.CachedAddressIsValid` updates/invalidates the locator and rejects a matching invalid address before worker lookup. Add the named tests to the existing `PlacementServiceFixture`. +7. **Custom/disabled caches and non-directory grain types retain the existing path** — **regression-testable now** for factory and locator selection; future shared-route non-interference needs the new API. Treat client grains and system targets as the explicit non-directory cases. Existing coverage includes custom/none factory tests and DHT/client/custom locator resolver tests; add the named system-target send guard after the new API lands. +8. **The original unbounded client `GrainId` mapping does not return** — current code already has no such map: `GrainReferenceActivator` keys by type/interface and `ClientMessageCenter` uses a fixed `ClientSenderBuckets` array of weak references. A durable count-based guard is **blocked** pending a route-cache test accessor; avoid private-field reflection. +9. **No production modifications and no test source additions during preparation** — satisfied. This turn creates only `.testagent/research.md`. +10. **Exact proposed names and target files/projects** — supplied in “Exact Test Targets”. + +## Existing Tests & Coverage Classification +- `GrainDirectoryCacheFactory.cs` / `LruGrainDirectoryCache.cs` → `test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs`: **partial** for the forthcoming scope. It covers TTL, ownership, custom/none selection, and disposal policy, but no shared route exists. +- `CachedGrainLocator.cs` → `test/Orleans.Core.Tests/Directory/CachedGrainLocatorTests.cs`: **substantial current behavior**, including cache population, dead silos, unregister-first, and races; no route binding. +- `GrainLocatorResolver.cs` / client-DHT-custom selection → `test/Orleans.Core.Tests/Directory/GrainLocatorResolverTests.cs`: **substantial for selection**, only three focused tests. +- `PlacementService.cs` → `test/Orleans.Core.Tests/Runtime/PlacementServiceTests.cs`: **partial**. Strong lifecycle/version/placement fixture, but only the stopped-state test currently exercises `AddressMessage`; no invalidation-header fast-path test. +- `ActivationDirectory.cs` / local receiver validity → indirect uses in `CachedGrainLocatorTests.cs` and `Membership/MembershipSystemTargetTests.cs`: **partial**, no dedicated exact-address/state route tests. +- `ConnectionManager.cs` / `MessageCenter.cs`: **partial integration coverage**, no directly paired connection-group route test found in the bounded test set. +- `GrainReference.cs` / `GrainReferenceActivator.cs`: existing default-cluster reference/cast tests cover identity and casting, but forthcoming route retention is **untested/absent**. +- `ClientMessageCenter.cs`: **partial existing behavior**, with no dedicated bounded test file for per-grain route growth. +- These classifications are structural, not numeric coverage measurements. + +## Testing Patterns and Helpers +- Representative conventions: + - `test/Orleans.Core.Tests/Runtime/PlacementServiceTests.cs` + - `test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs` +- Use xUnit `[Fact]`/`[Theory]`, `Assert.*`, async `Task`, and class-level `[TestSuite("BVT")]`, `[TestProvider("None")]`, `[TestCategory("BVT"), TestCategory("Directory" or "Placement")]`, plus an appropriate `[TestArea]`. +- Prefer deterministic `TaskCompletionSource(...RunContinuationsAsynchronously)`, `FakeTimeProvider`, explicit lifecycle stop, and `try/finally` disposal. +- Reuse `PlacementServiceFixture`, `MockClusterMembershipService`, `SiloLifecycleSubject`, NSubstitute, and `ConcurrentLruCacheExpirationCleanupListener`. +- Assert both the selected result and the bypass/fallback interaction (for example, no stale receiver send and one ordinary locator/connection lookup). Do not rely on timing delays or inspect private fields. + +## Priority +1. After the API lands, implement route-handle ownership/invalidation and stale-bind tests first; they protect lifetime and race correctness. +2. Add local/remote route validation and GC-retention tests. +3. Add header-bypass and compatibility-path regressions. +4. Add the client growth guard using a supported internal test accessor. + +## Preparation Constraints +- No build or test was run because the production API is absent and this was planning-only. +- No restore, clean, reset, stash, deletion, production edit, or test-source addition was performed. diff --git a/.testagent/status.md b/.testagent/status.md new file mode 100644 index 00000000000..e7040d2c194 --- /dev/null +++ b/.testagent/status.md @@ -0,0 +1,30 @@ +# Test Quality Status + +## Scope + +The shared directory-entry message target cache is covered by: + +- `test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs` +- `test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs` + +## Results + +- 28 focused tests pass on `net10.0`. +- No generated test is assertion-free or relies only on presence/truthiness checks. +- Assertions cover equality, identity, type, null, negative behavior, collection state, lifecycle state, and side effects. +- Cache lifetime tests verify replacement, conditional removal, clear, TTL expiration, eviction, tombstoning, target release, one-shot binding, and identity-checked clearing. +- Integration tests verify local target binding, invalidation and recapture, compatible-interface casts, and explicit remote, external-client, and stateless-worker exclusion. + +## Pseudo-Mutation Review + +Three high-risk mutations were injected individually and empirically killed by the focused tests: + +1. Breaking the atomic first-target bind caused six cache lifetime tests to fail. +2. Removing the directory-backed placement guard caused `StatelessWorkerCalls_DoNotAttachDirectoryEntries` to fail. +3. Allowing messages with cache invalidation headers onto the fast path caused `CacheInvalidationHeader_BypassesFastPathWithoutDiscardingLiveHandle` to fail. + +Every mutation was reverted immediately. The focused suites were rerun green afterward. + +## Remaining Risk + +The tests intentionally allow the same stale local-dispatch race as the existing activation-directory lookup path: invalidation can race after a valid target is read. Activation state and address validation preserve the existing recovery semantics. Remote connections and response routing remain on the established path. diff --git a/src/Orleans.Core.Abstractions/Runtime/GrainReference.cs b/src/Orleans.Core.Abstractions/Runtime/GrainReference.cs index 13f53615b76..51c0e771397 100644 --- a/src/Orleans.Core.Abstractions/Runtime/GrainReference.cs +++ b/src/Orleans.Core.Abstractions/Runtime/GrainReference.cs @@ -267,6 +267,18 @@ public class GrainReference : IAddressable, IEquatable, ISpanFor [NonSerialized] private readonly IdSpan _key; + [NonSerialized] + private object? _directoryCacheEntry; + + internal object? DirectoryCacheEntry + { + get => Volatile.Read(ref _directoryCacheEntry); + set => Volatile.Write(ref _directoryCacheEntry, value); + } + + internal void ClearDirectoryCacheEntry(object expected) + => Interlocked.CompareExchange(ref _directoryCacheEntry, null, expected); + /// /// Gets the grain reference functionality which is shared by all grain references of a given type. /// diff --git a/src/Orleans.Core/Runtime/GrainReferenceRuntime.cs b/src/Orleans.Core/Runtime/GrainReferenceRuntime.cs index 2cde50e9d0b..dbdca40fbe9 100644 --- a/src/Orleans.Core/Runtime/GrainReferenceRuntime.cs +++ b/src/Orleans.Core/Runtime/GrainReferenceRuntime.cs @@ -107,7 +107,13 @@ public object Cast(IAddressable grain, Type grainInterface) } var interfaceType = this.interfaceTypeResolver.GetGrainInterfaceType(grainInterface); - return this.referenceActivator.CreateReference(grainId, interfaceType); + var result = this.referenceActivator.CreateReference(grainId, interfaceType); + if (grain is GrainReference source) + { + result.DirectoryCacheEntry = source.DirectoryCacheEntry; + } + + return result; } /// diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index d605549c37e..c077a7c0f5f 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -216,7 +216,7 @@ public void SendRequest( } this.messagingTrace.OnSendRequest(message); - this.MessageCenter.AddressAndSendMessage(message); + this.MessageCenter.AddressAndSendMessage(message, target); } public void SendResponse(Message request, Response response) diff --git a/src/Orleans.Runtime/GrainDirectory/CachedGrainLocator.cs b/src/Orleans.Runtime/GrainDirectory/CachedGrainLocator.cs index bc2c1b6e083..a11c68e0bf2 100644 --- a/src/Orleans.Runtime/GrainDirectory/CachedGrainLocator.cs +++ b/src/Orleans.Runtime/GrainDirectory/CachedGrainLocator.cs @@ -218,6 +218,20 @@ public void UpdateCache(GrainId grainId, SiloAddress siloAddress) } public void InvalidateCache(GrainId grainId) => cache.Remove(grainId); public void InvalidateCache(GrainAddress address) => cache.Remove(address); + + internal bool TryGetCacheEntry(GrainId grainId, SiloAddress siloAddress, [NotNullWhen(true)] out GrainDirectoryCacheEntry? entry) + { + if (cache is IGrainDirectoryCacheEntrySource entrySource + && entrySource.TryGetEntry(grainId, out entry) + && entry.Address.SiloAddress?.Equals(siloAddress) == true) + { + return true; + } + + entry = null; + return false; + } + public bool TryLookupInCache(GrainId grainId, [NotNullWhen(true)] out GrainAddress? address) { var grainType = grainId.Type; diff --git a/src/Orleans.Runtime/GrainDirectory/DhtGrainLocator.cs b/src/Orleans.Runtime/GrainDirectory/DhtGrainLocator.cs index 44dd19dd708..d4926d17038 100644 --- a/src/Orleans.Runtime/GrainDirectory/DhtGrainLocator.cs +++ b/src/Orleans.Runtime/GrainDirectory/DhtGrainLocator.cs @@ -81,6 +81,17 @@ public static DhtGrainLocator FromLocalGrainDirectory(LocalGrainDirectory localG public void InvalidateCache(GrainAddress address) => _localGrainDirectory.InvalidateCacheEntry(address); public bool TryLookupInCache(GrainId grainId, [NotNullWhen(true)] out GrainAddress? address) => _localGrainDirectory.TryLocalLookup(grainId, out address); + internal bool TryGetCacheEntry(GrainId grainId, SiloAddress siloAddress, [NotNullWhen(true)] out GrainDirectoryCacheEntry? entry) + { + if (_localGrainDirectory is LocalGrainDirectory directory) + { + return directory.TryGetCacheEntry(grainId, siloAddress, out entry); + } + + entry = null; + return false; + } + private class BatchedDeregistrationWorker { private const int OperationBatchSizeLimit = 2_000; diff --git a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs new file mode 100644 index 00000000000..964daf09e8d --- /dev/null +++ b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs @@ -0,0 +1,55 @@ +using System.Threading; + +namespace Orleans.Runtime.GrainDirectory; + +internal sealed class GrainDirectoryCacheEntry : IDisposable +{ + private static readonly object Invalidated = new(); + private object? _messageTarget; + + public GrainDirectoryCacheEntry(GrainAddress address, int version) + { + Address = address; + Version = version; + } + + public GrainAddress Address { get; } + + public int Version { get; } + + public bool IsValid => !ReferenceEquals(Volatile.Read(ref _messageTarget), Invalidated); + + public bool TryGetMessageTarget(out IGrainContext? messageTarget) + { + var target = Volatile.Read(ref _messageTarget); + if (ReferenceEquals(target, Invalidated)) + { + messageTarget = null; + return false; + } + + messageTarget = target as IGrainContext; + return messageTarget is not null; + } + + public bool TrySetMessageTarget(IGrainContext messageTarget) + { + ArgumentNullException.ThrowIfNull(messageTarget); + var current = Volatile.Read(ref _messageTarget); + if (ReferenceEquals(current, Invalidated)) + { + return false; + } + + return ReferenceEquals(current, messageTarget) + || current is null && Interlocked.CompareExchange(ref _messageTarget, messageTarget, null) is null; + } + + public void ClearMessageTarget(IGrainContext messageTarget) + { + ArgumentNullException.ThrowIfNull(messageTarget); + Interlocked.CompareExchange(ref _messageTarget, null, messageTarget); + } + + public void Dispose() => Interlocked.Exchange(ref _messageTarget, Invalidated); +} diff --git a/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs b/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs index 85bdcf8afac..4e5355a85ae 100644 --- a/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs +++ b/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs @@ -54,6 +54,26 @@ public GrainLocator(GrainLocatorResolver grainLocatorResolver, DirectoryInstrume public bool TryLookupInCache(GrainId grainId, [NotNullWhen(true)] out GrainAddress? address) => GetGrainLocator(grainId.Type).TryLookupInCache(grainId, out address); + internal bool TryGetCacheEntry( + GrainId grainId, + SiloAddress siloAddress, + [NotNullWhen(true)] out GrainDirectoryCacheEntry? entry) + { + var grainLocator = GetGrainLocator(grainId.Type); + return grainLocator switch + { + CachedGrainLocator cached => cached.TryGetCacheEntry(grainId, siloAddress, out entry), + DhtGrainLocator dht => dht.TryGetCacheEntry(grainId, siloAddress, out entry), + _ => ReturnFalse(out entry), + }; + + static bool ReturnFalse(out GrainDirectoryCacheEntry? result) + { + result = null; + return false; + } + } + public void InvalidateCache(GrainId grainId) => GetGrainLocator(grainId.Type).InvalidateCache(grainId); public void InvalidateCache(GrainAddress address) => GetGrainLocator(address.GrainId.Type).InvalidateCache(address); diff --git a/src/Orleans.Runtime/GrainDirectory/IGrainDirectoryCache.cs b/src/Orleans.Runtime/GrainDirectory/IGrainDirectoryCache.cs index 58201553733..7bd43ab8c11 100644 --- a/src/Orleans.Runtime/GrainDirectory/IGrainDirectoryCache.cs +++ b/src/Orleans.Runtime/GrainDirectory/IGrainDirectoryCache.cs @@ -65,4 +65,9 @@ public static bool LookUp(this IGrainDirectoryCache cache, GrainId key, [NotNull return cache.LookUp(key, out result, out _); } } + + internal interface IGrainDirectoryCacheEntrySource + { + bool TryGetEntry(GrainId key, [NotNullWhen(true)] out GrainDirectoryCacheEntry? entry); + } } diff --git a/src/Orleans.Runtime/GrainDirectory/LocalGrainDirectory.cs b/src/Orleans.Runtime/GrainDirectory/LocalGrainDirectory.cs index 8426509d090..fe5db81b70a 100644 --- a/src/Orleans.Runtime/GrainDirectory/LocalGrainDirectory.cs +++ b/src/Orleans.Runtime/GrainDirectory/LocalGrainDirectory.cs @@ -836,6 +836,19 @@ public bool TryLocalLookup(GrainId grainId, [NotNullWhen(true)] out GrainAddress return IsDefunctActivation(cache, clusterMembershipService.CurrentSnapshot) ? null : cache; } + internal bool TryGetCacheEntry(GrainId grainId, SiloAddress siloAddress, [NotNullWhen(true)] out GrainDirectoryCacheEntry? entry) + { + if (DirectoryCache is IGrainDirectoryCacheEntrySource entrySource + && entrySource.TryGetEntry(grainId, out entry) + && entry.Address.SiloAddress?.Equals(siloAddress) == true) + { + return true; + } + + entry = null; + return false; + } + public async Task LookupAsync(GrainId grainId, int hopCount = 0) { if (hopCount > 0) diff --git a/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs b/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs index c99efdf18b8..697f245a3f3 100644 --- a/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs +++ b/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs @@ -6,9 +6,9 @@ namespace Orleans.Runtime.GrainDirectory; -internal sealed class LruGrainDirectoryCache : ConcurrentLruCache, IGrainDirectoryCache, IAsyncDisposable +internal sealed class LruGrainDirectoryCache : ConcurrentLruCache, IGrainDirectoryCache, IGrainDirectoryCacheEntrySource, IAsyncDisposable { - private static readonly Func<(GrainAddress Address, int Version), GrainAddress, bool> ActivationAddressesMatch = (value, state) => GrainAddress.MatchesGrainIdAndSilo(state, value.Address); + private static readonly Func ActivationAddressesMatch = (value, state) => GrainAddress.MatchesGrainIdAndSilo(state, value.Address); private readonly IDisposable _cacheSizeRegistration; public LruGrainDirectoryCache( @@ -27,7 +27,7 @@ public LruGrainDirectoryCache( : directoryInstruments.RegisterCacheSizeObserve(() => Count); } - public void AddOrUpdate(GrainAddress activationAddress, int version) => AddOrUpdate(activationAddress.GrainId, (activationAddress, version)); + public void AddOrUpdate(GrainAddress activationAddress, int version) => AddOrUpdate(activationAddress.GrainId, new GrainDirectoryCacheEntry(activationAddress, version)); public bool Remove(GrainId key) => TryRemove(key); @@ -35,10 +35,10 @@ public LruGrainDirectoryCache( public bool LookUp(GrainId key, [NotNullWhen(true)] out GrainAddress? result, out int version) { - if (TryGet(key, out var entry)) + if (TryGetEntry(key, out var entry)) { version = entry.Version; - result = entry.ActivationAddress; + result = entry.Address; return true; } @@ -53,11 +53,25 @@ public bool LookUp(GrainId key, [NotNullWhen(true)] out GrainAddress? result, ou { foreach (var entry in this) { - yield return (entry.Value.ActivationAddress, entry.Value.Version); + if (entry.Value.IsValid) + { + yield return (entry.Value.Address, entry.Value.Version); + } } } } + public bool TryGetEntry(GrainId key, [NotNullWhen(true)] out GrainDirectoryCacheEntry? entry) + { + if (TryGet(key, out entry) && entry.IsValid) + { + return true; + } + + entry = null; + return false; + } + public new async ValueTask DisposeAsync() { _cacheSizeRegistration.Dispose(); diff --git a/src/Orleans.Runtime/Messaging/MessageCenter.cs b/src/Orleans.Runtime/Messaging/MessageCenter.cs index d885ddf352b..62660ea9157 100644 --- a/src/Orleans.Runtime/Messaging/MessageCenter.cs +++ b/src/Orleans.Runtime/Messaging/MessageCenter.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -141,7 +143,11 @@ public Action? SniffIncomingMessage get => this.sniffIncomingMessageHandler; } - public void SendMessage(Message msg) + public void SendMessage(Message msg) => SendMessage(msg, null); + + private void SendMessage( + Message msg, + GrainDirectoryCacheEntry? directoryCacheEntry) { Debug.Assert(!msg.IsLocalOnly); @@ -202,13 +208,13 @@ public void SendMessage(Message msg) MessagingEvents.EmitSent(msg); - if (targetSilo.Matches(_siloAddress)) + if (directoryCacheEntry is not null || targetSilo.Matches(_siloAddress)) { LogTraceMessageLoopedBack(log, msg); _messagingInstruments.LocalMessagesSentCounterAggregator.Add(1); - this.ReceiveMessage(msg); + this.ReceiveMessage(msg, directoryCacheEntry); } else { @@ -240,7 +246,10 @@ public void SendMessage(Message msg) { _ = SendAsync(this, connectionTask, msg); - static async Task SendAsync(MessageCenter messageCenter, ValueTask connectionTask, Message msg) + static async Task SendAsync( + MessageCenter messageCenter, + ValueTask connectionTask, + Message msg) { try { @@ -479,17 +488,24 @@ private static bool MayForward(Message message, SiloMessagingOptions messagingOp /// - add ordering info and maintain send order /// /// - internal Task AddressAndSendMessage(Message message) + internal Task AddressAndSendMessage(Message message, GrainReference? target = null) { try { + if (TryGetDirectoryCacheEntry(target, message, out var directoryCacheEntry)) + { + message.TargetSilo = _siloAddress; + SendMessage(message, directoryCacheEntry); + return Task.CompletedTask; + } + var messageAddressingTask = placementService.AddressMessage(message); - if (messageAddressingTask.Status != TaskStatus.RanToCompletion) + if (!messageAddressingTask.IsCompletedSuccessfully) { - return SendMessageAsync(messageAddressingTask, message); + return SendMessageAsync(messageAddressingTask, message, target); } - SendMessage(message); + SendMessage(message, CaptureDirectoryCacheEntry(target, message)); } catch (Exception ex) { @@ -498,7 +514,7 @@ internal Task AddressAndSendMessage(Message message) return Task.CompletedTask; - async Task SendMessageAsync(Task addressMessageTask, Message m) + async Task SendMessageAsync(Task addressMessageTask, Message m, GrainReference? target) { try { @@ -510,7 +526,7 @@ async Task SendMessageAsync(Task addressMessageTask, Message m) return; } - SendMessage(m); + SendMessage(m, CaptureDirectoryCacheEntry(target, m)); } void OnAddressingFailure(Message m, Exception ex) @@ -520,6 +536,51 @@ void OnAddressingFailure(Message m, Exception ex) } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool TryGetDirectoryCacheEntry( + GrainReference? target, + Message message, + [NotNullWhen(true)] out GrainDirectoryCacheEntry? entry) + { + if (target?.DirectoryCacheEntry is not GrainDirectoryCacheEntry candidate) + { + entry = null; + return false; + } + + if (message.CacheInvalidationHeader is null && candidate.IsValid) + { + Debug.Assert(candidate.Address.GrainId.Equals(message.TargetGrain)); + Debug.Assert(candidate.Address.SiloAddress?.Matches(_siloAddress) == true); + entry = candidate; + return true; + } + + if (!candidate.IsValid) + { + target.ClearDirectoryCacheEntry(candidate); + } + + entry = null; + return false; + } + + private GrainDirectoryCacheEntry? CaptureDirectoryCacheEntry(GrainReference? target, Message message) + { + if (target is null + || message.CacheInvalidationHeader is not null + || message.TargetSilo is not { } targetSilo + || !targetSilo.Matches(_siloAddress) + || !placementService.IsUsingGrainDirectory(message.TargetGrain) + || !_grainLocator.TryGetCacheEntry(message.TargetGrain, targetSilo, out var entry)) + { + return null; + } + + target.DirectoryCacheEntry = entry; + return entry; + } + internal void SendResponse(Message request, Response response) { // create the response @@ -534,7 +595,9 @@ internal void SendResponse(Message request, Response response) SendMessage(message); } - public void ReceiveMessage(Message msg) + public void ReceiveMessage(Message msg) => ReceiveMessage(msg, null); + + private void ReceiveMessage(Message msg, GrainDirectoryCacheEntry? directoryCacheEntry) { Debug.Assert(!msg.IsLocalOnly); try @@ -550,10 +613,21 @@ public void ReceiveMessage(Message msg) } else { - var targetActivation = catalog.GetOrCreateActivation( - msg.TargetGrain, - msg.RequestContextData, - rehydrationContext: null); + IGrainContext? targetActivation = GetCachedActivation(directoryCacheEntry); + if (targetActivation is null) + { + targetActivation = catalog.GetOrCreateActivation( + msg.TargetGrain, + msg.RequestContextData, + rehydrationContext: null); + + if (targetActivation is ActivationData { IsValid: true } activation + && directoryCacheEntry is { } entry + && entry.Address.Matches(activation.Address)) + { + entry.TrySetMessageTarget(activation); + } + } if (targetActivation is null) { @@ -579,6 +653,23 @@ void HandleReceiveFailure(Message msg, Exception ex) } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ActivationData? GetCachedActivation(GrainDirectoryCacheEntry? directoryCacheEntry) + { + if (directoryCacheEntry?.TryGetMessageTarget(out var target) == true + && target is ActivationData activation) + { + if (activation.IsValid) + { + return activation; + } + + directoryCacheEntry.ClearMessageTarget(activation); + } + + return null; + } + private void ProcessMessageToNonExistentActivation(Message msg) { var target = msg.TargetGrain; diff --git a/src/Orleans.Runtime/Placement/PlacementService.cs b/src/Orleans.Runtime/Placement/PlacementService.cs index a491cc7e1b0..09b1ab8b136 100644 --- a/src/Orleans.Runtime/Placement/PlacementService.cs +++ b/src/Orleans.Runtime/Placement/PlacementService.cs @@ -103,6 +103,9 @@ Task ITestAccessor.GetOrPlaceActivationAsync(Message message) private bool IsStopping => _shutdownCts.IsCancellationRequested; + internal bool IsUsingGrainDirectory(GrainId grainId) + => _strategyResolver.GetPlacementStrategy(grainId.Type).IsUsingGrainDirectory; + void ILifecycleParticipant.Participate(ISiloLifecycle lifecycle) { lifecycle.Subscribe( diff --git a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs index 12c23e881ce..64910daddb7 100644 --- a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs +++ b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs @@ -1,3 +1,4 @@ +using System.Reflection; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Time.Testing; using Orleans.Configuration; @@ -256,4 +257,394 @@ private sealed class DisposableGrainDirectoryCache : TestGrainDirectoryCache, ID public void Dispose() => DisposeCalled = true; } + + [Fact] + public async Task CreateGrainDirectoryCache_AddOrUpdateInvalidatesReplacedRouteHandle() + { + var (cache, entrySource) = CreateEntryCache(); + var disposableCache = Assert.IsAssignableFrom(cache); + var grainId = CreateGrainId(); + var originalAddress = CreateGrainAddress(grainId, port: 11111); + var replacementAddress = CreateGrainAddress(grainId, port: 22222); + var originalTarget = CreateMessageTarget(); + var replacementTarget = CreateMessageTarget(); + + try + { + cache.AddOrUpdate(originalAddress, version: 1); + var originalEntry = GetEntry(entrySource, grainId); + Assert.True(originalEntry.TrySetMessageTarget(originalTarget)); + + cache.AddOrUpdate(replacementAddress, version: 2); + var replacementEntry = GetEntry(entrySource, grainId); + Assert.True(replacementEntry.TrySetMessageTarget(replacementTarget)); + + Assert.NotSame(originalEntry, replacementEntry); + AssertInvalidEntry(originalEntry); + AssertMessageTarget(replacementEntry, replacementTarget); + Assert.True(cache.LookUp(grainId, out var result, out var version)); + Assert.Equal(replacementAddress, result); + Assert.Equal(2, version); + } + finally + { + await disposableCache.DisposeAsync(); + } + } + + [Fact] + public async Task CreateGrainDirectoryCache_RemoveByGrainIdInvalidatesRouteHandle() + { + var (cache, entrySource) = CreateEntryCache(); + var disposableCache = Assert.IsAssignableFrom(cache); + var address = CreateGrainAddress(CreateGrainId(), port: 11111); + var target = CreateMessageTarget(); + + try + { + cache.AddOrUpdate(address, version: 3); + var entry = GetEntry(entrySource, address.GrainId); + Assert.True(entry.TrySetMessageTarget(target)); + + Assert.True(cache.Remove(address.GrainId)); + + AssertInvalidEntry(entry); + Assert.False(cache.LookUp(address.GrainId, out _, out _)); + Assert.False(cache.Remove(address.GrainId)); + AssertInvalidEntry(entry); + } + finally + { + await disposableCache.DisposeAsync(); + } + } + + [Fact] + public async Task CreateGrainDirectoryCache_RemoveByAddressInvalidatesOnlyMatchingRouteHandle() + { + var (cache, entrySource) = CreateEntryCache(); + var disposableCache = Assert.IsAssignableFrom(cache); + var grainId = CreateGrainId(); + var address = CreateGrainAddress(grainId, port: 11111); + var mismatchedAddress = CreateGrainAddress(grainId, port: 22222); + var controlAddress = CreateGrainAddress(CreateGrainId(), port: 33333); + var target = CreateMessageTarget(); + var controlTarget = CreateMessageTarget(); + + try + { + cache.AddOrUpdate(address, version: 4); + cache.AddOrUpdate(controlAddress, version: 5); + var entry = GetEntry(entrySource, grainId); + var controlEntry = GetEntry(entrySource, controlAddress.GrainId); + Assert.True(entry.TrySetMessageTarget(target)); + Assert.True(controlEntry.TrySetMessageTarget(controlTarget)); + + Assert.False(cache.Remove(mismatchedAddress)); + AssertMessageTarget(entry, target); + Assert.True(cache.LookUp(grainId, out var retainedAddress, out var retainedVersion)); + Assert.Equal(address, retainedAddress); + Assert.Equal(4, retainedVersion); + + Assert.True(cache.Remove(address)); + + AssertInvalidEntry(entry); + Assert.False(cache.LookUp(grainId, out _, out _)); + AssertMessageTarget(controlEntry, controlTarget); + Assert.True(cache.LookUp(controlAddress.GrainId, out var controlResult, out var controlVersion)); + Assert.Equal(controlAddress, controlResult); + Assert.Equal(5, controlVersion); + } + finally + { + await disposableCache.DisposeAsync(); + } + } + + [Fact] + public async Task CreateGrainDirectoryCache_ClearInvalidatesAllRouteHandles() + { + var (cache, entrySource) = CreateEntryCache(); + var disposableCache = Assert.IsAssignableFrom(cache); + var addresses = new[] + { + CreateGrainAddress(CreateGrainId(), port: 11111), + CreateGrainAddress(CreateGrainId(), port: 22222), + CreateGrainAddress(CreateGrainId(), port: 33333) + }; + var entries = new GrainDirectoryCacheEntry[addresses.Length]; + + try + { + for (var i = 0; i < addresses.Length; i++) + { + cache.AddOrUpdate(addresses[i], version: i + 1); + entries[i] = GetEntry(entrySource, addresses[i].GrainId); + Assert.True(entries[i].TrySetMessageTarget(CreateMessageTarget())); + } + + cache.Clear(); + + Assert.Empty(cache.KeyValues); + for (var i = 0; i < addresses.Length; i++) + { + AssertInvalidEntry(entries[i]); + Assert.False(cache.LookUp(addresses[i].GrainId, out _, out _)); + } + + cache.Clear(); + Assert.Empty(cache.KeyValues); + foreach (var entry in entries) + { + AssertInvalidEntry(entry); + } + } + finally + { + await disposableCache.DisposeAsync(); + } + } + + [Fact] + public async Task CreateGrainDirectoryCache_ExpirationInvalidatesRouteHandle() + { + var timeProvider = new FakeTimeProvider(); + var timeToLive = TimeSpan.FromMinutes(1); + var (cache, entrySource) = CreateEntryCache(cacheSize: 10, timeToLive, timeProvider); + var disposableCache = Assert.IsAssignableFrom(cache); + using var listener = new ConcurrentLruCacheExpirationCleanupListener(cache); + var expiredAddress = CreateGrainAddress(CreateGrainId(), port: 11111); + var freshAddress = CreateGrainAddress(CreateGrainId(), port: 22222); + + try + { + cache.AddOrUpdate(expiredAddress, version: 6); + var expiredEntry = GetEntry(entrySource, expiredAddress.GrainId); + Assert.True(expiredEntry.TrySetMessageTarget(CreateMessageTarget())); + + timeProvider.Advance(timeToLive); + Assert.Equal(0, await listener.WaitForCleanupAsync()); + + cache.AddOrUpdate(freshAddress, version: 7); + var freshEntry = GetEntry(entrySource, freshAddress.GrainId); + var freshTarget = CreateMessageTarget(); + Assert.True(freshEntry.TrySetMessageTarget(freshTarget)); + + timeProvider.Advance(timeToLive); + Assert.Equal(1, await listener.WaitForCleanupAsync()); + + AssertInvalidEntry(expiredEntry); + Assert.False(cache.LookUp(expiredAddress.GrainId, out _, out _)); + AssertMessageTarget(freshEntry, freshTarget); + Assert.True(cache.LookUp(freshAddress.GrainId, out var result, out var version)); + Assert.Equal(freshAddress, result); + Assert.Equal(7, version); + } + finally + { + await disposableCache.DisposeAsync(); + } + } + + [Fact] + public async Task CreateGrainDirectoryCache_EvictionInvalidatesRouteHandle() + { + var (cache, entrySource) = CreateEntryCache(cacheSize: 3); + var disposableCache = Assert.IsAssignableFrom(cache); + var lruCache = Assert.IsType(cache); + var addresses = new[] + { + CreateGrainAddress(CreateGrainId(), port: 11111), + CreateGrainAddress(CreateGrainId(), port: 22222), + CreateGrainAddress(CreateGrainId(), port: 33333), + CreateGrainAddress(CreateGrainId(), port: 44444) + }; + var entries = new GrainDirectoryCacheEntry[addresses.Length]; + var targets = new IGrainContext[addresses.Length]; + + try + { + for (var i = 0; i < entries.Length; i++) + { + entries[i] = new GrainDirectoryCacheEntry(addresses[i], version: i + 1); + targets[i] = CreateMessageTarget(); + Assert.True(entries[i].TrySetMessageTarget(targets[i])); + } + + for (var i = 0; i < 3; i++) + { + lruCache.AddOrUpdate(addresses[i].GrainId, entries[i]); + } + + lruCache.AddOrUpdate(addresses[3].GrainId, entries[3]); + + AssertInvalidEntry(entries[0]); + Assert.False(entrySource.TryGetEntry(addresses[0].GrainId, out _)); + for (var i = 1; i < entries.Length; i++) + { + Assert.Same(entries[i], GetEntry(entrySource, addresses[i].GrainId)); + AssertMessageTarget(entries[i], targets[i]); + Assert.True(cache.LookUp(addresses[i].GrainId, out var result, out var version)); + Assert.Equal(addresses[i], result); + Assert.Equal(i + 1, version); + } + + Assert.Equal(3, cache.KeyValues.Count()); + } + finally + { + await disposableCache.DisposeAsync(); + } + } + + [Fact] + public void GrainDirectoryCacheEntry_DisposedEntryCannotBindMessageTarget() + { + var address = CreateGrainAddress(CreateGrainId(), port: 11111); + var entry = new GrainDirectoryCacheEntry(address, version: 8); + + entry.Dispose(); + + AssertInvalidEntry(entry); + Assert.Equal(address, entry.Address); + Assert.Equal(8, entry.Version); + } + + [Fact] + public void GrainDirectoryCacheEntry_SecondTargetCannotReplaceBoundTarget() + { + var entry = new GrainDirectoryCacheEntry(CreateGrainAddress(CreateGrainId(), port: 11111), version: 9); + var originalTarget = CreateMessageTarget(); + + Assert.True(entry.TrySetMessageTarget(originalTarget)); + Assert.False(entry.TrySetMessageTarget(CreateMessageTarget())); + AssertMessageTarget(entry, originalTarget); + } + + [Fact] + public void GrainDirectoryCacheEntry_ClearRequiresBoundTargetIdentity() + { + var entry = new GrainDirectoryCacheEntry(CreateGrainAddress(CreateGrainId(), port: 11111), version: 10); + var originalTarget = CreateMessageTarget(); + var replacementTarget = CreateMessageTarget(); + Assert.True(entry.TrySetMessageTarget(originalTarget)); + + entry.ClearMessageTarget(CreateMessageTarget()); + AssertMessageTarget(entry, originalTarget); + + entry.ClearMessageTarget(originalTarget); + Assert.False(entry.TryGetMessageTarget(out _)); + Assert.True(entry.TrySetMessageTarget(replacementTarget)); + AssertMessageTarget(entry, replacementTarget); + } + + [Fact] + public async Task CreateGrainDirectoryCache_RemoveByGrainIdReleasesMessageTargetReference() + { + var (cache, entrySource) = CreateEntryCache(); + var disposableCache = Assert.IsAssignableFrom(cache); + var address = CreateGrainAddress(CreateGrainId(), port: 11111); + + try + { + cache.AddOrUpdate(address, version: 9); + var entry = GetEntry(entrySource, address.GrainId); + var targetReference = BindMessageTarget(entry); + + Assert.True(cache.Remove(address.GrainId)); + AssertInvalidEntry(entry); + AssertEventuallyCollected(targetReference); + } + finally + { + await disposableCache.DisposeAsync(); + } + } + + private static (IGrainDirectoryCache Cache, IGrainDirectoryCacheEntrySource EntrySource) CreateEntryCache( + int cacheSize = 10, + TimeSpan? timeToLive = null, + FakeTimeProvider? timeProvider = null) + { + var services = new ServiceCollection(); + if (timeProvider is not null) + { + services.AddKeyedSingleton(TimeProviderNames.GrainDirectory, timeProvider); + } + + var cache = GrainDirectoryCacheFactory.CreateGrainDirectoryCache( + services.BuildServiceProvider(), + new GrainDirectoryOptions + { + CacheSize = cacheSize, + MaximumCacheTTL = timeToLive ?? TimeSpan.FromHours(1) + }); + + return (cache, Assert.IsAssignableFrom(cache)); + } + + private static GrainDirectoryCacheEntry GetEntry(IGrainDirectoryCacheEntrySource entrySource, GrainId grainId) + { + Assert.True(entrySource.TryGetEntry(grainId, out var entry)); + return Assert.IsType(entry); + } + + private static void AssertMessageTarget(GrainDirectoryCacheEntry entry, IGrainContext expected) + { + Assert.True(entry.IsValid); + Assert.True(entry.TryGetMessageTarget(out var actual)); + Assert.Same(expected, actual); + } + + private static void AssertInvalidEntry(GrainDirectoryCacheEntry entry) + { + Assert.False(entry.IsValid); + Assert.False(entry.TryGetMessageTarget(out var actual)); + Assert.Null(actual); + Assert.False(entry.TrySetMessageTarget(CreateMessageTarget())); + } + + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static WeakReference BindMessageTarget(GrainDirectoryCacheEntry entry) + { + var target = CreateMessageTarget(); + Assert.True(entry.TrySetMessageTarget(target)); + return new WeakReference(target); + } + + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static void AssertEventuallyCollected(WeakReference targetReference) + { + for (var attempt = 0; attempt < 3; attempt++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + if (!targetReference.TryGetTarget(out _)) + { + return; + } + } + + Assert.False(targetReference.TryGetTarget(out _)); + } + + private static GrainId CreateGrainId() => GrainId.Parse($"user/{Guid.NewGuid():N}"); + + private static IGrainContext CreateMessageTarget() => DispatchProxy.Create(); + + private class GrainContextProxy : DispatchProxy + { + protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) + => targetMethod?.ReturnType.IsValueType == true ? Activator.CreateInstance(targetMethod.ReturnType) : null; + } + + private static GrainAddress CreateGrainAddress(GrainId grainId, int port) => new() + { + ActivationId = ActivationId.NewId(), + GrainId = grainId, + SiloAddress = SiloAddress.FromParsableString($"127.0.0.1:{port}@1"), + MembershipVersion = new MembershipVersion(1) + }; } diff --git a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs new file mode 100644 index 00000000000..a6007c3126d --- /dev/null +++ b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs @@ -0,0 +1,247 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Hosting; +using Orleans.Runtime; +using Orleans.Runtime.GrainDirectory; +using Orleans.Runtime.Messaging; +using Orleans.Runtime.Placement; +using Orleans.TestingHost; +using TestExtensions; +using UnitTests.GrainInterfaces; +using Xunit; + +namespace UnitTests.Messaging; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Runtime")] +[TestCategory("BVT"), TestCategory("Messaging")] +public sealed class SharedEntryMessageTargetFastPathTests : IClassFixture +{ + private static long _nextGrainKey = 10_000_000; + private readonly Fixture _fixture; + + public SharedEntryMessageTargetFastPathTests(Fixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task LocalDirectoryGrain_AfterTwoCalls_BindsEntryToExactActivation() + { + var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; + var grainFactory = GetPrimarySiloGrainFactory(primary); + var grain = grainFactory.GetGrain(Interlocked.Increment(ref _nextGrainKey)); + var grainReference = Assert.IsAssignableFrom(grain); + const string label = "local-fast-path"; + + RequestContext.Set(IPlacementDirector.PlacementHintKey, primary.SiloAddress); + try + { + await grain.SetLabel(label); + Assert.Equal(label, await grain.GetLabel()); + } + finally + { + RequestContext.Remove(IPlacementDirector.PlacementHintKey); + } + + var entry = Assert.IsType(grainReference.DirectoryCacheEntry); + Assert.True(entry.IsValid); + Assert.Equal(grainReference.GrainId, entry.Address.GrainId); + Assert.Equal(primary.SiloAddress, entry.Address.SiloAddress); + Assert.True(_fixture.HostedCluster.TryGetGrainContext(grainReference.GrainId, out var grainContext)); + var activation = Assert.IsType(grainContext); + Assert.Equal(activation.Address, entry.Address); + Assert.True(entry.TryGetMessageTarget(out var messageTarget)); + Assert.Same(activation, messageTarget); + } + + [Fact] + public async Task InvalidateCache_DisposesRetainedEntry_AndNextCallCapturesDifferentEntry() + { + var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; + var grainFactory = GetPrimarySiloGrainFactory(primary); + var grain = grainFactory.GetGrain(Interlocked.Increment(ref _nextGrainKey)); + var grainReference = Assert.IsAssignableFrom(grain); + const string label = "entry-before-invalidation"; + + RequestContext.Set(IPlacementDirector.PlacementHintKey, primary.SiloAddress); + try + { + await grain.SetLabel(label); + Assert.Equal(label, await grain.GetLabel()); + } + finally + { + RequestContext.Remove(IPlacementDirector.PlacementHintKey); + } + + var retainedEntry = Assert.IsType(grainReference.DirectoryCacheEntry); + Assert.True(retainedEntry.IsValid); + primary.ServiceProvider.GetRequiredService().InvalidateCache(grainReference.GrainId); + Assert.False(retainedEntry.IsValid); + Assert.False(retainedEntry.TryGetMessageTarget(out var disposedTarget)); + Assert.Null(disposedTarget); + Assert.Same(retainedEntry, grainReference.DirectoryCacheEntry); + + Assert.Equal(label, await grain.GetLabel()); + + var replacementEntry = Assert.IsType(grainReference.DirectoryCacheEntry); + Assert.NotSame(retainedEntry, replacementEntry); + Assert.True(replacementEntry.IsValid); + Assert.Equal(grainReference.GrainId, replacementEntry.Address.GrainId); + Assert.Equal(primary.SiloAddress, replacementEntry.Address.SiloAddress); + Assert.True(_fixture.HostedCluster.TryGetGrainContext(grainReference.GrainId, out var grainContext)); + var activation = Assert.IsType(grainContext); + Assert.True(replacementEntry.TryGetMessageTarget(out var replacementTarget)); + Assert.Same(activation, replacementTarget); + } + + [Fact] + public async Task RemoteDirectoryGrain_DoesNotAttachLocalTargetEntry() + { + var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; + var secondary = _fixture.HostedCluster.Silos.Single( + silo => !silo.SiloAddress.Equals(primary.SiloAddress)); + var grainFactory = GetPrimarySiloGrainFactory(primary); + var grain = grainFactory.GetGrain(Interlocked.Increment(ref _nextGrainKey)); + var grainReference = Assert.IsAssignableFrom(grain); + const string label = "remote-fast-path"; + + RequestContext.Set(IPlacementDirector.PlacementHintKey, secondary.SiloAddress); + try + { + await grain.SetLabel(label); + Assert.Equal(label, await grain.GetLabel()); + } + finally + { + RequestContext.Remove(IPlacementDirector.PlacementHintKey); + } + + Assert.Null(grainReference.DirectoryCacheEntry); + } + + [Fact] + public async Task CompatibleInterfaceCast_TransfersSameEntryHandle() + { + var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; + var grainFactory = GetPrimarySiloGrainFactory(primary); + var writer = grainFactory.GetGrain(Interlocked.Increment(ref _nextGrainKey)); + var writerReference = Assert.IsAssignableFrom(writer); + const int value = 1729; + + RequestContext.Set(IPlacementDirector.PlacementHintKey, primary.SiloAddress); + try + { + await writer.SetValue(-1); + await writer.SetValue(value); + } + finally + { + RequestContext.Remove(IPlacementDirector.PlacementHintKey); + } + + var writerEntry = Assert.IsType(writerReference.DirectoryCacheEntry); + Assert.True(writerEntry.IsValid); + + var reader = writer.AsReference(); + var readerReference = Assert.IsAssignableFrom(reader); + + Assert.NotSame(writerReference, readerReference); + Assert.Same(writerEntry, readerReference.DirectoryCacheEntry); + Assert.Equal(writerReference.GrainId, readerReference.GrainId); + Assert.Equal(value, await reader.GetValue()); + Assert.Same(writerEntry, readerReference.DirectoryCacheEntry); + } + + [Fact] + public async Task ExternalClientCalls_DoNotAttachPerGrainDirectoryEntries() + { + var grains = Enumerable.Range(0, 32) + .Select(_ => _fixture.Client.GetGrain(Interlocked.Increment(ref _nextGrainKey))) + .ToArray(); + + foreach (var grain in grains) + { + await grain.SetLabel("external-client"); + Assert.Null(Assert.IsAssignableFrom(grain).DirectoryCacheEntry); + } + } + + [Fact] + public async Task StatelessWorkerCalls_DoNotAttachDirectoryEntries() + { + var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; + var grainFactory = GetPrimarySiloGrainFactory(primary); + var grain = grainFactory.GetGrain(Guid.NewGuid()); + + await grain.Nop(); + + Assert.Null(Assert.IsAssignableFrom(grain).DirectoryCacheEntry); + } + + [Fact] + public async Task CacheInvalidationHeader_BypassesFastPathWithoutDiscardingLiveHandle() + { + var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; + var grainFactory = GetPrimarySiloGrainFactory(primary); + var grain = grainFactory.GetGrain(Interlocked.Increment(ref _nextGrainKey)); + var grainReference = Assert.IsAssignableFrom(grain); + RequestContext.Set(IPlacementDirector.PlacementHintKey, primary.SiloAddress); + try + { + await grain.SetLabel("cache-header"); + Assert.Equal("cache-header", await grain.GetLabel()); + } + finally + { + RequestContext.Remove(IPlacementDirector.PlacementHintKey); + } + + var retainedEntry = Assert.IsType(grainReference.DirectoryCacheEntry); + var unrelatedAddress = new GrainAddress + { + GrainId = GrainId.Create("unrelated", Guid.NewGuid().ToString()), + SiloAddress = primary.SiloAddress, + }; + var message = new Message + { + Direction = Message.Directions.Request, + TargetGrain = grainReference.GrainId, + CacheInvalidationHeader = [new GrainAddressCacheUpdate(unrelatedAddress, validAddress: null)], + }; + + var messageCenter = primary.ServiceProvider.GetRequiredService(); + Assert.False(messageCenter.TryGetDirectoryCacheEntry(grainReference, message, out var selectedEntry)); + + Assert.Null(selectedEntry); + Assert.True(retainedEntry.IsValid); + Assert.Same(retainedEntry, grainReference.DirectoryCacheEntry); + } + + private static IGrainFactory GetPrimarySiloGrainFactory(InProcessSiloHandle primary) + { + var runtimeClient = primary.ServiceProvider.GetRequiredService(); + var grainFactory = primary.ServiceProvider.GetRequiredService(); + Assert.Same(runtimeClient.ConcreteGrainFactory, grainFactory); + return grainFactory; + } + + public sealed class Fixture : BaseTestClusterFixture + { + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.Options.InitialSilosCount = 2; + builder.AddSiloBuilderConfigurator(); + } + } + + private sealed class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder siloBuilder) + { + siloBuilder.AddMemoryGrainStorage("MemoryStore"); + } + } +} From 59514a1b14af53287d166181cd4d1995c89c9b3e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 14:51:46 -0700 Subject: [PATCH 02/21] perf(runtime): streamline cached local dispatch --- .../Messaging/MessageCenter.cs | 72 ++++++++++++------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/src/Orleans.Runtime/Messaging/MessageCenter.cs b/src/Orleans.Runtime/Messaging/MessageCenter.cs index 62660ea9157..811d15627fc 100644 --- a/src/Orleans.Runtime/Messaging/MessageCenter.cs +++ b/src/Orleans.Runtime/Messaging/MessageCenter.cs @@ -192,6 +192,17 @@ private void SendMessage( return; } + if (directoryCacheEntry is not null) + { + Debug.Assert(msg.TargetSilo?.Matches(_siloAddress) == true); + Debug.Assert(!msg.TargetGrain.IsClient()); + Debug.Assert(msg.Direction is Message.Directions.Request or Message.Directions.OneWay); + MessagingEvents.EmitSent(msg); + _messagingInstruments.LocalMessagesSentCounterAggregator.Add(1); + ReceiveMessage(msg, directoryCacheEntry); + return; + } + // First check to see if it's really destined for a proxied client, instead of a local grain. if (TryDeliverToProxy(msg)) { @@ -208,7 +219,7 @@ private void SendMessage( MessagingEvents.EmitSent(msg); - if (directoryCacheEntry is not null || targetSilo.Matches(_siloAddress)) + if (targetSilo.Matches(_siloAddress)) { LogTraceMessageLoopedBack(log, msg); @@ -603,7 +614,11 @@ private void ReceiveMessage(Message msg, GrainDirectoryCacheEntry? directoryCach try { this.messagingTrace.OnIncomingMessageAgentReceiveMessage(msg); - if (TryDeliverToProxy(msg)) + if (directoryCacheEntry is not null) + { + ReceiveApplicationMessage(msg, directoryCacheEntry); + } + else if (TryDeliverToProxy(msg)) { return; } @@ -613,30 +628,7 @@ private void ReceiveMessage(Message msg, GrainDirectoryCacheEntry? directoryCach } else { - IGrainContext? targetActivation = GetCachedActivation(directoryCacheEntry); - if (targetActivation is null) - { - targetActivation = catalog.GetOrCreateActivation( - msg.TargetGrain, - msg.RequestContextData, - rehydrationContext: null); - - if (targetActivation is ActivationData { IsValid: true } activation - && directoryCacheEntry is { } entry - && entry.Address.Matches(activation.Address)) - { - entry.TrySetMessageTarget(activation); - } - } - - if (targetActivation is null) - { - ProcessMessageToNonExistentActivation(msg); - return; - } - - targetActivation.ReceiveMessage(msg); - _messageObserver?.Invoke(msg); + ReceiveApplicationMessage(msg, null); } } catch (Exception ex) @@ -653,6 +645,34 @@ void HandleReceiveFailure(Message msg, Exception ex) } } + private void ReceiveApplicationMessage(Message message, GrainDirectoryCacheEntry? directoryCacheEntry) + { + IGrainContext? targetActivation = GetCachedActivation(directoryCacheEntry); + if (targetActivation is null) + { + targetActivation = catalog.GetOrCreateActivation( + message.TargetGrain, + message.RequestContextData, + rehydrationContext: null); + + if (targetActivation is ActivationData { IsValid: true } activation + && directoryCacheEntry is { } entry + && entry.Address.Matches(activation.Address)) + { + entry.TrySetMessageTarget(activation); + } + } + + if (targetActivation is null) + { + ProcessMessageToNonExistentActivation(message); + return; + } + + targetActivation.ReceiveMessage(message); + _messageObserver?.Invoke(message); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static ActivationData? GetCachedActivation(GrainDirectoryCacheEntry? directoryCacheEntry) { From a0186301ad4e688a00a740c5f2905623e00a91a0 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 15:45:59 -0700 Subject: [PATCH 03/21] perf(benchmarks): add deterministic ping matrix --- .../Ping/AdaptiveConcurrencyLoadGenerator.cs | 26 +++++++ test/Benchmarks/Ping/AdaptivePingBenchmark.cs | 72 +++++++++++++++++++ test/Benchmarks/Program.cs | 4 ++ 3 files changed, 102 insertions(+) diff --git a/test/Benchmarks/Ping/AdaptiveConcurrencyLoadGenerator.cs b/test/Benchmarks/Ping/AdaptiveConcurrencyLoadGenerator.cs index c072a867690..bc19095f560 100644 --- a/test/Benchmarks/Ping/AdaptiveConcurrencyLoadGenerator.cs +++ b/test/Benchmarks/Ping/AdaptiveConcurrencyLoadGenerator.cs @@ -156,8 +156,34 @@ public async Task RunForeverAsync(CancellationToken cancellationToken = default) break; } } + } + public async Task RunFixedConcurrencyAsync(int repetitions, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfLessThan(repetitions, 1); + _cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + + try + { + await RunPhaseAsync(_warmupDuration, isWarmup: true); + GC.Collect(); + + var results = new double[repetitions]; + for (var i = 0; i < results.Length; i++) + { + results[i] = (await RunPhaseAsync(_measurementInterval, isWarmup: false)).Throughput; + } + + return results; + } + finally + { + await _cts.CancelAsync(); + _cts.Dispose(); + } + } + private async Task RunPhaseAsync(TimeSpan duration, bool isWarmup) { _completedBlocks = Channel.CreateUnbounded( diff --git a/test/Benchmarks/Ping/AdaptivePingBenchmark.cs b/test/Benchmarks/Ping/AdaptivePingBenchmark.cs index 0ad87561b74..90116853140 100644 --- a/test/Benchmarks/Ping/AdaptivePingBenchmark.cs +++ b/test/Benchmarks/Ping/AdaptivePingBenchmark.cs @@ -269,4 +269,76 @@ public static async Task RunAllScenariosAsync(int maxStableRounds = DefaultMaxSt Console.WriteLine(); } + + public static async Task RunDeterministicMatrixAsync( + int repetitions = 3, + TimeSpan? warmupDuration = null, + TimeSpan? measurementInterval = null) + { + int[] concurrencyLevels = [100, 250, 500]; + var scenarios = new (BenchmarkMode Mode, int NumSilos)[] + { + (BenchmarkMode.HostedClient, 1), + (BenchmarkMode.ExternalClient, 1), + (BenchmarkMode.ExternalClient, 2), + (BenchmarkMode.SiloToSilo, 2), + }; + var results = new List(); + + foreach (var (mode, numSilos) in scenarios) + { + using var benchmark = new AdaptivePingBenchmark(mode, numSilos); + try + { + foreach (var concurrency in concurrencyLevels) + { + var loadGenerator = new AdaptiveConcurrencyLoadGenerator( + issueRequest: grain => grain.Run(), + getStateForWorker: workerId => benchmark.GetGrainFactory().GetGrain(workerId), + requestsPerBlock: DefaultRequestsPerBlock, + warmupDuration: warmupDuration ?? TimeSpan.FromSeconds(5), + measurementInterval: measurementInterval ?? TimeSpan.FromSeconds(3), + minConcurrency: concurrency, + maxConcurrency: concurrency, + initialConcurrency: concurrency, + maxStableRounds: 1, + initialStepSize: 1, + sampleInterval: DefaultSampleInterval, + minimumRelativeImprovement: 0); + var samples = await loadGenerator.RunFixedConcurrencyAsync(repetitions); + Array.Sort(samples); + var median = samples.Length % 2 == 0 + ? (samples[(samples.Length / 2) - 1] + samples[samples.Length / 2]) / 2 + : samples[samples.Length / 2]; + results.Add(new(benchmark.Description, concurrency, median, samples[^1])); + Console.WriteLine($"{benchmark.Description}, concurrency {concurrency}: median {median:N0}/s, max {samples[^1]:N0}/s"); + } + } + finally + { + await benchmark.ShutdownAsync(); + } + + GC.Collect(); + await Task.Delay(1000); + } + + Console.WriteLine(); + Console.WriteLine("## Deterministic Ping Benchmark Results"); + Console.WriteLine(); + Console.WriteLine("| Scenario | Concurrency | Median Throughput | Maximum Throughput |"); + Console.WriteLine("|----------|------------:|------------------:|-------------------:|"); + foreach (var result in results) + { + Console.WriteLine($"| {result.Description} | {result.Concurrency} | {result.MedianThroughput:N0}/s | {result.MaximumThroughput:N0}/s |"); + } + + Console.WriteLine(); + } + + private readonly record struct DeterministicResult( + string Description, + int Concurrency, + double MedianThroughput, + double MaximumThroughput); } diff --git a/test/Benchmarks/Program.cs b/test/Benchmarks/Program.cs index 1536a92e789..457796e21c1 100644 --- a/test/Benchmarks/Program.cs +++ b/test/Benchmarks/Program.cs @@ -231,6 +231,10 @@ internal class Program { AdaptivePingBenchmark.RunAllScenariosAsync().GetAwaiter().GetResult(); }, + ["DeterministicPing_All"] = _ => + { + AdaptivePingBenchmark.RunDeterministicMatrixAsync().GetAwaiter().GetResult(); + }, ["ConcurrentPing_OneSilo_Forever"] = _ => { new PingBenchmark(numSilos: 1, startClient: true).PingConcurrentForever().GetAwaiter().GetResult(); From d4923a0c2a89f7a30f243d80da62152cec86424e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 17:50:18 -0700 Subject: [PATCH 04/21] perf(benchmarks): add focused routing measurements --- .../Caching/GrainDirectoryCacheBenchmark.cs | 82 ++++++++++++++++ test/Benchmarks/Ping/AdaptivePingBenchmark.cs | 93 ++++++++++++------- test/Benchmarks/Program.cs | 16 ++++ 3 files changed, 159 insertions(+), 32 deletions(-) create mode 100644 test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs diff --git a/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs b/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs new file mode 100644 index 00000000000..1a948cdeca4 --- /dev/null +++ b/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs @@ -0,0 +1,82 @@ +using BenchmarkDotNet.Attributes; +using Orleans.Caching; +using Orleans.Runtime; +using Orleans.Runtime.GrainDirectory; + +namespace Benchmarks.Caching; + +[MemoryDiagnoser] +[BenchmarkCategory("Caching")] +public class GrainDirectoryCacheBenchmark +{ + private ConcurrentLruCache _tupleCache = null!; + private LruGrainDirectoryCache _entryCache = null!; + private IGrainDirectoryCacheEntrySource _entrySource = null!; + private GrainId _target; + + [GlobalSetup] + public void Setup() + { + _tupleCache = new( + capacity: 1_024, + timeToLive: TimeSpan.FromMinutes(10), + timeProvider: TimeProvider.System); + _entryCache = new( + maxCacheSize: 1_024, + maxCacheTTL: TimeSpan.FromMinutes(10), + timeProvider: TimeProvider.System); + _entrySource = _entryCache; + + for (var i = 0; i < 1_024; i++) + { + var grainId = GrainId.Create("benchmark", i.ToString()); + var address = new GrainAddress + { + GrainId = grainId, + ActivationId = ActivationId.NewId(), + SiloAddress = SiloAddress.FromParsableString("127.0.0.1:11111@1"), + }; + _tupleCache.AddOrUpdate(grainId, (address, i)); + _entryCache.AddOrUpdate(address, i); + if (i == 512) + { + _target = grainId; + } + } + } + + [Benchmark(Baseline = true)] + public GrainAddress TupleValue() + { + _tupleCache.TryGet(_target, out var entry); + return entry.Address; + } + + [Benchmark] + public GrainAddress SharedEntryRaw() + { + _entryCache.TryGet(_target, out var entry); + return entry.ActivationAddress; + } + + [Benchmark] + public GrainAddress SharedEntryValidated() + { + _entryCache.LookUp(_target, out var address, out _); + return address!; + } + + [Benchmark] + public GrainAddress SharedHandle() + { + _entrySource.TryGetEntry(_target, out var entry); + return entry!.Address; + } + + [GlobalCleanup] + public async Task Cleanup() + { + await _tupleCache.DisposeAsync(); + await _entryCache.DisposeAsync(); + } +} diff --git a/test/Benchmarks/Ping/AdaptivePingBenchmark.cs b/test/Benchmarks/Ping/AdaptivePingBenchmark.cs index 90116853140..9b2fada0c47 100644 --- a/test/Benchmarks/Ping/AdaptivePingBenchmark.cs +++ b/test/Benchmarks/Ping/AdaptivePingBenchmark.cs @@ -275,7 +275,6 @@ public static async Task RunDeterministicMatrixAsync( TimeSpan? warmupDuration = null, TimeSpan? measurementInterval = null) { - int[] concurrencyLevels = [100, 250, 500]; var scenarios = new (BenchmarkMode Mode, int NumSilos)[] { (BenchmarkMode.HostedClient, 1), @@ -287,42 +286,72 @@ public static async Task RunDeterministicMatrixAsync( foreach (var (mode, numSilos) in scenarios) { - using var benchmark = new AdaptivePingBenchmark(mode, numSilos); - try - { - foreach (var concurrency in concurrencyLevels) - { - var loadGenerator = new AdaptiveConcurrencyLoadGenerator( - issueRequest: grain => grain.Run(), - getStateForWorker: workerId => benchmark.GetGrainFactory().GetGrain(workerId), - requestsPerBlock: DefaultRequestsPerBlock, - warmupDuration: warmupDuration ?? TimeSpan.FromSeconds(5), - measurementInterval: measurementInterval ?? TimeSpan.FromSeconds(3), - minConcurrency: concurrency, - maxConcurrency: concurrency, - initialConcurrency: concurrency, - maxStableRounds: 1, - initialStepSize: 1, - sampleInterval: DefaultSampleInterval, - minimumRelativeImprovement: 0); - var samples = await loadGenerator.RunFixedConcurrencyAsync(repetitions); - Array.Sort(samples); - var median = samples.Length % 2 == 0 - ? (samples[(samples.Length / 2) - 1] + samples[samples.Length / 2]) / 2 - : samples[samples.Length / 2]; - results.Add(new(benchmark.Description, concurrency, median, samples[^1])); - Console.WriteLine($"{benchmark.Description}, concurrency {concurrency}: median {median:N0}/s, max {samples[^1]:N0}/s"); - } - } - finally - { - await benchmark.ShutdownAsync(); - } + results.AddRange(await MeasureDeterministicScenarioAsync(mode, numSilos, repetitions, warmupDuration, measurementInterval)); GC.Collect(); await Task.Delay(1000); } + PrintDeterministicResults(results); + } + + public static async Task RunDeterministicScenarioAsync( + BenchmarkMode mode, + int numSilos, + int repetitions = 3, + TimeSpan? warmupDuration = null, + TimeSpan? measurementInterval = null) + { + var results = await MeasureDeterministicScenarioAsync(mode, numSilos, repetitions, warmupDuration, measurementInterval); + PrintDeterministicResults(results); + } + + private static async Task> MeasureDeterministicScenarioAsync( + BenchmarkMode mode, + int numSilos, + int repetitions, + TimeSpan? warmupDuration, + TimeSpan? measurementInterval) + { + int[] concurrencyLevels = [100, 250, 500]; + var results = new List(concurrencyLevels.Length); + using var benchmark = new AdaptivePingBenchmark(mode, numSilos); + try + { + foreach (var concurrency in concurrencyLevels) + { + var loadGenerator = new AdaptiveConcurrencyLoadGenerator( + issueRequest: grain => grain.Run(), + getStateForWorker: workerId => benchmark.GetGrainFactory().GetGrain(workerId), + requestsPerBlock: DefaultRequestsPerBlock, + warmupDuration: warmupDuration ?? TimeSpan.FromSeconds(5), + measurementInterval: measurementInterval ?? TimeSpan.FromSeconds(3), + minConcurrency: concurrency, + maxConcurrency: concurrency, + initialConcurrency: concurrency, + maxStableRounds: 1, + initialStepSize: 1, + sampleInterval: DefaultSampleInterval, + minimumRelativeImprovement: 0); + var samples = await loadGenerator.RunFixedConcurrencyAsync(repetitions); + Array.Sort(samples); + var median = samples.Length % 2 == 0 + ? (samples[(samples.Length / 2) - 1] + samples[samples.Length / 2]) / 2 + : samples[samples.Length / 2]; + results.Add(new(benchmark.Description, concurrency, median, samples[^1])); + Console.WriteLine($"{benchmark.Description}, concurrency {concurrency}: median {median:N0}/s, max {samples[^1]:N0}/s"); + } + } + finally + { + await benchmark.ShutdownAsync(); + } + + return results; + } + + private static void PrintDeterministicResults(List results) + { Console.WriteLine(); Console.WriteLine("## Deterministic Ping Benchmark Results"); Console.WriteLine(); diff --git a/test/Benchmarks/Program.cs b/test/Benchmarks/Program.cs index 457796e21c1..523fe9f3ee2 100644 --- a/test/Benchmarks/Program.cs +++ b/test/Benchmarks/Program.cs @@ -235,6 +235,22 @@ internal class Program { AdaptivePingBenchmark.RunDeterministicMatrixAsync().GetAwaiter().GetResult(); }, + ["DeterministicPing_HostedClient"] = _ => + { + AdaptivePingBenchmark.RunDeterministicScenarioAsync(AdaptivePingBenchmark.BenchmarkMode.HostedClient, numSilos: 1).GetAwaiter().GetResult(); + }, + ["DeterministicPing_ClientToOneSilo"] = _ => + { + AdaptivePingBenchmark.RunDeterministicScenarioAsync(AdaptivePingBenchmark.BenchmarkMode.ExternalClient, numSilos: 1).GetAwaiter().GetResult(); + }, + ["DeterministicPing_ClientToTwoSilos"] = _ => + { + AdaptivePingBenchmark.RunDeterministicScenarioAsync(AdaptivePingBenchmark.BenchmarkMode.ExternalClient, numSilos: 2).GetAwaiter().GetResult(); + }, + ["DeterministicPing_SiloToSilo"] = _ => + { + AdaptivePingBenchmark.RunDeterministicScenarioAsync(AdaptivePingBenchmark.BenchmarkMode.SiloToSilo, numSilos: 2).GetAwaiter().GetResult(); + }, ["ConcurrentPing_OneSilo_Forever"] = _ => { new PingBenchmark(numSilos: 1, startClient: true).PingConcurrentForever().GetAwaiter().GetResult(); From 19d236d06f69eedc386b6c210739dc1248140732 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 18:15:24 -0700 Subject: [PATCH 05/21] perf(runtime): embed message target in directory cache node --- .testagent/plan.md | 2 +- .testagent/research.md | 2 +- .testagent/status.md | 6 +-- .../Caching/ConcurrentLruCache.cs | 32 ++++++++++++--- .../GrainDirectoryCacheEntry.cs | 41 ++++++++++++++++--- .../GrainDirectory/LruGrainDirectoryCache.cs | 29 ++++++++----- .../Messaging/MessageCenter.cs | 21 ++++++---- .../GrainDirectoryCacheFactoryTests.cs | 27 ++++++------ .../SharedEntryMessageTargetFastPathTests.cs | 8 +++- 9 files changed, 119 insertions(+), 49 deletions(-) diff --git a/.testagent/plan.md b/.testagent/plan.md index 3fca47f12ee..cbf1d14c8c5 100644 --- a/.testagent/plan.md +++ b/.testagent/plan.md @@ -1,6 +1,6 @@ # Test Implementation Plan -> Final scope update: the implementation was narrowed to local `IGrainContext` targets. Remote connection and response-route cases below were superseded by explicit remote/client/stateless fallback tests. +> Final scope update: handler caching is narrowed to local `IGrainContext` targets. Remote references cache only the shared directory address and continue through the normal connection path; connection and response-route cases below remain deferred. ## Overview diff --git a/.testagent/research.md b/.testagent/research.md index 9927eb16884..34655c4b1a4 100644 --- a/.testagent/research.md +++ b/.testagent/research.md @@ -1,6 +1,6 @@ # Test Generation Research -> Final scope update: benchmarking showed material benefit only for in-process activation dispatch. The implemented cache target is therefore limited to `IGrainContext`; remote connection and response-route caching are deferred to separate experiments. +> Final scope update: cached handlers are limited to local `IGrainContext` instances. Remote grain references retain the same shared directory entry for address reuse, while remote connection and response-route caching remain separate experiments. ## Project Overview - **Path**: `C:\dev\copilot-worktrees\orleans\rb-scaling-guacamole` diff --git a/.testagent/status.md b/.testagent/status.md index e7040d2c194..8367cd19f63 100644 --- a/.testagent/status.md +++ b/.testagent/status.md @@ -12,8 +12,8 @@ The shared directory-entry message target cache is covered by: - 28 focused tests pass on `net10.0`. - No generated test is assertion-free or relies only on presence/truthiness checks. - Assertions cover equality, identity, type, null, negative behavior, collection state, lifecycle state, and side effects. -- Cache lifetime tests verify replacement, conditional removal, clear, TTL expiration, eviction, tombstoning, target release, one-shot binding, and identity-checked clearing. -- Integration tests verify local target binding, invalidation and recapture, compatible-interface casts, and explicit remote, external-client, and stateless-worker exclusion. +- Cache lifetime tests verify in-place replacement, conditional removal, clear, TTL expiration, eviction, tombstoning, target release, one-shot binding, and identity-checked clearing. +- Integration tests verify local target binding, remote address-only caching, invalidation and recapture, compatible-interface casts, and explicit external-client and stateless-worker exclusion. ## Pseudo-Mutation Review @@ -27,4 +27,4 @@ Every mutation was reverted immediately. The focused suites were rerun green aft ## Remaining Risk -The tests intentionally allow the same stale local-dispatch race as the existing activation-directory lookup path: invalidation can race after a valid target is read. Activation state and address validation preserve the existing recovery semantics. Remote connections and response routing remain on the established path. +The tests intentionally allow the same stale routing race as the existing directory and activation lookups: invalidation can race after a valid entry is read. Entry invalidation, activation state checks, membership validation, and existing forwarding semantics preserve recovery. Remote connections and response routing remain on the established path. diff --git a/src/Orleans.Core/Caching/ConcurrentLruCache.cs b/src/Orleans.Core/Caching/ConcurrentLruCache.cs index 5f1aef163cb..9f5fa8a80a4 100644 --- a/src/Orleans.Core/Caching/ConcurrentLruCache.cs +++ b/src/Orleans.Core/Caching/ConcurrentLruCache.cs @@ -64,6 +64,14 @@ public ConcurrentLruCache(int capacity) : this(capacity, comparer: null) { } + protected virtual void OnItemUpdating(LruItem item) + { + } + + protected virtual void OnItemRemoved(LruItem item) + { + } + /// /// Initializes a new instance of the ConcurrentLruCore class with the specified capacity and expire-after-access time to live. /// @@ -161,15 +169,25 @@ public V Get(K key) /// public bool TryGet(K key, [MaybeNullWhen(false)] out V value) { - if (_dictionary.TryGetValue(key, out var item)) + if (TryGetItem(key, out var item)) { value = item.Value; + return true; + } + + value = default; + return false; + } + + protected bool TryGetItem(K key, [NotNullWhen(true)] out LruItem? item) + { + if (_dictionary.TryGetValue(key, out item)) + { Touch(item); _telemetryPolicy.IncrementHit(); return true; } - value = default; _telemetryPolicy.IncrementMiss(); return false; } @@ -326,6 +344,7 @@ private void OnRemove(LruItem item, ItemRemovedReason reason) // from the queue. item.WasAccessed = false; item.WasRemoved = true; + OnItemRemoved(item); if (reason == ItemRemovedReason.Evicted) { @@ -351,6 +370,7 @@ public bool TryUpdate(K key, V value) { var oldValue = existing.Value; + OnItemUpdating(existing); existing.Value = value; UpdateTimestamp(existing); @@ -853,7 +873,7 @@ private static ItemDestination RouteCold(LruItem item) /// The value. // NOTE: Internal for testing [DebuggerDisplay("[{Key}] = {Value}")] - internal sealed class LruItem(K key, V value, long timestamp = 0) + internal class LruItem(K key, V value, long timestamp = 0) { private V _data = value; @@ -1057,8 +1077,10 @@ private async Task RunExpirationLoop() } } - private LruItem CreateItem(K key, V value) => - new(key, value, _expiresAfterAccess ? _timeProvider.GetTimestamp() : 0); + protected virtual LruItem CreateItem(K key, V value) => + new(key, value, GetCurrentTimestamp()); + + protected long GetCurrentTimestamp() => _expiresAfterAccess ? _timeProvider.GetTimestamp() : 0; [MethodImpl(MethodImplOptions.AggressiveInlining)] private void Touch(LruItem item) diff --git a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs index 964daf09e8d..13c6e6b2d50 100644 --- a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs +++ b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs @@ -1,21 +1,31 @@ using System.Threading; +using Orleans.Caching; namespace Orleans.Runtime.GrainDirectory; -internal sealed class GrainDirectoryCacheEntry : IDisposable +internal sealed class GrainDirectoryCacheEntry + : ConcurrentLruCache.LruItem, + IDisposable { private static readonly object Invalidated = new(); private object? _messageTarget; public GrainDirectoryCacheEntry(GrainAddress address, int version) + : this(address.GrainId, (address, version), timestamp: 0) { - Address = address; - Version = version; } - public GrainAddress Address { get; } + public GrainDirectoryCacheEntry( + GrainId grainId, + (GrainAddress ActivationAddress, int Version) value, + long timestamp) + : base(grainId, value, timestamp) + { + } + + public GrainAddress Address => Value.ActivationAddress; - public int Version { get; } + public int Version => Value.Version; public bool IsValid => !ReferenceEquals(Volatile.Read(ref _messageTarget), Invalidated); @@ -51,5 +61,24 @@ public void ClearMessageTarget(IGrainContext messageTarget) Interlocked.CompareExchange(ref _messageTarget, null, messageTarget); } - public void Dispose() => Interlocked.Exchange(ref _messageTarget, Invalidated); + public void Invalidate() => Interlocked.Exchange(ref _messageTarget, Invalidated); + + public void Dispose() => Invalidate(); + + public void ClearMessageTarget() + { + while (true) + { + var current = Volatile.Read(ref _messageTarget); + if (current is null || ReferenceEquals(current, Invalidated)) + { + return; + } + + if (ReferenceEquals(Interlocked.CompareExchange(ref _messageTarget, null, current), current)) + { + return; + } + } + } } diff --git a/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs b/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs index 697f245a3f3..ead830e7aa9 100644 --- a/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs +++ b/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs @@ -6,9 +6,9 @@ namespace Orleans.Runtime.GrainDirectory; -internal sealed class LruGrainDirectoryCache : ConcurrentLruCache, IGrainDirectoryCache, IGrainDirectoryCacheEntrySource, IAsyncDisposable +internal sealed class LruGrainDirectoryCache : ConcurrentLruCache, IGrainDirectoryCache, IGrainDirectoryCacheEntrySource, IAsyncDisposable { - private static readonly Func ActivationAddressesMatch = (value, state) => GrainAddress.MatchesGrainIdAndSilo(state, value.Address); + private static readonly Func<(GrainAddress Address, int Version), GrainAddress, bool> ActivationAddressesMatch = (value, state) => GrainAddress.MatchesGrainIdAndSilo(state, value.Address); private readonly IDisposable _cacheSizeRegistration; public LruGrainDirectoryCache( @@ -27,7 +27,7 @@ public LruGrainDirectoryCache( : directoryInstruments.RegisterCacheSizeObserve(() => Count); } - public void AddOrUpdate(GrainAddress activationAddress, int version) => AddOrUpdate(activationAddress.GrainId, new GrainDirectoryCacheEntry(activationAddress, version)); + public void AddOrUpdate(GrainAddress activationAddress, int version) => AddOrUpdate(activationAddress.GrainId, (activationAddress, version)); public bool Remove(GrainId key) => TryRemove(key); @@ -35,10 +35,10 @@ public LruGrainDirectoryCache( public bool LookUp(GrainId key, [NotNullWhen(true)] out GrainAddress? result, out int version) { - if (TryGetEntry(key, out var entry)) + if (TryGet(key, out var entry)) { version = entry.Version; - result = entry.Address; + result = entry.ActivationAddress; return true; } @@ -53,18 +53,18 @@ public bool LookUp(GrainId key, [NotNullWhen(true)] out GrainAddress? result, ou { foreach (var entry in this) { - if (entry.Value.IsValid) - { - yield return (entry.Value.Address, entry.Value.Version); - } + yield return (entry.Value.ActivationAddress, entry.Value.Version); } } } public bool TryGetEntry(GrainId key, [NotNullWhen(true)] out GrainDirectoryCacheEntry? entry) { - if (TryGet(key, out entry) && entry.IsValid) + if (TryGetItem(key, out var item) + && item is GrainDirectoryCacheEntry result + && result.IsValid) { + entry = result; return true; } @@ -72,6 +72,15 @@ public bool TryGetEntry(GrainId key, [NotNullWhen(true)] out GrainDirectoryCache return false; } + protected override LruItem CreateItem(GrainId key, (GrainAddress ActivationAddress, int Version) value) + => new GrainDirectoryCacheEntry(key, value, GetCurrentTimestamp()); + + protected override void OnItemUpdating(LruItem item) + => ((GrainDirectoryCacheEntry)item).ClearMessageTarget(); + + protected override void OnItemRemoved(LruItem item) + => ((GrainDirectoryCacheEntry)item).Invalidate(); + public new async ValueTask DisposeAsync() { _cacheSizeRegistration.Dispose(); diff --git a/src/Orleans.Runtime/Messaging/MessageCenter.cs b/src/Orleans.Runtime/Messaging/MessageCenter.cs index 811d15627fc..9d2102c4ade 100644 --- a/src/Orleans.Runtime/Messaging/MessageCenter.cs +++ b/src/Orleans.Runtime/Messaging/MessageCenter.cs @@ -505,8 +505,9 @@ internal Task AddressAndSendMessage(Message message, GrainReference? target = nu { if (TryGetDirectoryCacheEntry(target, message, out var directoryCacheEntry)) { - message.TargetSilo = _siloAddress; - SendMessage(message, directoryCacheEntry); + var targetSilo = directoryCacheEntry.Address.SiloAddress!; + message.TargetSilo = targetSilo; + SendMessage(message, targetSilo.Matches(_siloAddress) ? directoryCacheEntry : null); return Task.CompletedTask; } @@ -562,12 +563,17 @@ internal bool TryGetDirectoryCacheEntry( if (message.CacheInvalidationHeader is null && candidate.IsValid) { Debug.Assert(candidate.Address.GrainId.Equals(message.TargetGrain)); - Debug.Assert(candidate.Address.SiloAddress?.Matches(_siloAddress) == true); - entry = candidate; - return true; + if (candidate.Address.SiloAddress is { } targetSilo + && (targetSilo.Matches(_siloAddress) || !siloStatusOracle.IsDeadSilo(targetSilo))) + { + entry = candidate; + return true; + } } - if (!candidate.IsValid) + if (!candidate.IsValid + || candidate.Address.SiloAddress is not { } candidateSilo + || siloStatusOracle.IsDeadSilo(candidateSilo)) { target.ClearDirectoryCacheEntry(candidate); } @@ -581,7 +587,6 @@ internal bool TryGetDirectoryCacheEntry( if (target is null || message.CacheInvalidationHeader is not null || message.TargetSilo is not { } targetSilo - || !targetSilo.Matches(_siloAddress) || !placementService.IsUsingGrainDirectory(message.TargetGrain) || !_grainLocator.TryGetCacheEntry(message.TargetGrain, targetSilo, out var entry)) { @@ -589,7 +594,7 @@ internal bool TryGetDirectoryCacheEntry( } target.DirectoryCacheEntry = entry; - return entry; + return targetSilo.Matches(_siloAddress) ? entry : null; } internal void SendResponse(Message request, Response response) diff --git a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs index 64910daddb7..c0b25593726 100644 --- a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs +++ b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs @@ -259,7 +259,7 @@ private sealed class DisposableGrainDirectoryCache : TestGrainDirectoryCache, ID } [Fact] - public async Task CreateGrainDirectoryCache_AddOrUpdateInvalidatesReplacedRouteHandle() + public async Task CreateGrainDirectoryCache_AddOrUpdateUpdatesSharedRouteHandle() { var (cache, entrySource) = CreateEntryCache(); var disposableCache = Assert.IsAssignableFrom(cache); @@ -277,10 +277,13 @@ public async Task CreateGrainDirectoryCache_AddOrUpdateInvalidatesReplacedRouteH cache.AddOrUpdate(replacementAddress, version: 2); var replacementEntry = GetEntry(entrySource, grainId); - Assert.True(replacementEntry.TrySetMessageTarget(replacementTarget)); - Assert.NotSame(originalEntry, replacementEntry); - AssertInvalidEntry(originalEntry); + Assert.Same(originalEntry, replacementEntry); + Assert.True(replacementEntry.IsValid); + Assert.Equal(replacementAddress, replacementEntry.Address); + Assert.Equal(2, replacementEntry.Version); + Assert.False(replacementEntry.TryGetMessageTarget(out _)); + Assert.True(replacementEntry.TrySetMessageTarget(replacementTarget)); AssertMessageTarget(replacementEntry, replacementTarget); Assert.True(cache.LookUp(grainId, out var result, out var version)); Assert.Equal(replacementAddress, result); @@ -451,7 +454,6 @@ public async Task CreateGrainDirectoryCache_EvictionInvalidatesRouteHandle() { var (cache, entrySource) = CreateEntryCache(cacheSize: 3); var disposableCache = Assert.IsAssignableFrom(cache); - var lruCache = Assert.IsType(cache); var addresses = new[] { CreateGrainAddress(CreateGrainId(), port: 11111), @@ -464,19 +466,18 @@ public async Task CreateGrainDirectoryCache_EvictionInvalidatesRouteHandle() try { - for (var i = 0; i < entries.Length; i++) + for (var i = 0; i < 3; i++) { - entries[i] = new GrainDirectoryCacheEntry(addresses[i], version: i + 1); + cache.AddOrUpdate(addresses[i], version: i + 1); + entries[i] = GetEntry(entrySource, addresses[i].GrainId); targets[i] = CreateMessageTarget(); Assert.True(entries[i].TrySetMessageTarget(targets[i])); } - for (var i = 0; i < 3; i++) - { - lruCache.AddOrUpdate(addresses[i].GrainId, entries[i]); - } - - lruCache.AddOrUpdate(addresses[3].GrainId, entries[3]); + cache.AddOrUpdate(addresses[3], version: 4); + entries[3] = GetEntry(entrySource, addresses[3].GrainId); + targets[3] = CreateMessageTarget(); + Assert.True(entries[3].TrySetMessageTarget(targets[3])); AssertInvalidEntry(entries[0]); Assert.False(entrySource.TryGetEntry(addresses[0].GrainId, out _)); diff --git a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs index a6007c3126d..c31433ad41e 100644 --- a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs +++ b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs @@ -98,7 +98,7 @@ public async Task InvalidateCache_DisposesRetainedEntry_AndNextCallCapturesDiffe } [Fact] - public async Task RemoteDirectoryGrain_DoesNotAttachLocalTargetEntry() + public async Task RemoteDirectoryGrain_CachesAddressWithoutBindingLocalTarget() { var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; var secondary = _fixture.HostedCluster.Silos.Single( @@ -119,7 +119,11 @@ public async Task RemoteDirectoryGrain_DoesNotAttachLocalTargetEntry() RequestContext.Remove(IPlacementDirector.PlacementHintKey); } - Assert.Null(grainReference.DirectoryCacheEntry); + var entry = Assert.IsType(grainReference.DirectoryCacheEntry); + Assert.True(entry.IsValid); + Assert.Equal(grainReference.GrainId, entry.Address.GrainId); + Assert.Equal(secondary.SiloAddress, entry.Address.SiloAddress); + Assert.False(entry.TryGetMessageTarget(out _)); } [Fact] From 2b88b54a6e9267d8ae1556daf48000beacd8e0bd Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 18:40:58 -0700 Subject: [PATCH 06/21] refactor(runtime): generalize message target cache handle --- .../Runtime/GrainReference.cs | 12 +++---- .../Runtime/GrainReferenceRuntime.cs | 2 +- .../Messaging/MessageCenter.cs | 6 ++-- .../SharedEntryMessageTargetFastPathTests.cs | 36 +++++++++---------- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/Orleans.Core.Abstractions/Runtime/GrainReference.cs b/src/Orleans.Core.Abstractions/Runtime/GrainReference.cs index 51c0e771397..479ba58daf7 100644 --- a/src/Orleans.Core.Abstractions/Runtime/GrainReference.cs +++ b/src/Orleans.Core.Abstractions/Runtime/GrainReference.cs @@ -268,16 +268,16 @@ public class GrainReference : IAddressable, IEquatable, ISpanFor private readonly IdSpan _key; [NonSerialized] - private object? _directoryCacheEntry; + private object? _messageTargetCache; - internal object? DirectoryCacheEntry + internal object? MessageTargetCache { - get => Volatile.Read(ref _directoryCacheEntry); - set => Volatile.Write(ref _directoryCacheEntry, value); + get => Volatile.Read(ref _messageTargetCache); + set => Volatile.Write(ref _messageTargetCache, value); } - internal void ClearDirectoryCacheEntry(object expected) - => Interlocked.CompareExchange(ref _directoryCacheEntry, null, expected); + internal void ClearMessageTargetCache(object expected) + => Interlocked.CompareExchange(ref _messageTargetCache, null, expected); /// /// Gets the grain reference functionality which is shared by all grain references of a given type. diff --git a/src/Orleans.Core/Runtime/GrainReferenceRuntime.cs b/src/Orleans.Core/Runtime/GrainReferenceRuntime.cs index dbdca40fbe9..56246b243aa 100644 --- a/src/Orleans.Core/Runtime/GrainReferenceRuntime.cs +++ b/src/Orleans.Core/Runtime/GrainReferenceRuntime.cs @@ -110,7 +110,7 @@ public object Cast(IAddressable grain, Type grainInterface) var result = this.referenceActivator.CreateReference(grainId, interfaceType); if (grain is GrainReference source) { - result.DirectoryCacheEntry = source.DirectoryCacheEntry; + result.MessageTargetCache = source.MessageTargetCache; } return result; diff --git a/src/Orleans.Runtime/Messaging/MessageCenter.cs b/src/Orleans.Runtime/Messaging/MessageCenter.cs index 9d2102c4ade..149d7a0aedd 100644 --- a/src/Orleans.Runtime/Messaging/MessageCenter.cs +++ b/src/Orleans.Runtime/Messaging/MessageCenter.cs @@ -554,7 +554,7 @@ internal bool TryGetDirectoryCacheEntry( Message message, [NotNullWhen(true)] out GrainDirectoryCacheEntry? entry) { - if (target?.DirectoryCacheEntry is not GrainDirectoryCacheEntry candidate) + if (target?.MessageTargetCache is not GrainDirectoryCacheEntry candidate) { entry = null; return false; @@ -575,7 +575,7 @@ internal bool TryGetDirectoryCacheEntry( || candidate.Address.SiloAddress is not { } candidateSilo || siloStatusOracle.IsDeadSilo(candidateSilo)) { - target.ClearDirectoryCacheEntry(candidate); + target.ClearMessageTargetCache(candidate); } entry = null; @@ -593,7 +593,7 @@ internal bool TryGetDirectoryCacheEntry( return null; } - target.DirectoryCacheEntry = entry; + target.MessageTargetCache = entry; return targetSilo.Matches(_siloAddress) ? entry : null; } diff --git a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs index c31433ad41e..055e91c5e46 100644 --- a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs +++ b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs @@ -45,7 +45,7 @@ public async Task LocalDirectoryGrain_AfterTwoCalls_BindsEntryToExactActivation( RequestContext.Remove(IPlacementDirector.PlacementHintKey); } - var entry = Assert.IsType(grainReference.DirectoryCacheEntry); + var entry = Assert.IsType(grainReference.MessageTargetCache); Assert.True(entry.IsValid); Assert.Equal(grainReference.GrainId, entry.Address.GrainId); Assert.Equal(primary.SiloAddress, entry.Address.SiloAddress); @@ -76,17 +76,17 @@ public async Task InvalidateCache_DisposesRetainedEntry_AndNextCallCapturesDiffe RequestContext.Remove(IPlacementDirector.PlacementHintKey); } - var retainedEntry = Assert.IsType(grainReference.DirectoryCacheEntry); + var retainedEntry = Assert.IsType(grainReference.MessageTargetCache); Assert.True(retainedEntry.IsValid); primary.ServiceProvider.GetRequiredService().InvalidateCache(grainReference.GrainId); Assert.False(retainedEntry.IsValid); Assert.False(retainedEntry.TryGetMessageTarget(out var disposedTarget)); Assert.Null(disposedTarget); - Assert.Same(retainedEntry, grainReference.DirectoryCacheEntry); + Assert.Same(retainedEntry, grainReference.MessageTargetCache); Assert.Equal(label, await grain.GetLabel()); - var replacementEntry = Assert.IsType(grainReference.DirectoryCacheEntry); + var replacementEntry = Assert.IsType(grainReference.MessageTargetCache); Assert.NotSame(retainedEntry, replacementEntry); Assert.True(replacementEntry.IsValid); Assert.Equal(grainReference.GrainId, replacementEntry.Address.GrainId); @@ -119,7 +119,7 @@ public async Task RemoteDirectoryGrain_CachesAddressWithoutBindingLocalTarget() RequestContext.Remove(IPlacementDirector.PlacementHintKey); } - var entry = Assert.IsType(grainReference.DirectoryCacheEntry); + var entry = Assert.IsType(grainReference.MessageTargetCache); Assert.True(entry.IsValid); Assert.Equal(grainReference.GrainId, entry.Address.GrainId); Assert.Equal(secondary.SiloAddress, entry.Address.SiloAddress); @@ -146,31 +146,27 @@ public async Task CompatibleInterfaceCast_TransfersSameEntryHandle() RequestContext.Remove(IPlacementDirector.PlacementHintKey); } - var writerEntry = Assert.IsType(writerReference.DirectoryCacheEntry); + var writerEntry = Assert.IsType(writerReference.MessageTargetCache); Assert.True(writerEntry.IsValid); var reader = writer.AsReference(); var readerReference = Assert.IsAssignableFrom(reader); Assert.NotSame(writerReference, readerReference); - Assert.Same(writerEntry, readerReference.DirectoryCacheEntry); + Assert.Same(writerEntry, readerReference.MessageTargetCache); Assert.Equal(writerReference.GrainId, readerReference.GrainId); Assert.Equal(value, await reader.GetValue()); - Assert.Same(writerEntry, readerReference.DirectoryCacheEntry); + Assert.Same(writerEntry, readerReference.MessageTargetCache); } [Fact] - public async Task ExternalClientCalls_DoNotAttachPerGrainDirectoryEntries() + public async Task ExternalClientCalls_DoNotAttachMessageTargetCache() { - var grains = Enumerable.Range(0, 32) - .Select(_ => _fixture.Client.GetGrain(Interlocked.Increment(ref _nextGrainKey))) - .ToArray(); + var grain = _fixture.Client.GetGrain(Interlocked.Increment(ref _nextGrainKey)); - foreach (var grain in grains) - { - await grain.SetLabel("external-client"); - Assert.Null(Assert.IsAssignableFrom(grain).DirectoryCacheEntry); - } + await grain.SetLabel("external-client"); + + Assert.Null(Assert.IsAssignableFrom(grain).MessageTargetCache); } [Fact] @@ -182,7 +178,7 @@ public async Task StatelessWorkerCalls_DoNotAttachDirectoryEntries() await grain.Nop(); - Assert.Null(Assert.IsAssignableFrom(grain).DirectoryCacheEntry); + Assert.Null(Assert.IsAssignableFrom(grain).MessageTargetCache); } [Fact] @@ -203,7 +199,7 @@ public async Task CacheInvalidationHeader_BypassesFastPathWithoutDiscardingLiveH RequestContext.Remove(IPlacementDirector.PlacementHintKey); } - var retainedEntry = Assert.IsType(grainReference.DirectoryCacheEntry); + var retainedEntry = Assert.IsType(grainReference.MessageTargetCache); var unrelatedAddress = new GrainAddress { GrainId = GrainId.Create("unrelated", Guid.NewGuid().ToString()), @@ -221,7 +217,7 @@ public async Task CacheInvalidationHeader_BypassesFastPathWithoutDiscardingLiveH Assert.Null(selectedEntry); Assert.True(retainedEntry.IsValid); - Assert.Same(retainedEntry, grainReference.DirectoryCacheEntry); + Assert.Same(retainedEntry, grainReference.MessageTargetCache); } private static IGrainFactory GetPrimarySiloGrainFactory(InProcessSiloHandle primary) From e654f1ea65e91e2d0f80279513ae6a6ffb48c8ed Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 19:12:46 -0700 Subject: [PATCH 07/21] perf(runtime): cache remote connection groups --- .testagent/plan.md | 2 +- .testagent/research.md | 2 +- .testagent/status.md | 4 +- .../Networking/ConnectionManager.cs | 38 ++++++++++- .../GrainDirectoryCacheEntry.cs | 43 +++++++++++-- .../Messaging/MessageCenter.cs | 64 +++++++++++++++---- .../GrainDirectoryCacheFactoryTests.cs | 57 ++++++++++++----- .../SharedEntryMessageTargetFastPathTests.cs | 6 +- 8 files changed, 176 insertions(+), 40 deletions(-) diff --git a/.testagent/plan.md b/.testagent/plan.md index cbf1d14c8c5..2960fa11f26 100644 --- a/.testagent/plan.md +++ b/.testagent/plan.md @@ -1,6 +1,6 @@ # Test Implementation Plan -> Final scope update: handler caching is narrowed to local `IGrainContext` targets. Remote references cache only the shared directory address and continue through the normal connection path; connection and response-route cases below remain deferred. +> Final scope update: local handlers cache `IGrainContext` targets and remote references cache connection-manager groups while preserving group selection/reconnect behavior. Response-route cases remain deferred. ## Overview diff --git a/.testagent/research.md b/.testagent/research.md index 34655c4b1a4..fdbbd7696e7 100644 --- a/.testagent/research.md +++ b/.testagent/research.md @@ -1,6 +1,6 @@ # Test Generation Research -> Final scope update: cached handlers are limited to local `IGrainContext` instances. Remote grain references retain the same shared directory entry for address reuse, while remote connection and response-route caching remain separate experiments. +> Final scope update: cached targets are limited to local `IGrainContext` instances and remote `ConnectionManager.ConnectionEntry` groups. Remote references retain the shared directory entry for address reuse; response-route caching remains a separate experiment. ## Project Overview - **Path**: `C:\dev\copilot-worktrees\orleans\rb-scaling-guacamole` diff --git a/.testagent/status.md b/.testagent/status.md index 8367cd19f63..43d4aadc697 100644 --- a/.testagent/status.md +++ b/.testagent/status.md @@ -9,11 +9,11 @@ The shared directory-entry message target cache is covered by: ## Results -- 28 focused tests pass on `net10.0`. +- 29 focused tests pass on `net10.0`. - No generated test is assertion-free or relies only on presence/truthiness checks. - Assertions cover equality, identity, type, null, negative behavior, collection state, lifecycle state, and side effects. - Cache lifetime tests verify in-place replacement, conditional removal, clear, TTL expiration, eviction, tombstoning, target release, one-shot binding, and identity-checked clearing. -- Integration tests verify local target binding, remote address-only caching, invalidation and recapture, compatible-interface casts, and explicit external-client and stateless-worker exclusion. +- Integration tests verify local activation binding, remote connection-group binding, invalidation and recapture, compatible-interface casts, and explicit external-client and stateless-worker exclusion. ## Pseudo-Mutation Review diff --git a/src/Orleans.Core/Networking/ConnectionManager.cs b/src/Orleans.Core/Networking/ConnectionManager.cs index 553666bcd07..63a8db5bf5c 100644 --- a/src/Orleans.Core/Networking/ConnectionManager.cs +++ b/src/Orleans.Core/Networking/ConnectionManager.cs @@ -64,17 +64,28 @@ public ValueTask GetConnection(SiloAddress endpoint) return new(this.GetConnectionAsync(endpoint)); } public bool TryGetConnection(SiloAddress endpoint, [NotNullWhen(true)] out Connection? connection) + => TryGetConnection(endpoint, out connection, out _); + + internal bool TryGetConnection( + SiloAddress endpoint, + [NotNullWhen(true)] out Connection? connection, + [NotNullWhen(true)] out ConnectionEntry? connectionEntry) { if (this.connections.TryGetValue(endpoint, out var entry) && entry.NextConnection() is { } c) { connection = c; + connectionEntry = entry; return true; } connection = null; + connectionEntry = null; return false; } + internal bool TryGetConnectionEntry(SiloAddress endpoint, [NotNullWhen(true)] out ConnectionEntry? connectionEntry) + => connections.TryGetValue(endpoint, out connectionEntry); + /// /// Gets the minimum elapsed time since any connection to the specified silo last received a message, /// or if no connections exist or no messages have been received. @@ -197,7 +208,7 @@ public void OnConnectionTerminated(SiloAddress address, Connection connection, E } } - private ConnectionEntry GetOrCreateEntry(SiloAddress address) => connections.GetOrAdd(address, _ => new()); + private ConnectionEntry GetOrCreateEntry(SiloAddress address) => connections.GetOrAdd(address, static address => new(address)); private async Task ConnectAsync(SiloAddress address, ConnectionEntry entry) { @@ -256,7 +267,10 @@ public async Task CloseAsync(SiloAddress endpoint) connections = entry.Connections; if (entry.PendingConnection is null) { - this.connections.TryRemove(endpoint, out _); + if (this.connections.TryRemove(KeyValuePair.Create(endpoint, entry))) + { + connections = entry.Detach(); + } } } @@ -372,8 +386,13 @@ private async Task RunConnectionAsync(SiloAddress address, Connection connection return null; } - private sealed class ConnectionEntry + internal sealed class ConnectionEntry { + private int _detached; + + public ConnectionEntry(SiloAddress endpoint) => Endpoint = endpoint; + + public SiloAddress Endpoint { get; } public Task? PendingConnection { get; set; } public DateTime LastFailure { get; set; } public ImmutableArray Connections { get; set; } = ImmutableArray.Empty; @@ -398,6 +417,11 @@ public TimeSpan GetRemainingRetryDelay(ConnectionOptions options) public Connection? NextConnection() { + if (Volatile.Read(ref _detached) != 0) + { + return null; + } + var connections = this.Connections; if (connections.IsEmpty) { @@ -409,6 +433,14 @@ public TimeSpan GetRemainingRetryDelay(ConnectionOptions options) } public void RemoveDefunct() => Connections = Connections.RemoveAll(c => !c.IsValid); + + public ImmutableArray Detach() + { + Volatile.Write(ref _detached, 1); + var result = Connections; + Connections = ImmutableArray.Empty; + return result; + } } [LoggerMessage( diff --git a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs index 13c6e6b2d50..941f996563a 100644 --- a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs +++ b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs @@ -29,7 +29,7 @@ public GrainDirectoryCacheEntry( public bool IsValid => !ReferenceEquals(Volatile.Read(ref _messageTarget), Invalidated); - public bool TryGetMessageTarget(out IGrainContext? messageTarget) + public bool TryGetMessageTarget(out object? messageTarget) { var target = Volatile.Read(ref _messageTarget); if (ReferenceEquals(target, Invalidated)) @@ -38,13 +38,48 @@ public bool TryGetMessageTarget(out IGrainContext? messageTarget) return false; } - messageTarget = target as IGrainContext; + messageTarget = target; return messageTarget is not null; } - public bool TrySetMessageTarget(IGrainContext messageTarget) + public bool TrySetMessageTarget(object messageTarget, GrainAddress expectedAddress) { ArgumentNullException.ThrowIfNull(messageTarget); + ArgumentNullException.ThrowIfNull(expectedAddress); + if (!Address.Matches(expectedAddress) || !TrySetMessageTargetCore(messageTarget)) + { + return false; + } + + if (Address.Matches(expectedAddress)) + { + return true; + } + + ClearMessageTarget(messageTarget); + return false; + } + + public bool TrySetMessageTarget(object messageTarget, SiloAddress expectedSilo) + { + ArgumentNullException.ThrowIfNull(messageTarget); + ArgumentNullException.ThrowIfNull(expectedSilo); + if (Address.SiloAddress?.Equals(expectedSilo) != true || !TrySetMessageTargetCore(messageTarget)) + { + return false; + } + + if (Address.SiloAddress?.Equals(expectedSilo) == true) + { + return true; + } + + ClearMessageTarget(messageTarget); + return false; + } + + private bool TrySetMessageTargetCore(object messageTarget) + { var current = Volatile.Read(ref _messageTarget); if (ReferenceEquals(current, Invalidated)) { @@ -55,7 +90,7 @@ public bool TrySetMessageTarget(IGrainContext messageTarget) || current is null && Interlocked.CompareExchange(ref _messageTarget, messageTarget, null) is null; } - public void ClearMessageTarget(IGrainContext messageTarget) + public void ClearMessageTarget(object messageTarget) { ArgumentNullException.ThrowIfNull(messageTarget); Interlocked.CompareExchange(ref _messageTarget, null, messageTarget); diff --git a/src/Orleans.Runtime/Messaging/MessageCenter.cs b/src/Orleans.Runtime/Messaging/MessageCenter.cs index 149d7a0aedd..0d20f85ef6b 100644 --- a/src/Orleans.Runtime/Messaging/MessageCenter.cs +++ b/src/Orleans.Runtime/Messaging/MessageCenter.cs @@ -192,9 +192,8 @@ private void SendMessage( return; } - if (directoryCacheEntry is not null) + if (directoryCacheEntry is not null && msg.TargetSilo?.Matches(_siloAddress) == true) { - Debug.Assert(msg.TargetSilo?.Matches(_siloAddress) == true); Debug.Assert(!msg.TargetGrain.IsClient()); Debug.Assert(msg.Direction is Message.Directions.Request or Message.Directions.OneWay); MessagingEvents.EmitSent(msg); @@ -229,8 +228,14 @@ private void SendMessage( } else { - if (this.connectionManager.TryGetConnection(targetSilo, out var existingConnection)) + if (TryGetCachedConnection(directoryCacheEntry, targetSilo, out var cachedConnection)) { + cachedConnection.Send(msg); + return; + } + else if (this.connectionManager.TryGetConnection(targetSilo, out var existingConnection, out var connectionEntry)) + { + directoryCacheEntry?.TrySetMessageTarget(connectionEntry, targetSilo); existingConnection.Send(msg); return; } @@ -251,20 +256,34 @@ private void SendMessage( if (connectionTask.IsCompletedSuccessfully) { var sender = connectionTask.Result; + if (directoryCacheEntry is not null + && connectionManager.TryGetConnectionEntry(targetSilo, out var completedConnectionEntry)) + { + directoryCacheEntry.TrySetMessageTarget(completedConnectionEntry, targetSilo); + } + sender.Send(msg); } else { - _ = SendAsync(this, connectionTask, msg); + _ = SendAsync(this, connectionTask, msg, directoryCacheEntry, targetSilo); static async Task SendAsync( MessageCenter messageCenter, ValueTask connectionTask, - Message msg) + Message msg, + GrainDirectoryCacheEntry? directoryCacheEntry, + SiloAddress targetSilo) { try { var sender = await connectionTask; + if (directoryCacheEntry is not null + && messageCenter.connectionManager.TryGetConnectionEntry(targetSilo, out var connectionEntry)) + { + directoryCacheEntry.TrySetMessageTarget(connectionEntry, targetSilo); + } + sender.Send(msg); } catch (Exception exception) @@ -278,6 +297,29 @@ static async Task SendAsync( } } + private static bool TryGetCachedConnection( + GrainDirectoryCacheEntry? directoryCacheEntry, + SiloAddress targetSilo, + [NotNullWhen(true)] out Connection? connection) + { + if (directoryCacheEntry?.TryGetMessageTarget(out var target) == true + && target is { } messageTarget) + { + if (messageTarget is ConnectionManager.ConnectionEntry connectionEntry + && connectionEntry.Endpoint.Equals(targetSilo) + && connectionEntry.NextConnection() is { } result) + { + connection = result; + return true; + } + + directoryCacheEntry.ClearMessageTarget(messageTarget); + } + + connection = null; + return false; + } + public void DispatchLocalMessage(Message message) => ReceiveMessage(message); public void RejectMessage( @@ -507,7 +549,7 @@ internal Task AddressAndSendMessage(Message message, GrainReference? target = nu { var targetSilo = directoryCacheEntry.Address.SiloAddress!; message.TargetSilo = targetSilo; - SendMessage(message, targetSilo.Matches(_siloAddress) ? directoryCacheEntry : null); + SendMessage(message, directoryCacheEntry); return Task.CompletedTask; } @@ -594,7 +636,7 @@ internal bool TryGetDirectoryCacheEntry( } target.MessageTargetCache = entry; - return targetSilo.Matches(_siloAddress) ? entry : null; + return entry; } internal void SendResponse(Message request, Response response) @@ -664,7 +706,7 @@ private void ReceiveApplicationMessage(Message message, GrainDirectoryCacheEntry && directoryCacheEntry is { } entry && entry.Address.Matches(activation.Address)) { - entry.TrySetMessageTarget(activation); + entry.TrySetMessageTarget(activation, activation.Address); } } @@ -682,14 +724,14 @@ private void ReceiveApplicationMessage(Message message, GrainDirectoryCacheEntry private static ActivationData? GetCachedActivation(GrainDirectoryCacheEntry? directoryCacheEntry) { if (directoryCacheEntry?.TryGetMessageTarget(out var target) == true - && target is ActivationData activation) + && target is { } messageTarget) { - if (activation.IsValid) + if (messageTarget is ActivationData { IsValid: true } activation) { return activation; } - directoryCacheEntry.ClearMessageTarget(activation); + directoryCacheEntry.ClearMessageTarget(messageTarget); } return null; diff --git a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs index c0b25593726..37519a576c1 100644 --- a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs +++ b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs @@ -273,7 +273,7 @@ public async Task CreateGrainDirectoryCache_AddOrUpdateUpdatesSharedRouteHandle( { cache.AddOrUpdate(originalAddress, version: 1); var originalEntry = GetEntry(entrySource, grainId); - Assert.True(originalEntry.TrySetMessageTarget(originalTarget)); + Assert.True(originalEntry.TrySetMessageTarget(originalTarget, originalEntry.Address)); cache.AddOrUpdate(replacementAddress, version: 2); var replacementEntry = GetEntry(entrySource, grainId); @@ -283,7 +283,7 @@ public async Task CreateGrainDirectoryCache_AddOrUpdateUpdatesSharedRouteHandle( Assert.Equal(replacementAddress, replacementEntry.Address); Assert.Equal(2, replacementEntry.Version); Assert.False(replacementEntry.TryGetMessageTarget(out _)); - Assert.True(replacementEntry.TrySetMessageTarget(replacementTarget)); + Assert.True(replacementEntry.TrySetMessageTarget(replacementTarget, replacementEntry.Address)); AssertMessageTarget(replacementEntry, replacementTarget); Assert.True(cache.LookUp(grainId, out var result, out var version)); Assert.Equal(replacementAddress, result); @@ -307,7 +307,7 @@ public async Task CreateGrainDirectoryCache_RemoveByGrainIdInvalidatesRouteHandl { cache.AddOrUpdate(address, version: 3); var entry = GetEntry(entrySource, address.GrainId); - Assert.True(entry.TrySetMessageTarget(target)); + Assert.True(entry.TrySetMessageTarget(target, entry.Address)); Assert.True(cache.Remove(address.GrainId)); @@ -340,8 +340,8 @@ public async Task CreateGrainDirectoryCache_RemoveByAddressInvalidatesOnlyMatchi cache.AddOrUpdate(controlAddress, version: 5); var entry = GetEntry(entrySource, grainId); var controlEntry = GetEntry(entrySource, controlAddress.GrainId); - Assert.True(entry.TrySetMessageTarget(target)); - Assert.True(controlEntry.TrySetMessageTarget(controlTarget)); + Assert.True(entry.TrySetMessageTarget(target, entry.Address)); + Assert.True(controlEntry.TrySetMessageTarget(controlTarget, controlEntry.Address)); Assert.False(cache.Remove(mismatchedAddress)); AssertMessageTarget(entry, target); @@ -383,7 +383,7 @@ public async Task CreateGrainDirectoryCache_ClearInvalidatesAllRouteHandles() { cache.AddOrUpdate(addresses[i], version: i + 1); entries[i] = GetEntry(entrySource, addresses[i].GrainId); - Assert.True(entries[i].TrySetMessageTarget(CreateMessageTarget())); + Assert.True(entries[i].TrySetMessageTarget(CreateMessageTarget(), entries[i].Address)); } cache.Clear(); @@ -423,7 +423,7 @@ public async Task CreateGrainDirectoryCache_ExpirationInvalidatesRouteHandle() { cache.AddOrUpdate(expiredAddress, version: 6); var expiredEntry = GetEntry(entrySource, expiredAddress.GrainId); - Assert.True(expiredEntry.TrySetMessageTarget(CreateMessageTarget())); + Assert.True(expiredEntry.TrySetMessageTarget(CreateMessageTarget(), expiredEntry.Address)); timeProvider.Advance(timeToLive); Assert.Equal(0, await listener.WaitForCleanupAsync()); @@ -431,7 +431,7 @@ public async Task CreateGrainDirectoryCache_ExpirationInvalidatesRouteHandle() cache.AddOrUpdate(freshAddress, version: 7); var freshEntry = GetEntry(entrySource, freshAddress.GrainId); var freshTarget = CreateMessageTarget(); - Assert.True(freshEntry.TrySetMessageTarget(freshTarget)); + Assert.True(freshEntry.TrySetMessageTarget(freshTarget, freshEntry.Address)); timeProvider.Advance(timeToLive); Assert.Equal(1, await listener.WaitForCleanupAsync()); @@ -471,13 +471,13 @@ public async Task CreateGrainDirectoryCache_EvictionInvalidatesRouteHandle() cache.AddOrUpdate(addresses[i], version: i + 1); entries[i] = GetEntry(entrySource, addresses[i].GrainId); targets[i] = CreateMessageTarget(); - Assert.True(entries[i].TrySetMessageTarget(targets[i])); + Assert.True(entries[i].TrySetMessageTarget(targets[i], entries[i].Address)); } cache.AddOrUpdate(addresses[3], version: 4); entries[3] = GetEntry(entrySource, addresses[3].GrainId); targets[3] = CreateMessageTarget(); - Assert.True(entries[3].TrySetMessageTarget(targets[3])); + Assert.True(entries[3].TrySetMessageTarget(targets[3], entries[3].Address)); AssertInvalidEntry(entries[0]); Assert.False(entrySource.TryGetEntry(addresses[0].GrainId, out _)); @@ -517,8 +517,8 @@ public void GrainDirectoryCacheEntry_SecondTargetCannotReplaceBoundTarget() var entry = new GrainDirectoryCacheEntry(CreateGrainAddress(CreateGrainId(), port: 11111), version: 9); var originalTarget = CreateMessageTarget(); - Assert.True(entry.TrySetMessageTarget(originalTarget)); - Assert.False(entry.TrySetMessageTarget(CreateMessageTarget())); + Assert.True(entry.TrySetMessageTarget(originalTarget, entry.Address)); + Assert.False(entry.TrySetMessageTarget(CreateMessageTarget(), entry.Address)); AssertMessageTarget(entry, originalTarget); } @@ -528,17 +528,42 @@ public void GrainDirectoryCacheEntry_ClearRequiresBoundTargetIdentity() var entry = new GrainDirectoryCacheEntry(CreateGrainAddress(CreateGrainId(), port: 11111), version: 10); var originalTarget = CreateMessageTarget(); var replacementTarget = CreateMessageTarget(); - Assert.True(entry.TrySetMessageTarget(originalTarget)); + Assert.True(entry.TrySetMessageTarget(originalTarget, entry.Address)); entry.ClearMessageTarget(CreateMessageTarget()); AssertMessageTarget(entry, originalTarget); entry.ClearMessageTarget(originalTarget); Assert.False(entry.TryGetMessageTarget(out _)); - Assert.True(entry.TrySetMessageTarget(replacementTarget)); + Assert.True(entry.TrySetMessageTarget(replacementTarget, entry.Address)); AssertMessageTarget(entry, replacementTarget); } + [Fact] + public async Task GrainDirectoryCacheEntry_StaleAddressCannotRebindAfterUpdate() + { + var (cache, entrySource) = CreateEntryCache(); + var disposableCache = Assert.IsAssignableFrom(cache); + var grainId = CreateGrainId(); + var originalAddress = CreateGrainAddress(grainId, port: 11111); + var replacementAddress = CreateGrainAddress(grainId, port: 22222); + + try + { + cache.AddOrUpdate(originalAddress, version: 1); + var entry = GetEntry(entrySource, grainId); + cache.AddOrUpdate(replacementAddress, version: 2); + + Assert.False(entry.TrySetMessageTarget(CreateMessageTarget(), originalAddress)); + Assert.False(entry.TryGetMessageTarget(out _)); + Assert.Equal(replacementAddress, entry.Address); + } + finally + { + await disposableCache.DisposeAsync(); + } + } + [Fact] public async Task CreateGrainDirectoryCache_RemoveByGrainIdReleasesMessageTargetReference() { @@ -602,14 +627,14 @@ private static void AssertInvalidEntry(GrainDirectoryCacheEntry entry) Assert.False(entry.IsValid); Assert.False(entry.TryGetMessageTarget(out var actual)); Assert.Null(actual); - Assert.False(entry.TrySetMessageTarget(CreateMessageTarget())); + Assert.False(entry.TrySetMessageTarget(CreateMessageTarget(), entry.Address)); } [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private static WeakReference BindMessageTarget(GrainDirectoryCacheEntry entry) { var target = CreateMessageTarget(); - Assert.True(entry.TrySetMessageTarget(target)); + Assert.True(entry.TrySetMessageTarget(target, entry.Address)); return new WeakReference(target); } diff --git a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs index 055e91c5e46..91c96dd8dea 100644 --- a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs +++ b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs @@ -98,7 +98,7 @@ public async Task InvalidateCache_DisposesRetainedEntry_AndNextCallCapturesDiffe } [Fact] - public async Task RemoteDirectoryGrain_CachesAddressWithoutBindingLocalTarget() + public async Task RemoteDirectoryGrain_BindsConnectionGroupTarget() { var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; var secondary = _fixture.HostedCluster.Silos.Single( @@ -123,7 +123,9 @@ public async Task RemoteDirectoryGrain_CachesAddressWithoutBindingLocalTarget() Assert.True(entry.IsValid); Assert.Equal(grainReference.GrainId, entry.Address.GrainId); Assert.Equal(secondary.SiloAddress, entry.Address.SiloAddress); - Assert.False(entry.TryGetMessageTarget(out _)); + Assert.True(entry.TryGetMessageTarget(out var messageTarget)); + var connectionEntry = Assert.IsType(messageTarget); + Assert.Equal(secondary.SiloAddress, connectionEntry.Endpoint); } [Fact] From bfb902b54b9010cfe006d564ebc891e011c9d467 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 27 Aug 2026 20:01:44 -0700 Subject: [PATCH 08/21] chore: remove test planning artifacts --- .testagent/plan.md | 377 ----------------------------------------- .testagent/research.md | 147 ---------------- .testagent/status.md | 30 ---- 3 files changed, 554 deletions(-) delete mode 100644 .testagent/plan.md delete mode 100644 .testagent/research.md delete mode 100644 .testagent/status.md diff --git a/.testagent/plan.md b/.testagent/plan.md deleted file mode 100644 index 2960fa11f26..00000000000 --- a/.testagent/plan.md +++ /dev/null @@ -1,377 +0,0 @@ -# Test Implementation Plan - -> Final scope update: local handlers cache `IGrainContext` targets and remote references cache connection-manager groups while preserving group selection/reconnect behavior. Response-route cases remain deferred. - -## Overview - -This is a targeted, three-phase plan for the forthcoming shared directory-entry message-target cache. Existing coverage is substantial for locator selection and partial for cache lifetime, placement, local activation, and networking, while the shared route-handle API is currently absent. - -All test source additions are gated on the production API landing. Once available, tests will use the repository's xUnit v3/Microsoft Testing Platform conventions, Arrange-Act-Assert structure, existing BVT/category/area attributes, deterministic synchronization, and supported internal test accessors rather than reflection. - -## Hard Boundary: Not in This Preparation Turn - -- Do not modify production code. -- Do not create or edit any test source file. -- Do not build, discover, or run tests while the production API is absent. -- Do not restore, clean, reset, stash, or delete tracked files. -- This turn creates only `.testagent/plan.md`; do not create `.testagent/status.md`. -- The phases below begin only after the route-handle/cache-entry API and required test seams land. - -## Commands - -Run from the workspace root only after the production API lands and dependencies are already available. - -- **Scoped build (runtime tests)**: `dotnet build test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj --framework net10.0 --no-restore` -- **Scoped build (core tests)**: `dotnet build test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore` -- **Full build**: `dotnet build Orleans.slnx --no-restore --no-incremental -bl` -- **Scoped test (directory cache lifetime)**: `dotnet test --project test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj --framework net10.0 --no-restore --filter-class "*GrainDirectoryCacheFactoryTests*" --minimum-expected-tests 1` -- **Scoped test (stale async binding)**: `dotnet test --project test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore --filter-class "*CachedGrainLocatorTests*" --minimum-expected-tests 1` -- **Scoped test (shared/local/remote routes and growth)**: `dotnet test --project test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore --filter-class "*MessageTargetCacheTests*" --minimum-expected-tests 1` -- **Scoped test (placement and compatibility)**: `dotnet test --project test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore --filter-class "*PlacementServiceTests*" --minimum-expected-tests 1` -- **Scoped regression (locator selection)**: `dotnet test --project test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore --filter-class "*GrainLocatorResolverTests*" --minimum-expected-tests 1` -- **Full test (all target frameworks with a fresh build)**: `dotnet test --solution Orleans.slnx --no-restore --minimum-expected-tests 1 --max-parallel-test-modules 1` -- **Full discovery check (all target frameworks)**: `dotnet test --solution Orleans.slnx --no-restore --list-tests --minimum-expected-tests 1 --max-parallel-test-modules 1` -- **Lint**: No separate command; the build enforces style and treats warnings, including nullable warnings, as errors. - -## Phase Summary - -| Phase | Focus | Test files | Est. tests | -|---|---|---:|---:| -| 1 | Directory-entry ownership and stale async binds | 2 existing | 8 | -| 2 | Shared tombstones plus local/remote route validation | 1 new | 9 | -| 3 | Header/compatibility regressions and bounded growth | 2 (1 existing, 1 from Phase 2) | 8 | - ---- - -## Phase 1: Directory Entry Lifetime and Race Safety - -### Overview - -Establish the leaf/mid-layer lifetime contract first. Every directory cache removal path must invalidate the owned shared route handle, and delayed directory lookups must not bind a disposed or superseded generation. - -### Production Scope - -Assigned to this phase: - -- `src/Orleans.Runtime/GrainDirectory/IGrainDirectoryCache.cs` -- `src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs` -- `src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheFactory.cs` -- `src/Orleans.Runtime/Configuration/Options/GrainDirectoryOptions.cs` -- `src/Orleans.Runtime/GrainDirectory/CachedGrainLocator.cs` - -### Files to Test - -#### 1. GrainDirectoryCacheFactoryTests.cs - -- **Project**: `test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj` -- **Test File**: `test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs` -- **Test Class**: `GrainDirectoryCacheFactoryTests` -- **Methods/contracts**: factory creation; cache `AddOrUpdate`, both `Remove` variants, `Clear`, expiration cleanup, and capacity eviction. -- **Current status**: Cache mechanics are regression-testable now, but route-handle assertions are blocked. Add all tests only after the API lands. -- **Future seam/blocker**: The cache value must be a disposable directory entry which owns an observable route handle/tombstone. Tests need supported validity/generation observations, not private-field reflection. - -**Tests**: - -1. `CreateGrainDirectoryCache_AddOrUpdateInvalidatesReplacedRouteHandle` - - Arrange: Create the LRU cache with the existing factory fixture. Add a directory entry for one `GrainId` with address A and handle H1, then prepare address B with H2. - - Act: `AddOrUpdate` the same `GrainId` with the replacement entry. - - Assert: H1 is invalid/disposed exactly once; H2 remains valid and lookup returns address B. - - Secondary observable: A consumer retaining H1 cannot obtain a usable local receiver or remote connection from it. - -2. `CreateGrainDirectoryCache_RemoveByGrainIdInvalidatesRouteHandle` - - Arrange: Add one route-owning entry and retain its handle separately. - - Act: Remove by `GrainId`. - - Assert: The retained handle is a tombstone and cache lookup misses. - - Secondary observable: A second removal is harmless and does not repeat disposal. - -3. `CreateGrainDirectoryCache_RemoveByAddressInvalidatesOnlyMatchingRouteHandle` - - Arrange: Add an entry for address A and retain H1; create a different address B for the same grain. - - Act: First remove using B, then remove using A. - - Assert: The mismatched removal leaves H1 valid and the entry present; the matching removal invalidates H1 and removes the entry. - - Secondary observable: No unrelated route handle is invalidated. - -4. `CreateGrainDirectoryCache_ClearInvalidatesAllRouteHandles` - - Arrange: Add multiple distinct grain entries and retain every handle. - - Act: Call `Clear`. - - Assert: Every retained handle is invalid exactly once and every lookup misses. - - Secondary observable: A subsequent `Clear` is idempotent. - -5. `CreateGrainDirectoryCache_ExpirationInvalidatesRouteHandle` - - Arrange: Use `FakeTimeProvider`, the configured TTL, and `ConcurrentLruCacheExpirationCleanupListener`; add an entry and retain its handle. - - Act: Advance fake time beyond TTL and deterministically invoke/wait for expiration cleanup without wall-clock delays. - - Assert: The handle is invalid and lookup misses. - - Secondary observable: An unexpired control entry remains valid until its own deadline. - -6. `CreateGrainDirectoryCache_EvictionInvalidatesRouteHandle` - - Arrange: Use deterministic capacity `3`; add three entries, establish the intended LRU order, retain all handles, then prepare a fourth. - - Act: Add the fourth entry. - - Assert: Only the least-recently-used entry's handle is invalidated and its lookup misses. - - Secondary observable: The other three entries/handles remain usable. - -#### 2. CachedGrainLocatorTests.cs - -- **Project**: `test/Orleans.Core.Tests/Orleans.Core.Tests.csproj` -- **Test File**: `test/Orleans.Core.Tests/Directory/CachedGrainLocatorTests.cs` -- **Test Class**: `CachedGrainLocatorTests` -- **Methods/contracts**: asynchronous `Lookup` cache binding and `Unregister` invalidation. -- **Current status**: Lookup/unregister races are regression-testable now; generation/tombstone assertions are blocked. Existing `UnregisterRacesWithLookupSameId` remains the nearest current regression. -- **Future seam/blocker**: Binding must accept/check an atomic generation or token so an await continuation cannot bind a disposed or superseded handle. - -**Tests**: - -1. `Lookup_WhenInvalidatedBeforeAsyncLookupCompletes_DoesNotBindStaleRoute` - - Arrange: Substitute the custom directory and gate its lookup with `TaskCompletionSource` instances created using `RunContinuationsAsynchronously`. Start `Lookup`, wait until the fake records entry, retain the pending route handle/generation, then invalidate or supersede that entry. - - Act: Complete the old directory lookup with a valid-looking address and await the original operation. - - Assert: The stale generation is not bound to the returned address/target. - - Secondary observable: A subsequent lookup takes the ordinary directory path (recorded call count) or uses only the newer generation; it never uses the stale receiver/connection. - -2. `Unregister_WhenLookupCompletesAfterRemoval_DoesNotResurrectRouteHandle` - - Arrange: Start a gated `Lookup` for a grain and wait until the directory call is in flight. - - Act: Call `Unregister`/remove first, then release the old lookup result. - - Assert: The removed handle remains an invalid tombstone and no live cache entry is resurrected. - - Secondary observable: The next lookup invokes the directory again and can bind only a newly issued generation. - - Cleanup: Release all gates in `finally` so failures cannot leave tasks blocked. - -### Phase 1 Success Criteria - -- [ ] Production API landing gate is satisfied. -- [ ] Six lifetime paths invalidate exactly the intended handles. -- [ ] Both race tests use deterministic gates, not delays. -- [ ] Scoped runtime and core builds pass. -- [ ] Both scoped test classes pass. - ---- - -## Phase 2: Shared Tombstones and Direct Route Validation - -### Overview - -Verify shared identity/lifetime first, then local and remote fast-path validity. These tests cover top-level route consumption while isolating fallback behavior with fakes and call counters. - -### Production Scope - -Assigned to this phase: - -- `src/Orleans.Core.Abstractions/Runtime/GrainReference.cs` -- `src/Orleans.Runtime/Catalog/ActivationDirectory.cs` -- `src/Orleans.Runtime/Catalog/Catalog.cs` -- `src/Orleans.Runtime/Catalog/ActivationData.cs` -- `src/Orleans.Core.Abstractions/Core/IGrainContext.cs` -- `src/Orleans.Core/Networking/Connection.cs` -- `src/Orleans.Core/Networking/ConnectionManager.cs` -- `src/Orleans.Runtime/Messaging/MessageCenter.cs` - -### File to Test - -#### MessageTargetCacheTests.cs - -- **Project**: `test/Orleans.Core.Tests/Orleans.Core.Tests.csproj` -- **Test File**: `test/Orleans.Core.Tests/Runtime/MessageTargetCacheTests.cs` (new, only after API landing) -- **Test Class**: `MessageTargetCacheTests` -- **Methods/contracts**: future shared route acquisition/invalidation, local receiver probe, remote connection-group probe, and fallback. -- **Fixtures/fakes**: NSubstitute for interfaces; a controllable collectible grain-context fake exposing exact `GrainAddress` and `IsValid`; a controllable connection/group fake exposing address, group identity, and `IsValid`; ordinary lookup/acquisition delegates with invocation counters. -- **Future seam/blocker**: An observable atomic route-handle/tombstone, weak local receiver retention after invalidation, remote group identity/address validation, and ordinary fallback hooks are all required. - -**Shared identity and lifetime tests**: - -1. `GrainReferences_WithSameGrainIdShareRouteHandle` - - Arrange: Create two grain references for the same `GrainId` through the normal activator/reference path, including different compatible interface views if supported. - - Act: Obtain the supported route-handle view and bind/invalidate through one reference. - - Assert: Both references expose the same handle instance and observe the same state transition. - - Secondary observable: The supported route-entry count increases once, not once per reference/interface view. - -2. `GrainReference_RetainsDisposedRouteHandleTombstone` - - Arrange: Create a grain reference and its directory entry; retain the reference, dispose/remove the entry, and drop other strong references to the live entry. - - Act: Force bounded GC only if needed to separate entry lifetime from reference lifetime. - - Assert: The grain reference still retains the same invalid tombstone, rather than silently acquiring a new live handle. - - Secondary observable: A route probe through that reference falls back and cannot use the former target. - -3. `DisposedRouteHandle_DoesNotRetainLocalReceiver` - - Arrange: In a `[MethodImpl(MethodImplOptions.NoInlining)]` helper, create a collectible local receiver, bind it to the handle, return a retained grain reference plus `WeakReference` to the receiver, invalidate the handle, and ensure the helper drops all receiver/context strong references. - - Act: Perform a bounded collect/finalize/collect loop, following the collectible-context test pattern; do not use timing sleeps. - - Assert: The receiver weak reference is no longer alive while the grain reference and tombstone remain alive. - - Secondary observable: The retained tombstone is invalid and its local probe takes fallback. - -**Local route tests**: - -4. `LocalRoute_WhenAddressMatchesAndActivationIsValidUsesReceiver` - - Arrange: Bind a local receiver whose `IGrainContext.Address` exactly equals the cached `GrainAddress` and whose collectible context reports `IsValid = true`; install an ordinary locator fallback counter. - - Act: Resolve/send through the local route probe. - - Assert: The exact receiver is selected. - - Secondary observable: Ordinary locator/fallback is not called and no different activation receives the message. - -5. `LocalRoute_WhenActivationAddressDiffersFallsBack` - - Arrange: Bind a receiver with the same grain identity but a different activation/address instance; configure fallback to return a known target. - - Act: Probe the cached local route. - - Assert: The cached receiver is rejected and the known fallback result is selected. - - Secondary observable: Fallback is called exactly once and the mismatched receiver is not sent to. - -6. `LocalRoute_WhenActivationIsInvalidFallsBack` - - Arrange: Bind a receiver with an exactly matching address but `IsValid = false`; configure a known fallback. - - Act: Probe the cached local route. - - Assert: The invalid receiver is rejected and fallback is selected. - - Secondary observable: Fallback is called once and no direct send occurs. - -**Remote route tests**: - -7. `RemoteRoute_WhenConnectionGroupMatchesAddressUsesConnection` - - Arrange: Bind a valid connection from the exact cached connection group associated with the destination `SiloAddress`; install an ordinary `ConnectionManager` acquisition counter. - - Act: Resolve/send through the remote route. - - Assert: The bound connection is selected. - - Secondary observable: Ordinary connection acquisition is not called and destination address remains exact. - -8. `RemoteRoute_WhenConnectionGroupAddressDiffersFallsBack` - - Arrange: Bind a valid connection/group for silo A while the message destination is silo B; configure ordinary acquisition to return a known connection for B. - - Act: Probe/send the cached route. - - Assert: The stale group/connection is rejected and the B fallback connection is selected. - - Secondary observable: Ordinary acquisition is called once and the A connection receives no send. - -9. `RemoteRoute_WhenConnectionIsInvalidFallsBack` - - Arrange: Bind the exact destination group but make the selected connection report `IsValid = false`; configure a valid ordinary fallback connection. - - Act: Probe/send the route. - - Assert: The invalid cached connection is rejected and fallback is selected. - - Secondary observable: Ordinary acquisition is called once and the invalid connection receives no send. - -### Phase 2 Success Criteria - -- [ ] Shared references demonstrably use one observable handle. -- [ ] Disposed handles remain tombstones but do not root receivers. -- [ ] Local probes require exact address and valid state. -- [ ] Remote probes require exact group/address and valid connection. -- [ ] Every rejection proves both non-use of the stale target and use of ordinary fallback. -- [ ] Scoped core build and `MessageTargetCacheTests` pass. - ---- - -## Phase 3: Placement Compatibility, Invalidation Headers, and Growth Guard - -### Overview - -Protect existing routing behavior around the new fast path, then add a durable supported-count guard against accidental per-client-`GrainId` growth. - -### Production Scope - -Assigned to this phase: - -- `src/Orleans.Runtime/Placement/PlacementService.cs` -- `src/Orleans.Runtime/Core/InsideRuntimeClient.cs` -- `src/Orleans.Runtime/GrainDirectory/ClientGrainLocator.cs` -- `src/Orleans.Runtime/GrainDirectory/GrainLocatorResolver.cs` -- `src/Orleans.Runtime/GrainDirectory/GrainLocator.cs` -- `src/Orleans.Core/GrainReferences/GrainReferenceActivator.cs` -- `src/Orleans.Core/Messaging/ClientMessageCenter.cs` -- `src/Orleans.Core/Runtime/OutsideRuntimeClient.cs` - -### Files to Test - -#### 1. PlacementServiceTests.cs - -- **Project**: `test/Orleans.Core.Tests/Orleans.Core.Tests.csproj` -- **Test File**: `test/Orleans.Core.Tests/Runtime/PlacementServiceTests.cs` -- **Test Class/fixture**: Existing `PlacementServiceTests` and `PlacementServiceFixture` -- **Methods/contracts**: `AddressMessage`, invalidation-header handling, and compatibility path selection. -- **Current status**: Header bypass and current locator/factory selection are regression-testable now, but all additions remain gated. Shared-route non-interference assertions require the future API. -- **Fixtures/fakes**: Reuse `PlacementServiceFixture`, `MockClusterMembershipService`, `SiloLifecycleSubject`, and NSubstitute. Explicitly stop lifecycle resources and dispose in `finally`. -- **Future seam/blocker**: Fixture-visible route binding/state and supported interaction points for the ordinary locator/placement path. - -**Tests**: - -1. `AddressMessage_CacheInvalidationHeaderBypassesSharedRouteFastPath` - - Arrange: Bind a shared fast route to stale address/receiver A; create a message carrying a matching cache-invalidation header; configure ordinary locator/worker resolution to return B. - - Act: Call `AddressMessage`. - - Assert: The message is addressed through B, not A, and the stale handle is invalidated/bypassed. - - Secondary observable: Ordinary locator/placement is called once and A receives no send. - -2. `AddressMessage_CacheInvalidationHeaderWithReplacementUsesReplacementRoute` - - Arrange: Bind stale route A and create an invalidation header containing valid replacement address B; configure locator cache update observation. - - Act: Call `AddressMessage`. - - Assert: The resulting destination/route is B and A is not reused. - - Secondary observable: The locator/cache records the replacement once and the retained A handle remains invalid. - -3. `AddressMessage_CustomDirectoryCacheUsesExistingPlacementPath` - - Arrange: Configure the custom directory-cache strategy and a substitute custom cache/locator with a known result; capture the supported route-entry count. - - Act: Address a message for the applicable directory grain. - - Assert: The established custom-cache/placement result is used. - - Secondary observable: The custom dependency receives the expected call and no unsupported shared fast-route entry is created. - -4. `AddressMessage_DisabledDirectoryCacheUsesExistingPlacementPath` - - Arrange: Configure caching as disabled/none and a known ordinary locator/placement result. - - Act: Address the message. - - Assert: The ordinary existing path supplies the destination. - - Secondary observable: No shared directory route is consulted or retained; supported count remains unchanged. - -5. `AddressMessage_DefaultDirectoryGrainUsesExistingPlacementPath` - - Arrange: Use a normal default-directory/DHT grain and configure its existing locator/worker result. - - Act: Address the message. - - Assert: The existing default-directory placement path produces the expected destination. - - Secondary observable: The expected locator interaction occurs once; no custom/client locator is called. - -6. `AddressMessage_ClientGrainUsesExistingPlacementPath` - - Arrange: Use a client `GrainId` and configure the existing client locator/routing result. - - Act: Address the message. - - Assert: The client-grain path produces the expected destination. - - Secondary observable: No directory-entry route is allocated and directory locator/cache calls remain absent. - -7. `SendMessage_SystemTargetUsesExistingFullyAddressedPath` - - Arrange: Create a fully addressed system-target message with a known local or remote target and record the supported directory-route count. - - Act: Send the message through the normal `MessageCenter` path. - - Assert: The existing system-target send path selects the exact addressed target without consulting or allocating a directory-entry route. - - Secondary observable: The expected direct local/remote send occurs once, directory locator/cache calls remain absent, and the supported directory-route count is unchanged. - -#### 2. MessageTargetCacheTests.cs — Growth Slice - -- **Project**: `test/Orleans.Core.Tests/Orleans.Core.Tests.csproj` -- **Test File**: `test/Orleans.Core.Tests/Runtime/MessageTargetCacheTests.cs` -- **Test Class**: `MessageTargetCacheTests` -- **Future seam/blocker**: A supported route-entry count/accessor partitioned by grain kind. Never inspect dictionary/private field names by reflection. - -1. `RouteCache_ManyClientGrainIdsDoesNotCreatePerGrainEntries` - - Arrange: Record the supported client-route and directory-route counts. Create/address a large deterministic set of unique client `GrainId` values through the public activator/client path. Include one supported directory grain as a control proving the accessor can observe an eligible entry. - - Act: Exercise route acquisition for all client IDs and read the supported counters. - - Assert: Client-grain route-entry count remains at baseline (normally zero), while the control changes only its expected supported count. - - Secondary observable: Client routing still succeeds through the existing fixed client sender-bucket/weak-connection path, demonstrating that bounded behavior is functional rather than skipped. - - Synchronization/GC: No private reflection and no GC-based size inference; use the supported accessor after all deterministic operations complete. - -### Phase 3 Success Criteria - -- [ ] Matching invalidation headers bypass stale fast routes. -- [ ] Replacement headers select and record the replacement route. -- [ ] Custom and disabled caches plus client and system-target non-directory types retain their existing behavior; the default-directory path remains a compatibility control. -- [ ] Many unique client IDs do not create per-grain route entries. -- [ ] Scoped placement, route-cache, and locator-resolver regressions pass. -- [ ] Full build, discovery, and full test commands pass. - ---- - -## Requirement-to-Test Traceability - -| Acceptance behavior | Exact project/file | Exact proposed test name(s) | Preparation status / production seam | -|---|---|---|---| -| Replacement invalidates shared handle | `Orleans.Runtime.Tests` / `Directories/GrainDirectoryCacheFactoryTests.cs` | `CreateGrainDirectoryCache_AddOrUpdateInvalidatesReplacedRouteHandle` | Handle assertion blocked; needs disposable route-owning entry | -| Remove by grain ID invalidates | Same | `CreateGrainDirectoryCache_RemoveByGrainIdInvalidatesRouteHandle` | Handle assertion blocked | -| Remove by address affects only exact match | Same | `CreateGrainDirectoryCache_RemoveByAddressInvalidatesOnlyMatchingRouteHandle` | Handle assertion blocked | -| Clear invalidates all | Same | `CreateGrainDirectoryCache_ClearInvalidatesAllRouteHandles` | Handle assertion blocked | -| TTL invalidates | Same | `CreateGrainDirectoryCache_ExpirationInvalidatesRouteHandle` | Mechanics possible now; handle blocked; use fake time/listener | -| Capacity eviction invalidates | Same | `CreateGrainDirectoryCache_EvictionInvalidatesRouteHandle` | Mechanics possible now; handle blocked; capacity 3 | -| Stale async bind cannot resurrect route | `Orleans.Core.Tests` / `Directory/CachedGrainLocatorTests.cs` | `Lookup_WhenInvalidatedBeforeAsyncLookupCompletes_DoesNotBindStaleRoute`; `Unregister_WhenLookupCompletesAfterRemoval_DoesNotResurrectRouteHandle` | Race regression possible now; generation/tombstone blocked | -| References share route state | `Orleans.Core.Tests` / `Runtime/MessageTargetCacheTests.cs` | `GrainReferences_WithSameGrainIdShareRouteHandle` | Fully blocked on shared handle API | -| Reference retains disposed tombstone | Same | `GrainReference_RetainsDisposedRouteHandleTombstone` | Fully blocked | -| Tombstone does not retain receiver | Same | `DisposedRouteHandle_DoesNotRetainLocalReceiver` | Fully blocked; needs weak retention and observable tombstone | -| Local route requires exact address and valid receiver | Same | `LocalRoute_WhenAddressMatchesAndActivationIsValidUsesReceiver`; `LocalRoute_WhenActivationAddressDiffersFallsBack`; `LocalRoute_WhenActivationIsInvalidFallsBack` | Existing primitives testable; direct route probe blocked | -| Remote route requires matching group/address and valid connection, with fallback | Same | `RemoteRoute_WhenConnectionGroupMatchesAddressUsesConnection`; `RemoteRoute_WhenConnectionGroupAddressDiffersFallsBack`; `RemoteRoute_WhenConnectionIsInvalidFallsBack` | Existing manager behavior testable; cached group route blocked | -| Invalidation header bypasses/replaces fast route | `Orleans.Core.Tests` / `Runtime/PlacementServiceTests.cs` | `AddressMessage_CacheInvalidationHeaderBypassesSharedRouteFastPath`; `AddressMessage_CacheInvalidationHeaderWithReplacementUsesReplacementRoute` | Header regression possible now; shared-route observation blocked | -| Custom/disabled caches and non-directory grain types use the existing path | `Orleans.Core.Tests` / `Runtime/PlacementServiceTests.cs` and `Runtime/MessageTargetCacheTests.cs` | `AddressMessage_CustomDirectoryCacheUsesExistingPlacementPath`; `AddressMessage_DisabledDirectoryCacheUsesExistingPlacementPath`; `AddressMessage_ClientGrainUsesExistingPlacementPath`; `SendMessage_SystemTargetUsesExistingFullyAddressedPath` | Existing selection/send behavior possible now; shared-route non-interference count blocked | -| Default-directory compatibility control retains existing path | `Orleans.Core.Tests` / `Runtime/PlacementServiceTests.cs` | `AddressMessage_DefaultDirectoryGrainUsesExistingPlacementPath` | Existing locator selection possible now; route interaction observation blocked | -| No per-client-`GrainId` route growth | `Orleans.Core.Tests` / `Runtime/MessageTargetCacheTests.cs` | `RouteCache_ManyClientGrainIdsDoesNotCreatePerGrainEntries` | Blocked on supported kind-aware count/accessor | - -## Implementation Order and Validation - -1. Confirm the landed API supplies all six required seams: observable atomic tombstone, disposable owning entry, generation-aware bind, exact local probe, group-aware remote probe, and kind-aware count/accessor. -2. Implement Phase 1 tests and run only its scoped builds/tests. -3. Implement Phase 2 tests, using the new cohesive test file, and run its scoped core build/test. -4. Implement Phase 3 tests and run placement, message-target-cache, and locator-resolver scopes. -5. Run full build, discovery, and full test only after all scoped checks pass. -6. Run the mandatory final pre-completion gate against the final test set: invoke `test-gap-analysis` for each tested source/test pair, fix every in-scope mutation gap, invoke `assertion-quality`, fix weak/trivial assertions, and manually map every prompt scenario to the exact tests in this plan. -7. If coverage-gap review adds or changes a test, repeat the complete pre-completion gate. -8. Write `.testagent/status.md` only after implementation, quality review, and validation, recording checklist evidence, commands/results, quality findings/fixes, and blockers. -9. Preserve repository xUnit v3/MTP metadata: `[Fact]`/`[Theory]`, class-level `[TestSuite("BVT")]`, `[TestProvider("None")]`, appropriate `[TestCategory]` and `[TestArea]`, async `Task`, and focused assertions. diff --git a/.testagent/research.md b/.testagent/research.md deleted file mode 100644 index fdbbd7696e7..00000000000 --- a/.testagent/research.md +++ /dev/null @@ -1,147 +0,0 @@ -# Test Generation Research - -> Final scope update: cached targets are limited to local `IGrainContext` instances and remote `ConnectionManager.ConnectionEntry` groups. Remote references retain the shared directory entry for address reuse; response-route caching remains a separate experiment. - -## Project Overview -- **Path**: `C:\dev\copilot-worktrees\orleans\rb-scaling-guacamole` -- **Boundary**: Orleans runtime directory caching, silo/client message routing, local activation lookup, connection selection, and grain-reference creation. Samples, providers, streaming, persistence, and unrelated tests are out of scope. -- **Language / framework**: C# on SDK-style .NET; source targets `net8.0;net10.0`, and `global.json` selects SDK `10.0.400` with major roll-forward. -- **Test framework**: xUnit v3 `3.2.2` hosted by Microsoft Testing Platform (MTP) `2.3.3`. -- **Dependencies**: Central `PackageReference` versions in `Directory.Packages.props`; NSubstitute `5.3.0`, NSubstitute analyzer `1.0.17`, Microsoft.Extensions.TimeProvider.Testing `9.10.0`, and AwesomeAssertions `9.3.0`. -- **New-file registration**: SDK implicit `Compile` glob. A future `*.cs` test file needs no project edit. -- **Current state**: The shared directory-entry message-target/route-handle API does not exist. `GrainReference` currently stores only `GrainReferenceShared` and `IdSpan`; `IGrainDirectoryCache` stores `GrainAddress`; `PlacementService` resolves a silo for every not-fully-addressed message. - -## Static Pairing Heuristic -- The required Roslyn analyzer was executed **exactly once**, at the repository root. This is the narrowest common root because the bounded production files are under `src/` and their tests are under `test/`. -- Analyzer diagnostics: 3,844 C# files discovered; 2,837 source and 1,007 test files; indexing/scanning completed in 3.192 seconds. -- The file-based SDK emitted a leading `C...` build line into captured stdout, so `ConvertFrom-Json` rejected the combined stream before the relevant `source_to_tests` and `suggested_test_path` nodes could be extracted. It was not rerun, honoring the exactly-once constraint. -- Consequently, the pairings below are direct, bounded source/test references and naming conventions, **not recovered analyzer pairings**. The analyzer is a static identifier-pairing heuristic in any case, not line or branch coverage. - -## Dependency Graph and Production Seams -- **Leaf types**: - - `src/Orleans.Runtime/Catalog/ActivationDirectory.cs` — grain-id-to-context table; exact-instance conditional removal. - - `src/Orleans.Core.Abstractions/Runtime/GrainReference.cs` — reference identity and invocation surface. - - `src/Orleans.Runtime/GrainDirectory/IGrainDirectoryCache.cs` — current address-only cache contract. - - `src/Orleans.Core/Networking/Connection.cs` — exposes `IsValid`. -- **Mid-layer types**: - - `src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs` — wraps `ConcurrentLruCache`; replacement/removal/clear/expiry/eviction dispose cache values only when the value itself implements `IDisposable`. - - `src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheFactory.cs` and `Configuration/Options/GrainDirectoryOptions.cs` — LRU/custom/none selection and keyed `TimeProvider`. - - `src/Orleans.Runtime/GrainDirectory/CachedGrainLocator.cs` — async custom-directory lookup and cache population/invalidation. - - `src/Orleans.Runtime/GrainDirectory/ClientGrainLocator.cs`, `GrainLocatorResolver.cs`, and `GrainLocator.cs` — client/DHT/custom-directory path selection. - - `src/Orleans.Core/Networking/ConnectionManager.cs` — per-silo connection groups; `NextConnection()` validates the selected connection and the slow path removes defunct connections. - - `src/Orleans.Core/GrainReferences/GrainReferenceActivator.cs` — bounded today by `(GrainType, GrainInterfaceType)`, not `GrainId`. -- **Top-layer types**: - - `src/Orleans.Runtime/Placement/PlacementService.cs` — current cache fast path and cache-invalidation-header validation. - - `src/Orleans.Runtime/Messaging/MessageCenter.cs` — local loopback, remote send, fallback connection acquisition, and address/send sequencing. - - `src/Orleans.Runtime/Core/InsideRuntimeClient.cs` — applies invalidation headers before response handling. - - `src/Orleans.Core/Messaging/ClientMessageCenter.cs` and `Runtime/OutsideRuntimeClient.cs` — client path; routing is a fixed-size array of weak connection references, not a per-`GrainId` dictionary. - - `src/Orleans.Runtime/Catalog/Catalog.cs`, `ActivationData.cs`, and `src/Orleans.Core.Abstractions/Core/IGrainContext.cs` — local receiver address and validity. -- **Required future seams**: - 1. An observable shared route-handle/tombstone with atomic bind/invalidate state. - 2. A disposable directory-cache entry which owns that handle, so every LRU removal path invalidates it. - 3. Generation/token-aware binding so an await continuation cannot bind a disposed or superseded handle. - 4. A local-route probe which validates both exact `GrainAddress` and receiver validity without strongly retaining the receiver after invalidation. - 5. A remote-route probe which validates destination `SiloAddress`, connection-group identity, and connection validity, then exposes the ordinary fallback path. - 6. A test accessor/counter for route entries by grain kind; do not test private field names by reflection. - -## Build & Test Commands -Run from the repository root. `--no-restore` preserves the no-restore constraint once dependencies are already available. - -- **Build (scoped)**: - - `dotnet build test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore` - - `dotnet build test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj --framework net10.0 --no-restore` -- **Build (full)**: `dotnet build Orleans.slnx --no-restore --no-incremental -bl` -- **Test (scoped — future core route tests)**: `dotnet test --project test/Orleans.Core.Tests/Orleans.Core.Tests.csproj --framework net10.0 --no-restore --filter-class "*MessageTargetCacheTests*" --minimum-expected-tests 1` -- **Test (scoped — existing cache file)**: `dotnet test --project test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj --framework net10.0 --no-restore --filter-class "*GrainDirectoryCacheFactoryTests*" --minimum-expected-tests 1` -- **Test (scoped — current regression files)**: substitute one of `*PlacementServiceTests*`, `*CachedGrainLocatorTests*`, or `*GrainLocatorResolverTests*` for `*MessageTargetCacheTests*` in the core command. -- **Test (full, all target frameworks with a fresh build)**: `dotnet test --solution Orleans.slnx --no-restore --minimum-expected-tests 1 --max-parallel-test-modules 1` -- **Harness-equivalent discovery (all target frameworks)**: `dotnet test --solution Orleans.slnx --no-restore --list-tests --minimum-expected-tests 1 --max-parallel-test-modules 1` -- **Lint**: no separate command; `dotnet build` enforces style and treats warnings/nullable warnings as errors. - -## Exact Test Targets - -### Projects -1. `test/Orleans.Runtime.Tests/Orleans.Runtime.Tests.csproj` - - Target: `src/Orleans.Runtime/Orleans.Runtime.csproj`. - - Canonical file: `test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs`. -2. `test/Orleans.Core.Tests/Orleans.Core.Tests.csproj` - - Targets: `src/Orleans.Core/Orleans.Core.csproj`, `src/Orleans.Core.Abstractions/Orleans.Core.Abstractions.csproj`, and `src/Orleans.Runtime/Orleans.Runtime.csproj`. - - Canonical files: `Directory/CachedGrainLocatorTests.cs`, `Directory/GrainLocatorResolverTests.cs`, and `Runtime/PlacementServiceTests.cs`. - - One cohesive **future** file, added only after the API exists: `test/Orleans.Core.Tests/Runtime/MessageTargetCacheTests.cs`. - -### Exact proposed test names - -`test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs`: -- `CreateGrainDirectoryCache_AddOrUpdateInvalidatesReplacedRouteHandle` -- `CreateGrainDirectoryCache_RemoveByGrainIdInvalidatesRouteHandle` -- `CreateGrainDirectoryCache_RemoveByAddressInvalidatesOnlyMatchingRouteHandle` -- `CreateGrainDirectoryCache_ClearInvalidatesAllRouteHandles` -- `CreateGrainDirectoryCache_ExpirationInvalidatesRouteHandle` -- `CreateGrainDirectoryCache_EvictionInvalidatesRouteHandle` - -`test/Orleans.Core.Tests/Directory/CachedGrainLocatorTests.cs`: -- `Lookup_WhenInvalidatedBeforeAsyncLookupCompletes_DoesNotBindStaleRoute` -- `Unregister_WhenLookupCompletesAfterRemoval_DoesNotResurrectRouteHandle` - -`test/Orleans.Core.Tests/Runtime/PlacementServiceTests.cs`: -- `AddressMessage_CacheInvalidationHeaderBypassesSharedRouteFastPath` -- `AddressMessage_CacheInvalidationHeaderWithReplacementUsesReplacementRoute` -- `AddressMessage_CustomDirectoryCacheUsesExistingPlacementPath` -- `AddressMessage_DisabledDirectoryCacheUsesExistingPlacementPath` -- `AddressMessage_DefaultDirectoryGrainUsesExistingPlacementPath` -- `AddressMessage_ClientGrainUsesExistingPlacementPath` -- `SendMessage_SystemTargetUsesExistingFullyAddressedPath` - -`test/Orleans.Core.Tests/Runtime/MessageTargetCacheTests.cs` (future file): -- `GrainReferences_WithSameGrainIdShareRouteHandle` -- `GrainReference_RetainsDisposedRouteHandleTombstone` -- `DisposedRouteHandle_DoesNotRetainLocalReceiver` -- `LocalRoute_WhenAddressMatchesAndActivationIsValidUsesReceiver` -- `LocalRoute_WhenActivationAddressDiffersFallsBack` -- `LocalRoute_WhenActivationIsInvalidFallsBack` -- `RemoteRoute_WhenConnectionGroupMatchesAddressUsesConnection` -- `RemoteRoute_WhenConnectionGroupAddressDiffersFallsBack` -- `RemoteRoute_WhenConnectionIsInvalidFallsBack` -- `RouteCache_ManyClientGrainIdsDoesNotCreatePerGrainEntries` - -## Acceptance Checklist and Blockers -1. **Replacement/removal/clear/TTL/eviction invalidates a shared route handle** — **blocked on future API** for the handle assertion. Cache mechanics are regression-testable now: factory TTL coverage exists, and `ConcurrentLruCache` already disposes `IDisposable` values on update/remove/clear/expiry/eviction. Use `FakeTimeProvider` plus `ConcurrentLruCacheExpirationCleanupListener`; capacity `3` is the minimum deterministic eviction fixture. -2. **Stale asynchronous binding cannot resurrect invalid entries** — **partly regression-testable now** via `CachedGrainLocator` lookup/unregister races; handle generation/tombstone assertions are blocked. Existing `UnregisterRacesWithLookupSameId` is the closest regression. -3. **Grain references retain disposed tombstones without retaining receivers** — **fully blocked**. There is no route field/tombstone today. Use a `[MethodImpl(NoInlining)]` setup helper, `WeakReference`, and bounded collect/finalize/collect attempts, following `ProviderRegistrationResolverTests.GeneratedRegistry_DoesNotRootCollectibleAssemblyLoadContext`. -4. **Local activation exact-address/state validation** — exact-instance removal and validity are regression-testable in `ActivationDirectory`/`LocalActivationStatusChecker`; direct-route validation against `IGrainContext.Address` and `ICollectibleGrainContext.IsValid` is **blocked**. -5. **Remote connection-group route validation and fallback** — current `ConnectionManager` valid/defunct selection and `MessageCenter` fallback are regression-testable; cached route/group identity is **blocked**. -6. **Cache invalidation headers bypass the fast path** — **regression-testable now**. `PlacementService.CachedAddressIsValid` updates/invalidates the locator and rejects a matching invalid address before worker lookup. Add the named tests to the existing `PlacementServiceFixture`. -7. **Custom/disabled caches and non-directory grain types retain the existing path** — **regression-testable now** for factory and locator selection; future shared-route non-interference needs the new API. Treat client grains and system targets as the explicit non-directory cases. Existing coverage includes custom/none factory tests and DHT/client/custom locator resolver tests; add the named system-target send guard after the new API lands. -8. **The original unbounded client `GrainId` mapping does not return** — current code already has no such map: `GrainReferenceActivator` keys by type/interface and `ClientMessageCenter` uses a fixed `ClientSenderBuckets` array of weak references. A durable count-based guard is **blocked** pending a route-cache test accessor; avoid private-field reflection. -9. **No production modifications and no test source additions during preparation** — satisfied. This turn creates only `.testagent/research.md`. -10. **Exact proposed names and target files/projects** — supplied in “Exact Test Targets”. - -## Existing Tests & Coverage Classification -- `GrainDirectoryCacheFactory.cs` / `LruGrainDirectoryCache.cs` → `test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs`: **partial** for the forthcoming scope. It covers TTL, ownership, custom/none selection, and disposal policy, but no shared route exists. -- `CachedGrainLocator.cs` → `test/Orleans.Core.Tests/Directory/CachedGrainLocatorTests.cs`: **substantial current behavior**, including cache population, dead silos, unregister-first, and races; no route binding. -- `GrainLocatorResolver.cs` / client-DHT-custom selection → `test/Orleans.Core.Tests/Directory/GrainLocatorResolverTests.cs`: **substantial for selection**, only three focused tests. -- `PlacementService.cs` → `test/Orleans.Core.Tests/Runtime/PlacementServiceTests.cs`: **partial**. Strong lifecycle/version/placement fixture, but only the stopped-state test currently exercises `AddressMessage`; no invalidation-header fast-path test. -- `ActivationDirectory.cs` / local receiver validity → indirect uses in `CachedGrainLocatorTests.cs` and `Membership/MembershipSystemTargetTests.cs`: **partial**, no dedicated exact-address/state route tests. -- `ConnectionManager.cs` / `MessageCenter.cs`: **partial integration coverage**, no directly paired connection-group route test found in the bounded test set. -- `GrainReference.cs` / `GrainReferenceActivator.cs`: existing default-cluster reference/cast tests cover identity and casting, but forthcoming route retention is **untested/absent**. -- `ClientMessageCenter.cs`: **partial existing behavior**, with no dedicated bounded test file for per-grain route growth. -- These classifications are structural, not numeric coverage measurements. - -## Testing Patterns and Helpers -- Representative conventions: - - `test/Orleans.Core.Tests/Runtime/PlacementServiceTests.cs` - - `test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs` -- Use xUnit `[Fact]`/`[Theory]`, `Assert.*`, async `Task`, and class-level `[TestSuite("BVT")]`, `[TestProvider("None")]`, `[TestCategory("BVT"), TestCategory("Directory" or "Placement")]`, plus an appropriate `[TestArea]`. -- Prefer deterministic `TaskCompletionSource(...RunContinuationsAsynchronously)`, `FakeTimeProvider`, explicit lifecycle stop, and `try/finally` disposal. -- Reuse `PlacementServiceFixture`, `MockClusterMembershipService`, `SiloLifecycleSubject`, NSubstitute, and `ConcurrentLruCacheExpirationCleanupListener`. -- Assert both the selected result and the bypass/fallback interaction (for example, no stale receiver send and one ordinary locator/connection lookup). Do not rely on timing delays or inspect private fields. - -## Priority -1. After the API lands, implement route-handle ownership/invalidation and stale-bind tests first; they protect lifetime and race correctness. -2. Add local/remote route validation and GC-retention tests. -3. Add header-bypass and compatibility-path regressions. -4. Add the client growth guard using a supported internal test accessor. - -## Preparation Constraints -- No build or test was run because the production API is absent and this was planning-only. -- No restore, clean, reset, stash, deletion, production edit, or test-source addition was performed. diff --git a/.testagent/status.md b/.testagent/status.md deleted file mode 100644 index 43d4aadc697..00000000000 --- a/.testagent/status.md +++ /dev/null @@ -1,30 +0,0 @@ -# Test Quality Status - -## Scope - -The shared directory-entry message target cache is covered by: - -- `test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs` -- `test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs` - -## Results - -- 29 focused tests pass on `net10.0`. -- No generated test is assertion-free or relies only on presence/truthiness checks. -- Assertions cover equality, identity, type, null, negative behavior, collection state, lifecycle state, and side effects. -- Cache lifetime tests verify in-place replacement, conditional removal, clear, TTL expiration, eviction, tombstoning, target release, one-shot binding, and identity-checked clearing. -- Integration tests verify local activation binding, remote connection-group binding, invalidation and recapture, compatible-interface casts, and explicit external-client and stateless-worker exclusion. - -## Pseudo-Mutation Review - -Three high-risk mutations were injected individually and empirically killed by the focused tests: - -1. Breaking the atomic first-target bind caused six cache lifetime tests to fail. -2. Removing the directory-backed placement guard caused `StatelessWorkerCalls_DoNotAttachDirectoryEntries` to fail. -3. Allowing messages with cache invalidation headers onto the fast path caused `CacheInvalidationHeader_BypassesFastPathWithoutDiscardingLiveHandle` to fail. - -Every mutation was reverted immediately. The focused suites were rerun green afterward. - -## Remaining Risk - -The tests intentionally allow the same stale routing race as the existing directory and activation lookups: invalidation can race after a valid entry is read. Entry invalidation, activation state checks, membership validation, and existing forwarding semantics preserve recovery. Remote connections and response routing remain on the established path. From 471ba704ccf2ce53467bc3c9628ea37f8a913900 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 16:58:56 -0700 Subject: [PATCH 09/21] fix(runtime): prevent stale target binding during cache updates --- .../Caching/ConcurrentLruCache.cs | 7 +-- .../GrainDirectoryCacheEntry.cs | 49 +++++++++++++++++-- .../GrainDirectory/LruGrainDirectoryCache.cs | 4 +- .../GrainDirectoryCacheFactoryTests.cs | 34 +++++++++++++ 4 files changed, 84 insertions(+), 10 deletions(-) diff --git a/src/Orleans.Core/Caching/ConcurrentLruCache.cs b/src/Orleans.Core/Caching/ConcurrentLruCache.cs index 9f5fa8a80a4..fdec41202bf 100644 --- a/src/Orleans.Core/Caching/ConcurrentLruCache.cs +++ b/src/Orleans.Core/Caching/ConcurrentLruCache.cs @@ -64,9 +64,7 @@ public ConcurrentLruCache(int capacity) : this(capacity, comparer: null) { } - protected virtual void OnItemUpdating(LruItem item) - { - } + protected virtual void UpdateItem(LruItem item, V value) => item.Value = value; protected virtual void OnItemRemoved(LruItem item) { @@ -370,8 +368,7 @@ public bool TryUpdate(K key, V value) { var oldValue = existing.Value; - OnItemUpdating(existing); - existing.Value = value; + UpdateItem(existing, value); UpdateTimestamp(existing); _telemetryPolicy.IncrementUpdated(); diff --git a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs index 941f996563a..9bf836e95c6 100644 --- a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs +++ b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs @@ -8,6 +8,7 @@ internal sealed class GrainDirectoryCacheEntry IDisposable { private static readonly object Invalidated = new(); + private static readonly object Updating = new(); private object? _messageTarget; public GrainDirectoryCacheEntry(GrainAddress address, int version) @@ -27,12 +28,19 @@ public GrainDirectoryCacheEntry( public int Version => Value.Version; - public bool IsValid => !ReferenceEquals(Volatile.Read(ref _messageTarget), Invalidated); + public bool IsValid + { + get + { + var target = Volatile.Read(ref _messageTarget); + return !ReferenceEquals(target, Invalidated) && !ReferenceEquals(target, Updating); + } + } public bool TryGetMessageTarget(out object? messageTarget) { var target = Volatile.Read(ref _messageTarget); - if (ReferenceEquals(target, Invalidated)) + if (ReferenceEquals(target, Invalidated) || ReferenceEquals(target, Updating)) { messageTarget = null; return false; @@ -81,7 +89,7 @@ public bool TrySetMessageTarget(object messageTarget, SiloAddress expectedSilo) private bool TrySetMessageTargetCore(object messageTarget) { var current = Volatile.Read(ref _messageTarget); - if (ReferenceEquals(current, Invalidated)) + if (ReferenceEquals(current, Invalidated) || ReferenceEquals(current, Updating)) { return false; } @@ -100,6 +108,22 @@ public void ClearMessageTarget(object messageTarget) public void Dispose() => Invalidate(); + internal void Update((GrainAddress ActivationAddress, int Version) value) + { + var updateStarted = TryBeginUpdate(); + try + { + Value = value; + } + finally + { + if (updateStarted) + { + EndUpdate(); + } + } + } + public void ClearMessageTarget() { while (true) @@ -116,4 +140,23 @@ public void ClearMessageTarget() } } } + + internal bool TryBeginUpdate() + { + while (true) + { + var current = Volatile.Read(ref _messageTarget); + if (ReferenceEquals(current, Invalidated)) + { + return false; + } + + if (ReferenceEquals(Interlocked.CompareExchange(ref _messageTarget, Updating, current), current)) + { + return true; + } + } + } + + internal void EndUpdate() => Interlocked.CompareExchange(ref _messageTarget, null, Updating); } diff --git a/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs b/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs index ead830e7aa9..a1e584f0fb9 100644 --- a/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs +++ b/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs @@ -75,8 +75,8 @@ public bool TryGetEntry(GrainId key, [NotNullWhen(true)] out GrainDirectoryCache protected override LruItem CreateItem(GrainId key, (GrainAddress ActivationAddress, int Version) value) => new GrainDirectoryCacheEntry(key, value, GetCurrentTimestamp()); - protected override void OnItemUpdating(LruItem item) - => ((GrainDirectoryCacheEntry)item).ClearMessageTarget(); + protected override void UpdateItem(LruItem item, (GrainAddress ActivationAddress, int Version) value) + => ((GrainDirectoryCacheEntry)item).Update(value); protected override void OnItemRemoved(LruItem item) => ((GrainDirectoryCacheEntry)item).Invalidate(); diff --git a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs index 37519a576c1..fa3a758add2 100644 --- a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs +++ b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs @@ -564,6 +564,40 @@ public async Task GrainDirectoryCacheEntry_StaleAddressCannotRebindAfterUpdate() } } + [Fact] + public void GrainDirectoryCacheEntry_UpdateBlocksBindingUntilNewAddressIsPublished() + { + var grainId = CreateGrainId(); + var originalAddress = CreateGrainAddress(grainId, port: 11111); + var replacementAddress = CreateGrainAddress(grainId, port: 22222); + var entry = new GrainDirectoryCacheEntry(originalAddress, version: 1); + var originalTarget = CreateMessageTarget(); + + Assert.True(entry.TrySetMessageTarget(originalTarget, originalAddress)); + + Assert.True(entry.TryBeginUpdate()); + Assert.False(entry.IsValid); + Assert.False(entry.TryGetMessageTarget(out _)); + Assert.False(entry.TrySetMessageTarget(CreateMessageTarget(), originalAddress)); + + entry.Value = (replacementAddress, 2); + + Assert.False(entry.IsValid); + Assert.False(entry.TrySetMessageTarget(CreateMessageTarget(), replacementAddress)); + + entry.EndUpdate(); + + Assert.True(entry.IsValid); + Assert.Equal(replacementAddress, entry.Address); + Assert.Equal(2, entry.Version); + Assert.False(entry.TryGetMessageTarget(out _)); + Assert.False(entry.TrySetMessageTarget(CreateMessageTarget(), originalAddress)); + + var replacementTarget = CreateMessageTarget(); + Assert.True(entry.TrySetMessageTarget(replacementTarget, replacementAddress)); + AssertMessageTarget(entry, replacementTarget); + } + [Fact] public async Task CreateGrainDirectoryCache_RemoveByGrainIdReleasesMessageTargetReference() { From 086c2ed6d85c2deb9838c48f57b70bc617f4642a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 16:58:56 -0700 Subject: [PATCH 10/21] perf(benchmarks): strengthen deterministic ping measurements --- .../Ping/AdaptiveConcurrencyLoadGenerator.cs | 155 ++++++++++++++---- test/Benchmarks/Ping/AdaptivePingBenchmark.cs | 71 ++++++-- 2 files changed, 181 insertions(+), 45 deletions(-) diff --git a/test/Benchmarks/Ping/AdaptiveConcurrencyLoadGenerator.cs b/test/Benchmarks/Ping/AdaptiveConcurrencyLoadGenerator.cs index bc19095f560..4faeba716c6 100644 --- a/test/Benchmarks/Ping/AdaptiveConcurrencyLoadGenerator.cs +++ b/test/Benchmarks/Ping/AdaptiveConcurrencyLoadGenerator.cs @@ -159,32 +159,52 @@ public async Task RunForeverAsync(CancellationToken cancellationToken = default) } - public async Task RunFixedConcurrencyAsync(int repetitions, CancellationToken cancellationToken = default) + public async Task RunFixedConcurrencyAsync( + int repetitions, + CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfLessThan(repetitions, 1); + _cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var states = Enumerable.Range(0, _currentConcurrency).Select(_getStateForWorker).ToArray(); + + try { - ArgumentOutOfRangeException.ThrowIfLessThan(repetitions, 1); - _cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + await RunPhaseAsync(_warmupDuration, isWarmup: true, states); + GC.Collect(); - try + var results = new FixedConcurrencyMeasurement[repetitions]; + for (var i = 0; i < results.Length; i++) { - await RunPhaseAsync(_warmupDuration, isWarmup: true); - GC.Collect(); - - var results = new double[repetitions]; - for (var i = 0; i < results.Length; i++) + var measurement = await RunPhaseAsync(_measurementInterval, isWarmup: false, states); + if (measurement.Failures > 0) { - results[i] = (await RunPhaseAsync(_measurementInterval, isWarmup: false)).Throughput; + throw new InvalidOperationException($"Measurement completed with {measurement.Failures:N0} failed requests."); } - return results; - } - finally - { - await _cts.CancelAsync(); - _cts.Dispose(); + results[i] = new( + measurement.Throughput, + measurement.LatencyP50Microseconds, + measurement.LatencyP95Microseconds, + measurement.LatencyP99Microseconds, + measurement.AllocatedBytesPerRequest, + measurement.Gen0CollectionsPerMillionRequests, + measurement.CpuUtilization, + measurement.LockContentionsPerMillionRequests); } + + return results; + } + finally + { + await _cts.CancelAsync(); + _cts.Dispose(); } + } - private async Task RunPhaseAsync(TimeSpan duration, bool isWarmup) + private async Task RunPhaseAsync( + TimeSpan duration, + bool isWarmup, + IReadOnlyList? workerStates = null) { _completedBlocks = Channel.CreateUnbounded( new UnboundedChannelOptions @@ -197,18 +217,15 @@ private async Task RunPhaseAsync(TimeSpan duration, bool isWarmup) var workerTasks = new List(); // Link to main cancellation token so Ctrl+C stops workers immediately using var workerCts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token); - var states = new Dictionary(); + var aggregator = Task.Run(() => AggregateBlocksAsync(duration, isWarmup, workerCts)); // Start initial workers for (int i = 0; i < _currentConcurrency; i++) { - var state = _getStateForWorker(i); - states[i] = state; + var state = workerStates is null ? _getStateForWorker(i) : workerStates[i]; workerTasks.Add(RunWorkerAsync(state, i, workerCts.Token)); } - var aggregator = Task.Run(() => AggregateBlocksAsync(duration, isWarmup, workerCts)); - var measurement = await aggregator; // Signal workers to stop @@ -246,20 +263,30 @@ private async Task AggregateBlocksAsync(TimeSpan duration, bool isW completionsBySample.Add(0); } - long totalCompleted = 0; + long totalSuccessful = 0; long totalFailures = 0; long minStartTime = long.MaxValue; long maxEndTime = long.MinValue; + var latencySamples = new List(); + var allocatedBytesBefore = GC.GetTotalAllocatedBytes(precise: false); + var gen0CollectionsBefore = GC.CollectionCount(0); + var lockContentionsBefore = Monitor.LockContentionCount; + using var process = Process.GetCurrentProcess(); + var processorTimeBefore = process.TotalProcessorTime; void RecordBlock(WorkBlock block) { - totalCompleted += block.Completed; + totalSuccessful += block.Successes; totalFailures += block.Failures; if (block.StartTimestamp < minStartTime) minStartTime = block.StartTimestamp; if (block.EndTimestamp > maxEndTime) maxEndTime = block.EndTimestamp; + if (block.FirstRequestEndTimestamp > block.StartTimestamp) + { + latencySamples.Add(block.FirstRequestEndTimestamp - block.StartTimestamp); + } var sampleIndex = GetSampleIndex(block.EndTimestamp); - completionsBySample[sampleIndex] += block.Completed; + completionsBySample[sampleIndex] += block.Successes; } int GetSampleIndex(long timestamp) @@ -312,18 +339,32 @@ int GetSampleIndex(long timestamp) RecordBlock(block); } - var totalSeconds = totalCompleted == 0 || maxEndTime <= minStartTime + var totalSeconds = totalSuccessful == 0 || maxEndTime <= minStartTime ? duration.TotalSeconds : (maxEndTime - minStartTime) / StopwatchTickPerSecond; - var throughput = totalSeconds > 0 ? totalCompleted / totalSeconds : 0; + var throughput = totalSeconds > 0 ? totalSuccessful / totalSeconds : 0; var sampleEndTime = maxEndTime > endTime ? maxEndTime : endTime; var samples = CreateThroughputSamples(completionsBySample, startTime, sampleEndTime, sampleTicks); - var measurement = new Measurement(throughput, samples); + var allocatedBytes = GC.GetTotalAllocatedBytes(precise: false) - allocatedBytesBefore; + var gen0Collections = GC.CollectionCount(0) - gen0CollectionsBefore; + var lockContentions = Monitor.LockContentionCount - lockContentionsBefore; + var processorTime = process.TotalProcessorTime - processorTimeBefore; + var measurement = new Measurement( + throughput, + samples, + totalFailures, + GetPercentileMicroseconds(latencySamples, 0.50), + GetPercentileMicroseconds(latencySamples, 0.95), + GetPercentileMicroseconds(latencySamples, 0.99), + totalSuccessful == 0 ? 0 : allocatedBytes / (double)totalSuccessful, + totalSuccessful == 0 ? 0 : gen0Collections * 1_000_000d / totalSuccessful, + totalSeconds <= 0 ? 0 : processorTime.TotalSeconds / (totalSeconds * Environment.ProcessorCount) * 100, + totalSuccessful == 0 ? 0 : lockContentions * 1_000_000d / totalSuccessful); if (isWarmup) { var failureInfo = totalFailures > 0 ? $" ({totalFailures} failures)" : ""; - Console.WriteLine($" Warmup: {throughput:N0}/s, {totalCompleted:N0} requests in {totalSeconds:F1}s{failureInfo}"); + Console.WriteLine($" Warmup: {throughput:N0}/s, {totalSuccessful:N0} requests in {totalSeconds:F1}s{failureInfo}"); } return measurement; @@ -443,6 +484,18 @@ private static double[] CreateThroughputSamples(IReadOnlyList completionsB return samples; } + private static double GetPercentileMicroseconds(List samples, double percentile) + { + if (samples.Count == 0) + { + return 0; + } + + samples.Sort(); + var index = (int)Math.Ceiling(percentile * samples.Count) - 1; + return samples[Math.Clamp(index, 0, samples.Count - 1)] * 1_000_000d / StopwatchTickPerSecond; + } + private bool IsStatisticallySignificantImprovement(Measurement candidate, Measurement baseline) { if (!baseline.HasValue) @@ -526,14 +579,42 @@ private static double GetTwoSidedTCriticalValue95(double degreesOfFreedom) }; } + public readonly record struct FixedConcurrencyMeasurement( + double Throughput, + double LatencyP50Microseconds, + double LatencyP95Microseconds, + double LatencyP99Microseconds, + double AllocatedBytesPerRequest, + double Gen0CollectionsPerMillionRequests, + double CpuUtilization, + double LockContentionsPerMillionRequests); + private readonly struct Measurement { - public Measurement(double throughput, double[] samples) + public Measurement( + double throughput, + double[] samples, + long failures, + double latencyP50Microseconds, + double latencyP95Microseconds, + double latencyP99Microseconds, + double allocatedBytesPerRequest, + double gen0CollectionsPerMillionRequests, + double cpuUtilization, + double lockContentionsPerMillionRequests) { Throughput = throughput; SampleCount = samples.Length; SampleMean = SampleCount == 0 ? throughput : samples.Average(); SampleVariance = CalculateSampleVariance(samples, SampleMean); + Failures = failures; + LatencyP50Microseconds = latencyP50Microseconds; + LatencyP95Microseconds = latencyP95Microseconds; + LatencyP99Microseconds = latencyP99Microseconds; + AllocatedBytesPerRequest = allocatedBytesPerRequest; + Gen0CollectionsPerMillionRequests = gen0CollectionsPerMillionRequests; + CpuUtilization = cpuUtilization; + LockContentionsPerMillionRequests = lockContentionsPerMillionRequests; HasValue = true; } @@ -542,6 +623,14 @@ public Measurement(double throughput, double[] samples) public int SampleCount { get; } public double SampleMean { get; } public double SampleVariance { get; } + public long Failures { get; } + public double LatencyP50Microseconds { get; } + public double LatencyP95Microseconds { get; } + public double LatencyP99Microseconds { get; } + public double AllocatedBytesPerRequest { get; } + public double Gen0CollectionsPerMillionRequests { get; } + public double CpuUtilization { get; } + public double LockContentionsPerMillionRequests { get; } private static double CalculateSampleVariance(double[] samples, double mean) { @@ -584,6 +673,11 @@ private async Task RunWorkerAsync(TState state, int workerId, CancellationToken { workBlock.Failures++; } + + if (workBlock.Completed == 1) + { + workBlock.FirstRequestEndTimestamp = Stopwatch.GetTimestamp(); + } } workBlock.EndTimestamp = Stopwatch.GetTimestamp(); @@ -609,6 +703,7 @@ private async Task RunWorkerAsync(TState state, int workerId, CancellationToken private struct WorkBlock { public long StartTimestamp; + public long FirstRequestEndTimestamp; public long EndTimestamp; public int Successes; public int Failures; diff --git a/test/Benchmarks/Ping/AdaptivePingBenchmark.cs b/test/Benchmarks/Ping/AdaptivePingBenchmark.cs index 9b2fada0c47..72bb0f876c0 100644 --- a/test/Benchmarks/Ping/AdaptivePingBenchmark.cs +++ b/test/Benchmarks/Ping/AdaptivePingBenchmark.cs @@ -271,7 +271,7 @@ public static async Task RunAllScenariosAsync(int maxStableRounds = DefaultMaxSt } public static async Task RunDeterministicMatrixAsync( - int repetitions = 3, + int repetitions = 5, TimeSpan? warmupDuration = null, TimeSpan? measurementInterval = null) { @@ -298,7 +298,7 @@ public static async Task RunDeterministicMatrixAsync( public static async Task RunDeterministicScenarioAsync( BenchmarkMode mode, int numSilos, - int repetitions = 3, + int repetitions = 5, TimeSpan? warmupDuration = null, TimeSpan? measurementInterval = null) { @@ -313,7 +313,7 @@ private static async Task> MeasureDeterministicScenari TimeSpan? warmupDuration, TimeSpan? measurementInterval) { - int[] concurrencyLevels = [100, 250, 500]; + int[] concurrencyLevels = [1, 16, 100, 250, 500]; var results = new List(concurrencyLevels.Length); using var benchmark = new AdaptivePingBenchmark(mode, numSilos); try @@ -333,13 +333,26 @@ private static async Task> MeasureDeterministicScenari initialStepSize: 1, sampleInterval: DefaultSampleInterval, minimumRelativeImprovement: 0); - var samples = await loadGenerator.RunFixedConcurrencyAsync(repetitions); - Array.Sort(samples); - var median = samples.Length % 2 == 0 - ? (samples[(samples.Length / 2) - 1] + samples[samples.Length / 2]) / 2 - : samples[samples.Length / 2]; - results.Add(new(benchmark.Description, concurrency, median, samples[^1])); - Console.WriteLine($"{benchmark.Description}, concurrency {concurrency}: median {median:N0}/s, max {samples[^1]:N0}/s"); + var measurements = await loadGenerator.RunFixedConcurrencyAsync(repetitions); + var throughput = measurements.Select(static measurement => measurement.Throughput).Order().ToArray(); + var result = new DeterministicResult( + benchmark.Description, + concurrency, + GetPercentile(throughput, 0.50), + GetPercentile(throughput, 0.95), + GetPercentile(throughput, 0.99), + GetMedian(measurements, static measurement => measurement.LatencyP50Microseconds), + GetMedian(measurements, static measurement => measurement.LatencyP95Microseconds), + GetMedian(measurements, static measurement => measurement.LatencyP99Microseconds), + GetMedian(measurements, static measurement => measurement.AllocatedBytesPerRequest), + GetMedian(measurements, static measurement => measurement.Gen0CollectionsPerMillionRequests), + GetMedian(measurements, static measurement => measurement.CpuUtilization), + GetMedian(measurements, static measurement => measurement.LockContentionsPerMillionRequests)); + results.Add(result); + Console.WriteLine( + $"{benchmark.Description}, concurrency {concurrency}: " + + $"P50 {result.P50Throughput:N0}/s, P95 {result.P95Throughput:N0}/s, " + + $"latency {result.LatencyP50Microseconds:N1}/{result.LatencyP95Microseconds:N1}/{result.LatencyP99Microseconds:N1} us"); } } finally @@ -355,19 +368,47 @@ private static void PrintDeterministicResults(List results) Console.WriteLine(); Console.WriteLine("## Deterministic Ping Benchmark Results"); Console.WriteLine(); - Console.WriteLine("| Scenario | Concurrency | Median Throughput | Maximum Throughput |"); - Console.WriteLine("|----------|------------:|------------------:|-------------------:|"); + Console.WriteLine("| Scenario | Concurrency | P50 throughput | P95 throughput | P99 throughput | Latency P50/P95/P99 (us) | B/request | Gen0/M requests | CPU | Contentions/M requests |"); + Console.WriteLine("|----------|------------:|---------------:|---------------:|---------------:|-------------------------:|----------:|----------------:|----:|-----------------------:|"); foreach (var result in results) { - Console.WriteLine($"| {result.Description} | {result.Concurrency} | {result.MedianThroughput:N0}/s | {result.MaximumThroughput:N0}/s |"); + Console.WriteLine( + $"| {result.Description} | {result.Concurrency} | {result.P50Throughput:N0}/s | " + + $"{result.P95Throughput:N0}/s | {result.P99Throughput:N0}/s | " + + $"{result.LatencyP50Microseconds:N1}/{result.LatencyP95Microseconds:N1}/{result.LatencyP99Microseconds:N1} | " + + $"{result.AllocatedBytesPerRequest:N1} | {result.Gen0CollectionsPerMillionRequests:N2} | " + + $"{result.CpuUtilization:N1}% | {result.LockContentionsPerMillionRequests:N2} |"); } Console.WriteLine(); } + private static double GetPercentile(double[] sortedSamples, double percentile) + { + var index = (int)Math.Ceiling(percentile * sortedSamples.Length) - 1; + return sortedSamples[Math.Clamp(index, 0, sortedSamples.Length - 1)]; + } + + private static double GetMedian( + AdaptiveConcurrencyLoadGenerator.FixedConcurrencyMeasurement[] measurements, + Func.FixedConcurrencyMeasurement, double> selector) + { + var values = measurements.Select(selector).Order().ToArray(); + var midpoint = values.Length / 2; + return values.Length % 2 == 0 ? (values[midpoint - 1] + values[midpoint]) / 2 : values[midpoint]; + } + private readonly record struct DeterministicResult( string Description, int Concurrency, - double MedianThroughput, - double MaximumThroughput); + double P50Throughput, + double P95Throughput, + double P99Throughput, + double LatencyP50Microseconds, + double LatencyP95Microseconds, + double LatencyP99Microseconds, + double AllocatedBytesPerRequest, + double Gen0CollectionsPerMillionRequests, + double CpuUtilization, + double LockContentionsPerMillionRequests); } From 270a77f9d115ac1f4b6741c3eacca1458fca7606 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 18:06:27 -0700 Subject: [PATCH 11/21] fix(runtime): bound cached message target handles --- .../GrainDirectoryCacheEntry.cs | 4 + .../Messaging/MessageCenter.cs | 39 +++-- .../GrainDirectoryCacheFactoryTests.cs | 148 ++++++++++++++++++ .../SharedEntryMessageTargetFastPathTests.cs | 135 ++++++++++++++-- 4 files changed, 307 insertions(+), 19 deletions(-) diff --git a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs index 9bf836e95c6..40f329cf1d6 100644 --- a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs +++ b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs @@ -9,6 +9,7 @@ internal sealed class GrainDirectoryCacheEntry { private static readonly object Invalidated = new(); private static readonly object Updating = new(); + private readonly WeakReference _referenceHandle; private object? _messageTarget; public GrainDirectoryCacheEntry(GrainAddress address, int version) @@ -22,12 +23,15 @@ public GrainDirectoryCacheEntry( long timestamp) : base(grainId, value, timestamp) { + _referenceHandle = new(this); } public GrainAddress Address => Value.ActivationAddress; public int Version => Value.Version; + public WeakReference ReferenceHandle => _referenceHandle; + public bool IsValid { get diff --git a/src/Orleans.Runtime/Messaging/MessageCenter.cs b/src/Orleans.Runtime/Messaging/MessageCenter.cs index 0d20f85ef6b..1a419484160 100644 --- a/src/Orleans.Runtime/Messaging/MessageCenter.cs +++ b/src/Orleans.Runtime/Messaging/MessageCenter.cs @@ -27,6 +27,7 @@ internal partial class MessageCenter : IMessageCenter, IAsyncDisposable private readonly SiloMessagingOptions messagingOptions; private readonly PlacementService placementService; private readonly GrainLocator _grainLocator; + private readonly IClusterMembershipService _clusterMembershipService; private readonly Action? _messageObserver; private readonly ILogger log; private readonly Catalog catalog; @@ -48,6 +49,7 @@ public MessageCenter( IOptions messagingOptions, PlacementService placementService, GrainLocator grainLocator, + IClusterMembershipService clusterMembershipService, IMessageStatisticsSink messageStatisticsSink) { this.catalog = catalog; @@ -59,6 +61,7 @@ public MessageCenter( _messagingProcessingInstruments = messagingProcessingInstruments; this.placementService = placementService; _grainLocator = grainLocator; + _clusterMembershipService = clusterMembershipService; _messageObserver = messageStatisticsSink.GetMessageObserver(); this.log = logger; this.messageFactory = messageFactory; @@ -596,34 +599,52 @@ internal bool TryGetDirectoryCacheEntry( Message message, [NotNullWhen(true)] out GrainDirectoryCacheEntry? entry) { - if (target?.MessageTargetCache is not GrainDirectoryCacheEntry candidate) + var cache = target?.MessageTargetCache; + if (cache is not WeakReference handle + || !handle.TryGetTarget(out var candidate)) { + if (target is not null && cache is not null) + { + target.ClearMessageTargetCache(cache); + } + entry = null; return false; } if (message.CacheInvalidationHeader is null && candidate.IsValid) { - Debug.Assert(candidate.Address.GrainId.Equals(message.TargetGrain)); - if (candidate.Address.SiloAddress is { } targetSilo - && (targetSilo.Matches(_siloAddress) || !siloStatusOracle.IsDeadSilo(targetSilo))) + var candidateAddress = candidate.Address; + Debug.Assert(candidateAddress.GrainId.Equals(message.TargetGrain)); + if (candidateAddress.SiloAddress is { } targetSilo + && (targetSilo.Matches(_siloAddress) || !IsKnownDeadSilo(candidateAddress))) { entry = candidate; return true; } } - if (!candidate.IsValid - || candidate.Address.SiloAddress is not { } candidateSilo - || siloStatusOracle.IsDeadSilo(candidateSilo)) + var clearHandle = !candidate.IsValid; + if (!clearHandle) { - target.ClearMessageTargetCache(candidate); + var candidateAddress = candidate.Address; + clearHandle = candidateAddress.SiloAddress is not { } candidateSilo + || !candidateSilo.Matches(_siloAddress) && IsKnownDeadSilo(candidateAddress); + } + + if (clearHandle) + { + target!.ClearMessageTargetCache(handle); } entry = null; return false; } + private bool IsKnownDeadSilo(GrainAddress address) + => address.SiloAddress is not { } siloAddress + || _clusterMembershipService.CurrentSnapshot.GetSiloStatus(siloAddress, address.MembershipVersion) == SiloStatus.Dead; + private GrainDirectoryCacheEntry? CaptureDirectoryCacheEntry(GrainReference? target, Message message) { if (target is null @@ -635,7 +656,7 @@ internal bool TryGetDirectoryCacheEntry( return null; } - target.MessageTargetCache = entry; + target.MessageTargetCache = entry.ReferenceHandle; return entry; } diff --git a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs index fa3a758add2..f768918ee0f 100644 --- a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs +++ b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs @@ -598,6 +598,25 @@ public void GrainDirectoryCacheEntry_UpdateBlocksBindingUntilNewAddressIsPublish AssertMessageTarget(entry, replacementTarget); } + [Fact] + public void GrainDirectoryCacheEntry_InvalidationDuringUpdateCannotReopenBinding() + { + var grainId = CreateGrainId(); + var originalAddress = CreateGrainAddress(grainId, port: 11111); + var replacementAddress = CreateGrainAddress(grainId, port: 22222); + var entry = new GrainDirectoryCacheEntry(originalAddress, version: 1); + + Assert.True(entry.TryBeginUpdate()); + + entry.Invalidate(); + entry.Value = (replacementAddress, 2); + entry.EndUpdate(); + + AssertInvalidEntry(entry); + Assert.Equal(replacementAddress, entry.Address); + Assert.Equal(2, entry.Version); + } + [Fact] public async Task CreateGrainDirectoryCache_RemoveByGrainIdReleasesMessageTargetReference() { @@ -621,6 +640,30 @@ public async Task CreateGrainDirectoryCache_RemoveByGrainIdReleasesMessageTarget } } + [Fact] + public void CreateGrainDirectoryCache_LongLivedHandleDoesNotRetainRemovedEntry() + { + var handle = CreateRemovedEntryHandle(); + + AssertEventuallyCollected(handle); + GC.KeepAlive(handle); + } + + [Fact] + public void CreateGrainDirectoryCache_HighCardinalityHandlesRemainBoundedAndReleaseObjectGraphs() + { + const int cacheSize = 64; + const int entryCount = 5_000; + var (handles, targets) = CreateHighCardinalityHandles(cacheSize, entryCount); + + Assert.Equal(entryCount, handles.Length); + Assert.Equal(entryCount, targets.Length); + AssertEventuallyCollected(handles); + AssertEventuallyCollected(targets); + GC.KeepAlive(handles); + GC.KeepAlive(targets); + } + private static (IGrainDirectoryCache Cache, IGrainDirectoryCacheEntrySource EntrySource) CreateEntryCache( int cacheSize = 10, TimeSpan? timeToLive = null, @@ -672,6 +715,69 @@ private static WeakReference BindMessageTarget(GrainDirectoryCach return new WeakReference(target); } + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static WeakReference AddRemoveAndGetEntryHandle( + IGrainDirectoryCache cache, + IGrainDirectoryCacheEntrySource entrySource, + GrainAddress address) + { + cache.AddOrUpdate(address, version: 1); + var entry = GetEntry(entrySource, address.GrainId); + var handle = entry.ReferenceHandle; + Assert.True(cache.Remove(address.GrainId)); + Assert.False(handle.TryGetTarget(out var retained) && retained.IsValid); + return handle; + } + + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static WeakReference CreateRemovedEntryHandle() + { + var (cache, entrySource) = CreateEntryCache(); + var disposableCache = Assert.IsAssignableFrom(cache); + try + { + return AddRemoveAndGetEntryHandle( + cache, + entrySource, + CreateGrainAddress(CreateGrainId(), port: 11111)); + } + finally + { + disposableCache.DisposeAsync().AsTask().GetAwaiter().GetResult(); + } + } + + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static ( + WeakReference[] Handles, + WeakReference[] Targets) CreateHighCardinalityHandles(int cacheSize, int entryCount) + { + var (cache, entrySource) = CreateEntryCache(cacheSize); + var disposableCache = Assert.IsAssignableFrom(cache); + var handles = new WeakReference[entryCount]; + var targets = new WeakReference[entryCount]; + try + { + for (var i = 0; i < entryCount; i++) + { + var address = CreateGrainAddress(CreateGrainId(), port: 11111 + (i % 100)); + cache.AddOrUpdate(address, version: i); + var entry = GetEntry(entrySource, address.GrainId); + var target = new object(); + Assert.True(entry.TrySetMessageTarget(target, address)); + handles[i] = entry.ReferenceHandle; + targets[i] = new(target); + } + + Assert.InRange(cache.KeyValues.Count(), 0, cacheSize); + return (handles, targets); + } + finally + { + disposableCache.DisposeAsync().AsTask().GetAwaiter().GetResult(); + } + } + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private static void AssertEventuallyCollected(WeakReference targetReference) { @@ -690,6 +796,48 @@ private static void AssertEventuallyCollected(WeakReference targe Assert.False(targetReference.TryGetTarget(out _)); } + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static void AssertEventuallyCollected(WeakReference entryReference) + { + for (var attempt = 0; attempt < 3; attempt++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + if (!entryReference.TryGetTarget(out _)) + { + return; + } + } + + Assert.False(entryReference.TryGetTarget(out _)); + } + + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static void AssertEventuallyCollected(WeakReference[] entryReferences) + { + CollectGarbage(); + Assert.All(entryReferences, entryReference => Assert.False(entryReference.TryGetTarget(out _))); + } + + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static void AssertEventuallyCollected(WeakReference[] targetReferences) + { + CollectGarbage(); + Assert.All(targetReferences, targetReference => Assert.False(targetReference.TryGetTarget(out _))); + } + + private static void CollectGarbage() + { + for (var attempt = 0; attempt < 3; attempt++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + } + } + private static GrainId CreateGrainId() => GrainId.Parse($"user/{Guid.NewGuid():N}"); private static IGrainContext CreateMessageTarget() => DispatchProxy.Create(); diff --git a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs index 91c96dd8dea..e61376b459a 100644 --- a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs +++ b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs @@ -45,7 +45,7 @@ public async Task LocalDirectoryGrain_AfterTwoCalls_BindsEntryToExactActivation( RequestContext.Remove(IPlacementDirector.PlacementHintKey); } - var entry = Assert.IsType(grainReference.MessageTargetCache); + var entry = GetEntry(grainReference); Assert.True(entry.IsValid); Assert.Equal(grainReference.GrainId, entry.Address.GrainId); Assert.Equal(primary.SiloAddress, entry.Address.SiloAddress); @@ -76,17 +76,17 @@ public async Task InvalidateCache_DisposesRetainedEntry_AndNextCallCapturesDiffe RequestContext.Remove(IPlacementDirector.PlacementHintKey); } - var retainedEntry = Assert.IsType(grainReference.MessageTargetCache); + var retainedEntry = GetEntry(grainReference); Assert.True(retainedEntry.IsValid); primary.ServiceProvider.GetRequiredService().InvalidateCache(grainReference.GrainId); Assert.False(retainedEntry.IsValid); Assert.False(retainedEntry.TryGetMessageTarget(out var disposedTarget)); Assert.Null(disposedTarget); - Assert.Same(retainedEntry, grainReference.MessageTargetCache); + Assert.Same(retainedEntry.ReferenceHandle, grainReference.MessageTargetCache); Assert.Equal(label, await grain.GetLabel()); - var replacementEntry = Assert.IsType(grainReference.MessageTargetCache); + var replacementEntry = GetEntry(grainReference); Assert.NotSame(retainedEntry, replacementEntry); Assert.True(replacementEntry.IsValid); Assert.Equal(grainReference.GrainId, replacementEntry.Address.GrainId); @@ -119,7 +119,7 @@ public async Task RemoteDirectoryGrain_BindsConnectionGroupTarget() RequestContext.Remove(IPlacementDirector.PlacementHintKey); } - var entry = Assert.IsType(grainReference.MessageTargetCache); + var entry = GetEntry(grainReference); Assert.True(entry.IsValid); Assert.Equal(grainReference.GrainId, entry.Address.GrainId); Assert.Equal(secondary.SiloAddress, entry.Address.SiloAddress); @@ -128,6 +128,84 @@ public async Task RemoteDirectoryGrain_BindsConnectionGroupTarget() Assert.Equal(secondary.SiloAddress, connectionEntry.Endpoint); } + [Fact] + public async Task RemoteDirectoryGrain_ConnectionCloseRebindsToNewConnectionGroup() + { + var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; + var secondary = _fixture.HostedCluster.Silos.Single( + silo => !silo.SiloAddress.Equals(primary.SiloAddress)); + var grainFactory = GetPrimarySiloGrainFactory(primary); + var grain = grainFactory.GetGrain(Interlocked.Increment(ref _nextGrainKey)); + var grainReference = Assert.IsAssignableFrom(grain); + const string label = "remote-reconnect"; + + RequestContext.Set(IPlacementDirector.PlacementHintKey, secondary.SiloAddress); + try + { + await grain.SetLabel(label); + Assert.Equal(label, await grain.GetLabel()); + } + finally + { + RequestContext.Remove(IPlacementDirector.PlacementHintKey); + } + + var entry = GetEntry(grainReference); + Assert.True(entry.TryGetMessageTarget(out var originalTarget)); + var originalConnectionEntry = Assert.IsType(originalTarget); + var connectionManager = primary.ServiceProvider.GetRequiredService(); + + await connectionManager.CloseAsync(secondary.SiloAddress); + + Assert.Null(originalConnectionEntry.NextConnection()); + Assert.Equal(label, await grain.GetLabel()); + + var reboundEntry = GetEntry(grainReference); + Assert.True(reboundEntry.TryGetMessageTarget(out var reboundTarget)); + var reboundConnectionEntry = Assert.IsType(reboundTarget); + Assert.NotSame(originalConnectionEntry, reboundConnectionEntry); + Assert.Equal(secondary.SiloAddress, reboundConnectionEntry.Endpoint); + Assert.NotNull(reboundConnectionEntry.NextConnection()); + } + + [Fact] + public async Task LocalDirectoryGrain_DeactivationRebindsToNewActivation() + { + var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; + var grainFactory = GetPrimarySiloGrainFactory(primary); + var grain = grainFactory.GetGrain(Guid.NewGuid()); + var grainReference = Assert.IsAssignableFrom(grain); + + RequestContext.Set(IPlacementDirector.PlacementHintKey, primary.SiloAddress); + try + { + _ = await grain.GetActivationId(); + _ = await grain.GetActivationId(); + } + finally + { + RequestContext.Remove(IPlacementDirector.PlacementHintKey); + } + + var originalEntry = GetEntry(grainReference); + Assert.True(originalEntry.TryGetMessageTarget(out var originalTarget)); + var originalActivation = Assert.IsType(originalTarget); + var originalActivationId = await grain.GetActivationId(); + + await grain.Deactivate(); + var reboundActivationId = await grain.GetActivationId(); + Assert.Equal(reboundActivationId, await grain.GetActivationId()); + + Assert.NotEqual(originalActivationId, reboundActivationId); + Assert.False(originalActivation.IsValid); + var reboundEntry = GetEntry(grainReference); + Assert.True(reboundEntry.TryGetMessageTarget(out var reboundTarget)); + var reboundActivation = Assert.IsType(reboundTarget); + Assert.NotSame(originalActivation, reboundActivation); + Assert.True(reboundActivation.IsValid); + Assert.Equal(reboundEntry.Address, reboundActivation.Address); + } + [Fact] public async Task CompatibleInterfaceCast_TransfersSameEntryHandle() { @@ -148,17 +226,17 @@ public async Task CompatibleInterfaceCast_TransfersSameEntryHandle() RequestContext.Remove(IPlacementDirector.PlacementHintKey); } - var writerEntry = Assert.IsType(writerReference.MessageTargetCache); + var writerEntry = GetEntry(writerReference); Assert.True(writerEntry.IsValid); var reader = writer.AsReference(); var readerReference = Assert.IsAssignableFrom(reader); Assert.NotSame(writerReference, readerReference); - Assert.Same(writerEntry, readerReference.MessageTargetCache); + Assert.Same(writerEntry.ReferenceHandle, readerReference.MessageTargetCache); Assert.Equal(writerReference.GrainId, readerReference.GrainId); Assert.Equal(value, await reader.GetValue()); - Assert.Same(writerEntry, readerReference.MessageTargetCache); + Assert.Same(writerEntry.ReferenceHandle, readerReference.MessageTargetCache); } [Fact] @@ -201,7 +279,7 @@ public async Task CacheInvalidationHeader_BypassesFastPathWithoutDiscardingLiveH RequestContext.Remove(IPlacementDirector.PlacementHintKey); } - var retainedEntry = Assert.IsType(grainReference.MessageTargetCache); + var retainedEntry = GetEntry(grainReference); var unrelatedAddress = new GrainAddress { GrainId = GrainId.Create("unrelated", Guid.NewGuid().ToString()), @@ -219,7 +297,37 @@ public async Task CacheInvalidationHeader_BypassesFastPathWithoutDiscardingLiveH Assert.Null(selectedEntry); Assert.True(retainedEntry.IsValid); - Assert.Same(retainedEntry, grainReference.MessageTargetCache); + Assert.Same(retainedEntry.ReferenceHandle, grainReference.MessageTargetCache); + } + + [Fact] + public void PurgedSiloEntry_IsRejectedAndClearedFromGrainReference() + { + var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; + var grainFactory = GetPrimarySiloGrainFactory(primary); + var grain = grainFactory.GetGrain(Interlocked.Increment(ref _nextGrainKey)); + var grainReference = Assert.IsAssignableFrom(grain); + var address = new GrainAddress + { + GrainId = grainReference.GrainId, + ActivationId = ActivationId.NewId(), + SiloAddress = SiloAddress.FromParsableString("127.0.0.1:54321@1"), + MembershipVersion = new MembershipVersion(0), + }; + var entry = new GrainDirectoryCacheEntry(address, version: 0); + grainReference.MessageTargetCache = entry.ReferenceHandle; + var message = new Message + { + Direction = Message.Directions.Request, + TargetGrain = grainReference.GrainId, + }; + + var messageCenter = primary.ServiceProvider.GetRequiredService(); + Assert.False(messageCenter.TryGetDirectoryCacheEntry(grainReference, message, out var selectedEntry)); + + Assert.Null(selectedEntry); + Assert.Null(grainReference.MessageTargetCache); + Assert.True(entry.IsValid); } private static IGrainFactory GetPrimarySiloGrainFactory(InProcessSiloHandle primary) @@ -230,6 +338,13 @@ private static IGrainFactory GetPrimarySiloGrainFactory(InProcessSiloHandle prim return grainFactory; } + private static GrainDirectoryCacheEntry GetEntry(GrainReference grainReference) + { + var handle = Assert.IsType>(grainReference.MessageTargetCache); + Assert.True(handle.TryGetTarget(out var entry)); + return entry; + } + public sealed class Fixture : BaseTestClusterFixture { protected override void ConfigureTestCluster(TestClusterBuilder builder) From d587fc61c08c9a565acfd59797d2ae2999a78f6e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 18:10:13 -0700 Subject: [PATCH 12/21] perf(benchmarks): measure retained directory handles --- .../Caching/GrainDirectoryCacheBenchmark.cs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs b/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs index 1a948cdeca4..ae34180cc4c 100644 --- a/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs +++ b/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs @@ -12,6 +12,7 @@ public class GrainDirectoryCacheBenchmark private ConcurrentLruCache _tupleCache = null!; private LruGrainDirectoryCache _entryCache = null!; private IGrainDirectoryCacheEntrySource _entrySource = null!; + private WeakReference _entryHandle = null!; private GrainId _target; [GlobalSetup] @@ -43,6 +44,13 @@ public void Setup() _target = grainId; } } + + if (!_entrySource.TryGetEntry(_target, out var entry)) + { + throw new InvalidOperationException("Benchmark entry was not added."); + } + + _entryHandle = entry.ReferenceHandle; } [Benchmark(Baseline = true)] @@ -67,12 +75,19 @@ public GrainAddress SharedEntryValidated() } [Benchmark] - public GrainAddress SharedHandle() + public GrainAddress SharedEntrySource() { _entrySource.TryGetEntry(_target, out var entry); return entry!.Address; } + [Benchmark] + public GrainAddress SharedHandle() + { + _entryHandle.TryGetTarget(out var entry); + return entry!.Address; + } + [GlobalCleanup] public async Task Cleanup() { From fd3c3acce9e6ac2ff3d0b9e01d5479326b2c9c07 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 18:31:20 -0700 Subject: [PATCH 13/21] perf(runtime): limit target caching to local activations --- .../Networking/ConnectionManager.cs | 38 ++----------- .../Messaging/MessageCenter.cs | 54 ++----------------- .../SharedEntryMessageTargetFastPathTests.cs | 49 ++--------------- 3 files changed, 12 insertions(+), 129 deletions(-) diff --git a/src/Orleans.Core/Networking/ConnectionManager.cs b/src/Orleans.Core/Networking/ConnectionManager.cs index 63a8db5bf5c..553666bcd07 100644 --- a/src/Orleans.Core/Networking/ConnectionManager.cs +++ b/src/Orleans.Core/Networking/ConnectionManager.cs @@ -64,28 +64,17 @@ public ValueTask GetConnection(SiloAddress endpoint) return new(this.GetConnectionAsync(endpoint)); } public bool TryGetConnection(SiloAddress endpoint, [NotNullWhen(true)] out Connection? connection) - => TryGetConnection(endpoint, out connection, out _); - - internal bool TryGetConnection( - SiloAddress endpoint, - [NotNullWhen(true)] out Connection? connection, - [NotNullWhen(true)] out ConnectionEntry? connectionEntry) { if (this.connections.TryGetValue(endpoint, out var entry) && entry.NextConnection() is { } c) { connection = c; - connectionEntry = entry; return true; } connection = null; - connectionEntry = null; return false; } - internal bool TryGetConnectionEntry(SiloAddress endpoint, [NotNullWhen(true)] out ConnectionEntry? connectionEntry) - => connections.TryGetValue(endpoint, out connectionEntry); - /// /// Gets the minimum elapsed time since any connection to the specified silo last received a message, /// or if no connections exist or no messages have been received. @@ -208,7 +197,7 @@ public void OnConnectionTerminated(SiloAddress address, Connection connection, E } } - private ConnectionEntry GetOrCreateEntry(SiloAddress address) => connections.GetOrAdd(address, static address => new(address)); + private ConnectionEntry GetOrCreateEntry(SiloAddress address) => connections.GetOrAdd(address, _ => new()); private async Task ConnectAsync(SiloAddress address, ConnectionEntry entry) { @@ -267,10 +256,7 @@ public async Task CloseAsync(SiloAddress endpoint) connections = entry.Connections; if (entry.PendingConnection is null) { - if (this.connections.TryRemove(KeyValuePair.Create(endpoint, entry))) - { - connections = entry.Detach(); - } + this.connections.TryRemove(endpoint, out _); } } @@ -386,13 +372,8 @@ private async Task RunConnectionAsync(SiloAddress address, Connection connection return null; } - internal sealed class ConnectionEntry + private sealed class ConnectionEntry { - private int _detached; - - public ConnectionEntry(SiloAddress endpoint) => Endpoint = endpoint; - - public SiloAddress Endpoint { get; } public Task? PendingConnection { get; set; } public DateTime LastFailure { get; set; } public ImmutableArray Connections { get; set; } = ImmutableArray.Empty; @@ -417,11 +398,6 @@ public TimeSpan GetRemainingRetryDelay(ConnectionOptions options) public Connection? NextConnection() { - if (Volatile.Read(ref _detached) != 0) - { - return null; - } - var connections = this.Connections; if (connections.IsEmpty) { @@ -433,14 +409,6 @@ public TimeSpan GetRemainingRetryDelay(ConnectionOptions options) } public void RemoveDefunct() => Connections = Connections.RemoveAll(c => !c.IsValid); - - public ImmutableArray Detach() - { - Volatile.Write(ref _detached, 1); - var result = Connections; - Connections = ImmutableArray.Empty; - return result; - } } [LoggerMessage( diff --git a/src/Orleans.Runtime/Messaging/MessageCenter.cs b/src/Orleans.Runtime/Messaging/MessageCenter.cs index 1a419484160..287e7c6ac4d 100644 --- a/src/Orleans.Runtime/Messaging/MessageCenter.cs +++ b/src/Orleans.Runtime/Messaging/MessageCenter.cs @@ -231,14 +231,8 @@ private void SendMessage( } else { - if (TryGetCachedConnection(directoryCacheEntry, targetSilo, out var cachedConnection)) + if (this.connectionManager.TryGetConnection(targetSilo, out var existingConnection)) { - cachedConnection.Send(msg); - return; - } - else if (this.connectionManager.TryGetConnection(targetSilo, out var existingConnection, out var connectionEntry)) - { - directoryCacheEntry?.TrySetMessageTarget(connectionEntry, targetSilo); existingConnection.Send(msg); return; } @@ -259,34 +253,17 @@ private void SendMessage( if (connectionTask.IsCompletedSuccessfully) { var sender = connectionTask.Result; - if (directoryCacheEntry is not null - && connectionManager.TryGetConnectionEntry(targetSilo, out var completedConnectionEntry)) - { - directoryCacheEntry.TrySetMessageTarget(completedConnectionEntry, targetSilo); - } - sender.Send(msg); } else { - _ = SendAsync(this, connectionTask, msg, directoryCacheEntry, targetSilo); - - static async Task SendAsync( - MessageCenter messageCenter, - ValueTask connectionTask, - Message msg, - GrainDirectoryCacheEntry? directoryCacheEntry, - SiloAddress targetSilo) + _ = SendAsync(this, connectionTask, msg); + + static async Task SendAsync(MessageCenter messageCenter, ValueTask connectionTask, Message msg) { try { var sender = await connectionTask; - if (directoryCacheEntry is not null - && messageCenter.connectionManager.TryGetConnectionEntry(targetSilo, out var connectionEntry)) - { - directoryCacheEntry.TrySetMessageTarget(connectionEntry, targetSilo); - } - sender.Send(msg); } catch (Exception exception) @@ -300,29 +277,6 @@ static async Task SendAsync( } } - private static bool TryGetCachedConnection( - GrainDirectoryCacheEntry? directoryCacheEntry, - SiloAddress targetSilo, - [NotNullWhen(true)] out Connection? connection) - { - if (directoryCacheEntry?.TryGetMessageTarget(out var target) == true - && target is { } messageTarget) - { - if (messageTarget is ConnectionManager.ConnectionEntry connectionEntry - && connectionEntry.Endpoint.Equals(targetSilo) - && connectionEntry.NextConnection() is { } result) - { - connection = result; - return true; - } - - directoryCacheEntry.ClearMessageTarget(messageTarget); - } - - connection = null; - return false; - } - public void DispatchLocalMessage(Message message) => ReceiveMessage(message); public void RejectMessage( diff --git a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs index e61376b459a..f1a90327aa9 100644 --- a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs +++ b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs @@ -98,7 +98,7 @@ public async Task InvalidateCache_DisposesRetainedEntry_AndNextCallCapturesDiffe } [Fact] - public async Task RemoteDirectoryGrain_BindsConnectionGroupTarget() + public async Task RemoteDirectoryGrain_DoesNotBindConnectionTarget() { var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; var secondary = _fixture.HostedCluster.Silos.Single( @@ -123,49 +123,7 @@ public async Task RemoteDirectoryGrain_BindsConnectionGroupTarget() Assert.True(entry.IsValid); Assert.Equal(grainReference.GrainId, entry.Address.GrainId); Assert.Equal(secondary.SiloAddress, entry.Address.SiloAddress); - Assert.True(entry.TryGetMessageTarget(out var messageTarget)); - var connectionEntry = Assert.IsType(messageTarget); - Assert.Equal(secondary.SiloAddress, connectionEntry.Endpoint); - } - - [Fact] - public async Task RemoteDirectoryGrain_ConnectionCloseRebindsToNewConnectionGroup() - { - var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; - var secondary = _fixture.HostedCluster.Silos.Single( - silo => !silo.SiloAddress.Equals(primary.SiloAddress)); - var grainFactory = GetPrimarySiloGrainFactory(primary); - var grain = grainFactory.GetGrain(Interlocked.Increment(ref _nextGrainKey)); - var grainReference = Assert.IsAssignableFrom(grain); - const string label = "remote-reconnect"; - - RequestContext.Set(IPlacementDirector.PlacementHintKey, secondary.SiloAddress); - try - { - await grain.SetLabel(label); - Assert.Equal(label, await grain.GetLabel()); - } - finally - { - RequestContext.Remove(IPlacementDirector.PlacementHintKey); - } - - var entry = GetEntry(grainReference); - Assert.True(entry.TryGetMessageTarget(out var originalTarget)); - var originalConnectionEntry = Assert.IsType(originalTarget); - var connectionManager = primary.ServiceProvider.GetRequiredService(); - - await connectionManager.CloseAsync(secondary.SiloAddress); - - Assert.Null(originalConnectionEntry.NextConnection()); - Assert.Equal(label, await grain.GetLabel()); - - var reboundEntry = GetEntry(grainReference); - Assert.True(reboundEntry.TryGetMessageTarget(out var reboundTarget)); - var reboundConnectionEntry = Assert.IsType(reboundTarget); - Assert.NotSame(originalConnectionEntry, reboundConnectionEntry); - Assert.Equal(secondary.SiloAddress, reboundConnectionEntry.Endpoint); - Assert.NotNull(reboundConnectionEntry.NextConnection()); + Assert.False(entry.TryGetMessageTarget(out _)); } [Fact] @@ -198,6 +156,9 @@ public async Task LocalDirectoryGrain_DeactivationRebindsToNewActivation() Assert.NotEqual(originalActivationId, reboundActivationId); Assert.False(originalActivation.IsValid); + primary.ServiceProvider.GetRequiredService().InvalidateCache(grainReference.GrainId); + Assert.Equal(reboundActivationId, await grain.GetActivationId()); + Assert.Equal(reboundActivationId, await grain.GetActivationId()); var reboundEntry = GetEntry(grainReference); Assert.True(reboundEntry.TryGetMessageTarget(out var reboundTarget)); var reboundActivation = Assert.IsType(reboundTarget); From bf421fcc5c3bf838fab1350fe1bbdeb4343000b5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 19:23:29 -0700 Subject: [PATCH 14/21] perf(benchmarks): measure directory entry allocation cost --- .../Caching/GrainDirectoryCacheBenchmark.cs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs b/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs index ae34180cc4c..bd9ca52155e 100644 --- a/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs +++ b/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs @@ -95,3 +95,32 @@ public async Task Cleanup() await _entryCache.DisposeAsync(); } } + +[MemoryDiagnoser] +[BenchmarkCategory("Caching")] +public class GrainDirectoryCacheAllocationBenchmark +{ + private GrainId _grainId; + private GrainAddress _address = null!; + + [GlobalSetup] + public void Setup() + { + _grainId = GrainId.Create("benchmark", "allocation"); + _address = new GrainAddress + { + GrainId = _grainId, + ActivationId = ActivationId.NewId(), + SiloAddress = SiloAddress.FromParsableString("127.0.0.1:11111@1"), + }; + } + + [Benchmark(Baseline = true)] + public object TupleEntry() + => new ConcurrentLruCache.LruItem( + _grainId, + (_address, 1)); + + [Benchmark] + public object SharedEntry() => new GrainDirectoryCacheEntry(_address, version: 1); +} From f2d2168111839d6674ddc445ef0efa1099aaca98 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 19:56:22 -0700 Subject: [PATCH 15/21] fix(runtime): refresh retained directory cache entries --- .../Caching/ConcurrentLruCache.cs | 4 +-- .../GrainDirectoryCacheEntry.cs | 16 ++++++++- .../GrainDirectory/LruGrainDirectoryCache.cs | 4 ++- .../Messaging/MessageCenter.cs | 3 +- .../GrainDirectoryCacheFactoryTests.cs | 34 +++++++++++++++++++ 5 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/Orleans.Core/Caching/ConcurrentLruCache.cs b/src/Orleans.Core/Caching/ConcurrentLruCache.cs index fdec41202bf..0cfee70840d 100644 --- a/src/Orleans.Core/Caching/ConcurrentLruCache.cs +++ b/src/Orleans.Core/Caching/ConcurrentLruCache.cs @@ -181,7 +181,7 @@ protected bool TryGetItem(K key, [NotNullWhen(true)] out LruItem? item) { if (_dictionary.TryGetValue(key, out item)) { - Touch(item); + TouchItem(item); _telemetryPolicy.IncrementHit(); return true; } @@ -1080,7 +1080,7 @@ protected virtual LruItem CreateItem(K key, V value) => protected long GetCurrentTimestamp() => _expiresAfterAccess ? _timeProvider.GetTimestamp() : 0; [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void Touch(LruItem item) + protected void TouchItem(LruItem item) { if (_expiresAfterAccess) { diff --git a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs index 40f329cf1d6..6539f7678a8 100644 --- a/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs +++ b/src/Orleans.Runtime/GrainDirectory/GrainDirectoryCacheEntry.cs @@ -9,20 +9,23 @@ internal sealed class GrainDirectoryCacheEntry { private static readonly object Invalidated = new(); private static readonly object Updating = new(); + private readonly LruGrainDirectoryCache? _owner; private readonly WeakReference _referenceHandle; private object? _messageTarget; public GrainDirectoryCacheEntry(GrainAddress address, int version) - : this(address.GrainId, (address, version), timestamp: 0) + : this(owner: null, address.GrainId, (address, version), timestamp: 0) { } public GrainDirectoryCacheEntry( + LruGrainDirectoryCache? owner, GrainId grainId, (GrainAddress ActivationAddress, int Version) value, long timestamp) : base(grainId, value, timestamp) { + _owner = owner; _referenceHandle = new(this); } @@ -32,6 +35,17 @@ public GrainDirectoryCacheEntry( public WeakReference ReferenceHandle => _referenceHandle; + public bool TryTouch() + { + if (!IsValid) + { + return false; + } + + _owner?.Touch(this); + return IsValid; + } + public bool IsValid { get diff --git a/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs b/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs index a1e584f0fb9..de0f01071ac 100644 --- a/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs +++ b/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs @@ -73,7 +73,9 @@ public bool TryGetEntry(GrainId key, [NotNullWhen(true)] out GrainDirectoryCache } protected override LruItem CreateItem(GrainId key, (GrainAddress ActivationAddress, int Version) value) - => new GrainDirectoryCacheEntry(key, value, GetCurrentTimestamp()); + => new GrainDirectoryCacheEntry(this, key, value, GetCurrentTimestamp()); + + internal void Touch(GrainDirectoryCacheEntry entry) => TouchItem(entry); protected override void UpdateItem(LruItem item, (GrainAddress ActivationAddress, int Version) value) => ((GrainDirectoryCacheEntry)item).Update(value); diff --git a/src/Orleans.Runtime/Messaging/MessageCenter.cs b/src/Orleans.Runtime/Messaging/MessageCenter.cs index 287e7c6ac4d..fada653c357 100644 --- a/src/Orleans.Runtime/Messaging/MessageCenter.cs +++ b/src/Orleans.Runtime/Messaging/MessageCenter.cs @@ -571,7 +571,8 @@ internal bool TryGetDirectoryCacheEntry( var candidateAddress = candidate.Address; Debug.Assert(candidateAddress.GrainId.Equals(message.TargetGrain)); if (candidateAddress.SiloAddress is { } targetSilo - && (targetSilo.Matches(_siloAddress) || !IsKnownDeadSilo(candidateAddress))) + && (targetSilo.Matches(_siloAddress) || !IsKnownDeadSilo(candidateAddress)) + && candidate.TryTouch()) { entry = candidate; return true; diff --git a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs index f768918ee0f..6bfb0579e4e 100644 --- a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs +++ b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs @@ -449,6 +449,40 @@ public async Task CreateGrainDirectoryCache_ExpirationInvalidatesRouteHandle() } } + [Fact] + public async Task CreateGrainDirectoryCache_RetainedHandleTouchRefreshesExpiration() + { + var timeProvider = new FakeTimeProvider(); + var timeToLive = TimeSpan.FromMinutes(1); + var (cache, entrySource) = CreateEntryCache(cacheSize: 10, timeToLive, timeProvider); + var disposableCache = Assert.IsAssignableFrom(cache); + using var listener = new ConcurrentLruCacheExpirationCleanupListener(cache); + var address = CreateGrainAddress(CreateGrainId(), port: 11111); + + try + { + cache.AddOrUpdate(address, version: 1); + var entry = GetEntry(entrySource, address.GrainId); + + timeProvider.Advance(timeToLive); + Assert.Equal(0, await listener.WaitForCleanupAsync()); + Assert.True(entry.TryTouch()); + + timeProvider.Advance(timeToLive); + Assert.Equal(0, await listener.WaitForCleanupAsync()); + Assert.True(entry.IsValid); + Assert.Single(cache.KeyValues); + + timeProvider.Advance(timeToLive); + Assert.Equal(1, await listener.WaitForCleanupAsync()); + AssertInvalidEntry(entry); + } + finally + { + await disposableCache.DisposeAsync(); + } + } + [Fact] public async Task CreateGrainDirectoryCache_EvictionInvalidatesRouteHandle() { From c83c32df2541d6fee9c3f2bbf8dd7ac4b8ec53a9 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 19:56:23 -0700 Subject: [PATCH 16/21] fix(benchmarks): honor deterministic run cancellation --- test/Benchmarks/Ping/AdaptiveConcurrencyLoadGenerator.cs | 3 +++ test/Benchmarks/Ping/AdaptivePingBenchmark.cs | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/Benchmarks/Ping/AdaptiveConcurrencyLoadGenerator.cs b/test/Benchmarks/Ping/AdaptiveConcurrencyLoadGenerator.cs index 4faeba716c6..29fc36aa7b5 100644 --- a/test/Benchmarks/Ping/AdaptiveConcurrencyLoadGenerator.cs +++ b/test/Benchmarks/Ping/AdaptiveConcurrencyLoadGenerator.cs @@ -170,12 +170,15 @@ public async Task RunFixedConcurrencyAsync( try { await RunPhaseAsync(_warmupDuration, isWarmup: true, states); + cancellationToken.ThrowIfCancellationRequested(); GC.Collect(); var results = new FixedConcurrencyMeasurement[repetitions]; for (var i = 0; i < results.Length; i++) { + cancellationToken.ThrowIfCancellationRequested(); var measurement = await RunPhaseAsync(_measurementInterval, isWarmup: false, states); + cancellationToken.ThrowIfCancellationRequested(); if (measurement.Failures > 0) { throw new InvalidOperationException($"Measurement completed with {measurement.Failures:N0} failed requests."); diff --git a/test/Benchmarks/Ping/AdaptivePingBenchmark.cs b/test/Benchmarks/Ping/AdaptivePingBenchmark.cs index 72bb0f876c0..cac3740db4b 100644 --- a/test/Benchmarks/Ping/AdaptivePingBenchmark.cs +++ b/test/Benchmarks/Ping/AdaptivePingBenchmark.cs @@ -320,6 +320,7 @@ private static async Task> MeasureDeterministicScenari { foreach (var concurrency in concurrencyLevels) { + benchmark._cts.Token.ThrowIfCancellationRequested(); var loadGenerator = new AdaptiveConcurrencyLoadGenerator( issueRequest: grain => grain.Run(), getStateForWorker: workerId => benchmark.GetGrainFactory().GetGrain(workerId), @@ -333,7 +334,7 @@ private static async Task> MeasureDeterministicScenari initialStepSize: 1, sampleInterval: DefaultSampleInterval, minimumRelativeImprovement: 0); - var measurements = await loadGenerator.RunFixedConcurrencyAsync(repetitions); + var measurements = await loadGenerator.RunFixedConcurrencyAsync(repetitions, benchmark._cts.Token); var throughput = measurements.Select(static measurement => measurement.Throughput).Order().ToArray(); var result = new DeterministicResult( benchmark.Description, From c774bd0bb18d5afb46446ccf78a8e5b322cac238 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 20:16:57 -0700 Subject: [PATCH 17/21] perf(runtime): retain only local directory entries --- src/Orleans.Runtime/Messaging/MessageCenter.cs | 12 +++--------- .../SharedEntryMessageTargetFastPathTests.cs | 8 ++------ 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/Orleans.Runtime/Messaging/MessageCenter.cs b/src/Orleans.Runtime/Messaging/MessageCenter.cs index fada653c357..86a0ca63073 100644 --- a/src/Orleans.Runtime/Messaging/MessageCenter.cs +++ b/src/Orleans.Runtime/Messaging/MessageCenter.cs @@ -27,7 +27,6 @@ internal partial class MessageCenter : IMessageCenter, IAsyncDisposable private readonly SiloMessagingOptions messagingOptions; private readonly PlacementService placementService; private readonly GrainLocator _grainLocator; - private readonly IClusterMembershipService _clusterMembershipService; private readonly Action? _messageObserver; private readonly ILogger log; private readonly Catalog catalog; @@ -49,7 +48,6 @@ public MessageCenter( IOptions messagingOptions, PlacementService placementService, GrainLocator grainLocator, - IClusterMembershipService clusterMembershipService, IMessageStatisticsSink messageStatisticsSink) { this.catalog = catalog; @@ -61,7 +59,6 @@ public MessageCenter( _messagingProcessingInstruments = messagingProcessingInstruments; this.placementService = placementService; _grainLocator = grainLocator; - _clusterMembershipService = clusterMembershipService; _messageObserver = messageStatisticsSink.GetMessageObserver(); this.log = logger; this.messageFactory = messageFactory; @@ -571,7 +568,7 @@ internal bool TryGetDirectoryCacheEntry( var candidateAddress = candidate.Address; Debug.Assert(candidateAddress.GrainId.Equals(message.TargetGrain)); if (candidateAddress.SiloAddress is { } targetSilo - && (targetSilo.Matches(_siloAddress) || !IsKnownDeadSilo(candidateAddress)) + && targetSilo.Matches(_siloAddress) && candidate.TryTouch()) { entry = candidate; @@ -584,7 +581,7 @@ internal bool TryGetDirectoryCacheEntry( { var candidateAddress = candidate.Address; clearHandle = candidateAddress.SiloAddress is not { } candidateSilo - || !candidateSilo.Matches(_siloAddress) && IsKnownDeadSilo(candidateAddress); + || !candidateSilo.Matches(_siloAddress); } if (clearHandle) @@ -596,15 +593,12 @@ internal bool TryGetDirectoryCacheEntry( return false; } - private bool IsKnownDeadSilo(GrainAddress address) - => address.SiloAddress is not { } siloAddress - || _clusterMembershipService.CurrentSnapshot.GetSiloStatus(siloAddress, address.MembershipVersion) == SiloStatus.Dead; - private GrainDirectoryCacheEntry? CaptureDirectoryCacheEntry(GrainReference? target, Message message) { if (target is null || message.CacheInvalidationHeader is not null || message.TargetSilo is not { } targetSilo + || !targetSilo.Matches(_siloAddress) || !placementService.IsUsingGrainDirectory(message.TargetGrain) || !_grainLocator.TryGetCacheEntry(message.TargetGrain, targetSilo, out var entry)) { diff --git a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs index f1a90327aa9..a75d7d3298f 100644 --- a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs +++ b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs @@ -119,11 +119,7 @@ public async Task RemoteDirectoryGrain_DoesNotBindConnectionTarget() RequestContext.Remove(IPlacementDirector.PlacementHintKey); } - var entry = GetEntry(grainReference); - Assert.True(entry.IsValid); - Assert.Equal(grainReference.GrainId, entry.Address.GrainId); - Assert.Equal(secondary.SiloAddress, entry.Address.SiloAddress); - Assert.False(entry.TryGetMessageTarget(out _)); + Assert.Null(grainReference.MessageTargetCache); } [Fact] @@ -262,7 +258,7 @@ public async Task CacheInvalidationHeader_BypassesFastPathWithoutDiscardingLiveH } [Fact] - public void PurgedSiloEntry_IsRejectedAndClearedFromGrainReference() + public void RemoteSiloEntry_IsRejectedAndClearedFromGrainReference() { var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; var grainFactory = GetPrimarySiloGrainFactory(primary); From eec280ce27f81cab7bb141700f3ec6725f86a321 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 20:34:08 -0700 Subject: [PATCH 18/21] perf(benchmarks): include cache touch in handle cost --- test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs b/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs index bd9ca52155e..d8891de7796 100644 --- a/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs +++ b/test/Benchmarks/Caching/GrainDirectoryCacheBenchmark.cs @@ -85,7 +85,8 @@ public GrainAddress SharedEntrySource() public GrainAddress SharedHandle() { _entryHandle.TryGetTarget(out var entry); - return entry!.Address; + entry!.TryTouch(); + return entry.Address; } [GlobalCleanup] From d4059ab24cee771ddea616a46d8c40af1b712989 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 20:44:22 -0700 Subject: [PATCH 19/21] test(runtime): stabilize activation replacement coverage --- .../SharedEntryMessageTargetFastPathTests.cs | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs index a75d7d3298f..34c34fee187 100644 --- a/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs +++ b/test/Orleans.Runtime.Tests/SharedEntryMessageTargetFastPathTests.cs @@ -123,7 +123,7 @@ public async Task RemoteDirectoryGrain_DoesNotBindConnectionTarget() } [Fact] - public async Task LocalDirectoryGrain_DeactivationRebindsToNewActivation() + public async Task LocalDirectoryGrain_DeactivationDoesNotReuseInvalidActivation() { var primary = (InProcessSiloHandle)_fixture.HostedCluster.Primary!; var grainFactory = GetPrimarySiloGrainFactory(primary); @@ -147,20 +147,24 @@ public async Task LocalDirectoryGrain_DeactivationRebindsToNewActivation() var originalActivationId = await grain.GetActivationId(); await grain.Deactivate(); - var reboundActivationId = await grain.GetActivationId(); - Assert.Equal(reboundActivationId, await grain.GetActivationId()); - - Assert.NotEqual(originalActivationId, reboundActivationId); - Assert.False(originalActivation.IsValid); - primary.ServiceProvider.GetRequiredService().InvalidateCache(grainReference.GrainId); - Assert.Equal(reboundActivationId, await grain.GetActivationId()); - Assert.Equal(reboundActivationId, await grain.GetActivationId()); - var reboundEntry = GetEntry(grainReference); - Assert.True(reboundEntry.TryGetMessageTarget(out var reboundTarget)); - var reboundActivation = Assert.IsType(reboundTarget); - Assert.NotSame(originalActivation, reboundActivation); - Assert.True(reboundActivation.IsValid); - Assert.Equal(reboundEntry.Address, reboundActivation.Address); + RequestContext.Set(IPlacementDirector.PlacementHintKey, primary.SiloAddress); + try + { + var reboundActivationId = await grain.GetActivationId(); + Assert.Equal(reboundActivationId, await grain.GetActivationId()); + + Assert.NotEqual(originalActivationId, reboundActivationId); + Assert.False(originalActivation.IsValid); + Assert.True(_fixture.HostedCluster.TryGetGrainContext(grainReference.GrainId, out var reboundContext)); + var reboundActivation = Assert.IsType(reboundContext); + Assert.NotSame(originalActivation, reboundActivation); + Assert.True(reboundActivation.IsValid); + Assert.Equal(primary.SiloAddress, reboundActivation.Address.SiloAddress); + } + finally + { + RequestContext.Remove(IPlacementDirector.PlacementHintKey); + } } [Fact] From 7bab3d160061f9fd4dd2e36bbd063dcf7b47dca1 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 28 Aug 2026 20:59:43 -0700 Subject: [PATCH 20/21] fix(runtime): release cached targets on cache disposal --- .../GrainDirectory/LruGrainDirectoryCache.cs | 1 + .../GrainDirectoryCacheFactoryTests.cs | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs b/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs index de0f01071ac..ab15c2fb7b0 100644 --- a/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs +++ b/src/Orleans.Runtime/GrainDirectory/LruGrainDirectoryCache.cs @@ -86,6 +86,7 @@ protected override void OnItemRemoved(LruItem item) public new async ValueTask DisposeAsync() { _cacheSizeRegistration.Dispose(); + Clear(); await base.DisposeAsync(); } diff --git a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs index 6bfb0579e4e..f01295c794a 100644 --- a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs +++ b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs @@ -683,6 +683,22 @@ public void CreateGrainDirectoryCache_LongLivedHandleDoesNotRetainRemovedEntry() GC.KeepAlive(handle); } + [Fact] + public async Task CreateGrainDirectoryCache_DisposeInvalidatesHandleAndReleasesTarget() + { + var (cache, entrySource) = CreateEntryCache(); + var disposableCache = Assert.IsAssignableFrom(cache); + var address = CreateGrainAddress(CreateGrainId(), port: 11111); + cache.AddOrUpdate(address, version: 1); + var entry = GetEntry(entrySource, address.GrainId); + var targetReference = BindMessageTarget(entry); + + await disposableCache.DisposeAsync(); + + AssertInvalidEntry(entry); + AssertEventuallyCollected(targetReference); + } + [Fact] public void CreateGrainDirectoryCache_HighCardinalityHandlesRemainBoundedAndReleaseObjectGraphs() { From bad19e7742e1a793e51f7423cb0423143e0bbb0a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 29 Aug 2026 03:20:38 -0700 Subject: [PATCH 21/21] test(runtime): clarify latency samples and GC checks --- test/Benchmarks/Ping/AdaptivePingBenchmark.cs | 4 +-- .../GrainDirectoryCacheFactoryTests.cs | 25 ++++++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/test/Benchmarks/Ping/AdaptivePingBenchmark.cs b/test/Benchmarks/Ping/AdaptivePingBenchmark.cs index cac3740db4b..85f1251b49e 100644 --- a/test/Benchmarks/Ping/AdaptivePingBenchmark.cs +++ b/test/Benchmarks/Ping/AdaptivePingBenchmark.cs @@ -353,7 +353,7 @@ private static async Task> MeasureDeterministicScenari Console.WriteLine( $"{benchmark.Description}, concurrency {concurrency}: " + $"P50 {result.P50Throughput:N0}/s, P95 {result.P95Throughput:N0}/s, " - + $"latency {result.LatencyP50Microseconds:N1}/{result.LatencyP95Microseconds:N1}/{result.LatencyP99Microseconds:N1} us"); + + $"first-completion sample {result.LatencyP50Microseconds:N1}/{result.LatencyP95Microseconds:N1}/{result.LatencyP99Microseconds:N1} us"); } } finally @@ -369,7 +369,7 @@ private static void PrintDeterministicResults(List results) Console.WriteLine(); Console.WriteLine("## Deterministic Ping Benchmark Results"); Console.WriteLine(); - Console.WriteLine("| Scenario | Concurrency | P50 throughput | P95 throughput | P99 throughput | Latency P50/P95/P99 (us) | B/request | Gen0/M requests | CPU | Contentions/M requests |"); + Console.WriteLine("| Scenario | Concurrency | P50 throughput | P95 throughput | P99 throughput | First-completion sample P50/P95/P99 (us) | B/request | Gen0/M requests | CPU | Contentions/M requests |"); Console.WriteLine("|----------|------------:|---------------:|---------------:|---------------:|-------------------------:|----------:|----------------:|----:|-----------------------:|"); foreach (var result in results) { diff --git a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs index f01295c794a..8ce9cafa67d 100644 --- a/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs +++ b/test/Orleans.Runtime.Tests/Directories/GrainDirectoryCacheFactoryTests.cs @@ -831,11 +831,9 @@ private static ( [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private static void AssertEventuallyCollected(WeakReference targetReference) { - for (var attempt = 0; attempt < 3; attempt++) + for (var attempt = 0; attempt < 5; attempt++) { - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); + ForceFullCollection(); if (!targetReference.TryGetTarget(out _)) { @@ -849,11 +847,9 @@ private static void AssertEventuallyCollected(WeakReference targe [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private static void AssertEventuallyCollected(WeakReference entryReference) { - for (var attempt = 0; attempt < 3; attempt++) + for (var attempt = 0; attempt < 5; attempt++) { - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); + ForceFullCollection(); if (!entryReference.TryGetTarget(out _)) { @@ -880,14 +876,19 @@ private static void AssertEventuallyCollected(WeakReference[] targetRefe private static void CollectGarbage() { - for (var attempt = 0; attempt < 3; attempt++) + for (var attempt = 0; attempt < 5; attempt++) { - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); + ForceFullCollection(); } } + private static void ForceFullCollection() + { + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, blocking: true, compacting: true); + GC.WaitForPendingFinalizers(); + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, blocking: true, compacting: true); + } + private static GrainId CreateGrainId() => GrainId.Parse($"user/{Guid.NewGuid():N}"); private static IGrainContext CreateMessageTarget() => DispatchProxy.Create();