From 310c127a5f50f513d33435944d07e5457be68884 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 12 Feb 2026 21:22:10 -0800 Subject: [PATCH 1/5] Fix activation-count placement fallback with partial stats --- .../ActivationCountPlacementDirector.cs | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs b/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs index d2fb585f2a7..8c5dda029a8 100644 --- a/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs +++ b/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs @@ -25,7 +25,6 @@ private class CachedLocalStat // Track created activations on this silo between statistic intervals. private readonly ConcurrentDictionary _localCache = new(); - private readonly SiloAddress _localAddress; private readonly int _chooseHowMany; public ActivationCountPlacementDirector( @@ -33,7 +32,7 @@ public ActivationCountPlacementDirector( DeploymentLoadPublisher deploymentLoadPublisher, IOptions options) { - _localAddress = localSiloDetails.SiloAddress; + _ = localSiloDetails; _chooseHowMany = options.Value.ChooseOutOf; if (_chooseHowMany <= 0) throw new ArgumentException($"{nameof(ActivationCountBasedPlacementOptions)}.{nameof(ActivationCountBasedPlacementOptions.ChooseOutOf)} is {_chooseHowMany}. It must be greater than zero."); deploymentLoadPublisher?.SubscribeToStatisticsChangeEvents(this); @@ -41,18 +40,29 @@ public ActivationCountPlacementDirector( private SiloAddress SelectSiloPowerOfK(SiloAddress[] silos) { - var compatibleSilos = silos.ToSet(); - // Exclude overloaded and non-compatible silos var relevantSilos = new List>(); - var totalSilos = 0; - foreach (var kv in _localCache) + var totalSilos = _localCache.Count; + var compatibleSilosWithStats = 0; + var compatibleSilosWithoutStats = 0; + SiloAddress randomCompatibleSiloWithoutStats = default; + foreach (var silo in silos) { - totalSilos++; - if (kv.Value.SiloStats.IsOverloaded) continue; - if (!compatibleSilos.Contains(kv.Key)) continue; + if (!_localCache.TryGetValue(silo, out var localSiloStat)) + { + compatibleSilosWithoutStats++; + if (Random.Shared.Next(compatibleSilosWithoutStats) == 0) + { + randomCompatibleSiloWithoutStats = silo; + } + + continue; + } + + compatibleSilosWithStats++; + if (localSiloStat.SiloStats.IsOverloaded) continue; - relevantSilos.Add(kv); + relevantSilos.Add(new(silo, localSiloStat)); } if (relevantSilos.Count > 0) @@ -87,6 +97,18 @@ private SiloAddress SelectSiloPowerOfK(SiloAddress[] silos) return minLoadedSilo.Key; } + // If there are no stats for any compatible silos, fall back to random placement. + if (compatibleSilosWithStats == 0) + { + return silos[Random.Shared.Next(silos.Length)]; + } + + // Some compatible silos might not have published statistics yet. + if (compatibleSilosWithoutStats > 0) + { + return randomCompatibleSiloWithoutStats; + } + // There are no compatible, non-overloaded silos. var all = _localCache.ToList(); throw new SiloUnavailableException($"Unable to select a candidate from {all.Count} silos: {Utils.EnumerableToString(all, kvp => $"SiloAddress = {kvp.Key} -> {kvp.Value}")}"); @@ -104,10 +126,10 @@ private SiloAddress OnAddActivationInternal(PlacementTarget target, IPlacementCo return placementHint; } - // If the cache was not populated, just place locally + // If there are no statistics yet, fall back to random placement. if (_localCache.IsEmpty) { - return _localAddress; + return compatibleSilos[Random.Shared.Next(compatibleSilos.Length)]; } return SelectSiloPowerOfK(compatibleSilos); From c4f2343202dd6b489acb415122bb82e3060c24e8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 12 Feb 2026 21:57:29 -0800 Subject: [PATCH 2/5] Restore local fallback when placement stats cache is empty --- .../Placement/ActivationCountPlacementDirector.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs b/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs index 8c5dda029a8..8e02ba8b405 100644 --- a/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs +++ b/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs @@ -25,6 +25,7 @@ private class CachedLocalStat // Track created activations on this silo between statistic intervals. private readonly ConcurrentDictionary _localCache = new(); + private readonly SiloAddress _localAddress; private readonly int _chooseHowMany; public ActivationCountPlacementDirector( @@ -32,7 +33,7 @@ public ActivationCountPlacementDirector( DeploymentLoadPublisher deploymentLoadPublisher, IOptions options) { - _ = localSiloDetails; + _localAddress = localSiloDetails.SiloAddress; _chooseHowMany = options.Value.ChooseOutOf; if (_chooseHowMany <= 0) throw new ArgumentException($"{nameof(ActivationCountBasedPlacementOptions)}.{nameof(ActivationCountBasedPlacementOptions.ChooseOutOf)} is {_chooseHowMany}. It must be greater than zero."); deploymentLoadPublisher?.SubscribeToStatisticsChangeEvents(this); @@ -126,10 +127,10 @@ private SiloAddress OnAddActivationInternal(PlacementTarget target, IPlacementCo return placementHint; } - // If there are no statistics yet, fall back to random placement. + // If the cache was not populated, just place locally. if (_localCache.IsEmpty) { - return compatibleSilos[Random.Shared.Next(compatibleSilos.Length)]; + return _localAddress; } return SelectSiloPowerOfK(compatibleSilos); From 2f1702420cae047067a1d704b4cd1875e6c21b33 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 13 Feb 2026 10:22:57 -0800 Subject: [PATCH 3/5] Simplify activation-count fallback control flow --- .../ActivationCountPlacementDirector.cs | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs b/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs index 8e02ba8b405..7a6e4f3aae0 100644 --- a/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs +++ b/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs @@ -41,26 +41,29 @@ public ActivationCountPlacementDirector( private SiloAddress SelectSiloPowerOfK(SiloAddress[] silos) { + if (silos.Length == 0) + { + throw new SiloUnavailableException("Unable to select a candidate because there are no compatible silos."); + } + // Exclude overloaded and non-compatible silos var relevantSilos = new List>(); var totalSilos = _localCache.Count; - var compatibleSilosWithStats = 0; var compatibleSilosWithoutStats = 0; - SiloAddress randomCompatibleSiloWithoutStats = default; + SiloAddress sampledCompatibleSiloWithoutStats = default; foreach (var silo in silos) { if (!_localCache.TryGetValue(silo, out var localSiloStat)) { compatibleSilosWithoutStats++; - if (Random.Shared.Next(compatibleSilosWithoutStats) == 0) + if (compatibleSilosWithoutStats == 1 || Random.Shared.Next(compatibleSilosWithoutStats) == 0) { - randomCompatibleSiloWithoutStats = silo; + sampledCompatibleSiloWithoutStats = silo; } continue; } - compatibleSilosWithStats++; if (localSiloStat.SiloStats.IsOverloaded) continue; relevantSilos.Add(new(silo, localSiloStat)); @@ -98,21 +101,15 @@ private SiloAddress SelectSiloPowerOfK(SiloAddress[] silos) return minLoadedSilo.Key; } - // If there are no stats for any compatible silos, fall back to random placement. - if (compatibleSilosWithStats == 0) - { - return silos[Random.Shared.Next(silos.Length)]; - } - // Some compatible silos might not have published statistics yet. if (compatibleSilosWithoutStats > 0) { - return randomCompatibleSiloWithoutStats; + return sampledCompatibleSiloWithoutStats; } - // There are no compatible, non-overloaded silos. + // All compatible silos have published stats and are overloaded. var all = _localCache.ToList(); - throw new SiloUnavailableException($"Unable to select a candidate from {all.Count} silos: {Utils.EnumerableToString(all, kvp => $"SiloAddress = {kvp.Key} -> {kvp.Value}")}"); + throw new SiloUnavailableException($"Unable to select a candidate from {silos.Length} compatible silos: {Utils.EnumerableToString(all, kvp => $"SiloAddress = {kvp.Key} -> {kvp.Value}")}"); } public override Task OnAddActivation(PlacementStrategy strategy, PlacementTarget target, IPlacementContext context) => Task.FromResult(OnAddActivationInternal(target, context)); From 93e78df22ecff3544f95ded5c366925fd41745f4 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 13 Feb 2026 11:29:07 -0800 Subject: [PATCH 4/5] Fix activation-count placement local compatibility --- .../ActivationCountPlacementDirector.cs | 12 ++++- .../ActivationCountPlacementDirectorTests.cs | 54 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 test/NonSilo.Tests/Runtime/ActivationCountPlacementDirectorTests.cs diff --git a/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs b/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs index 7a6e4f3aae0..b80815650db 100644 --- a/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs +++ b/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs @@ -124,10 +124,18 @@ private SiloAddress OnAddActivationInternal(PlacementTarget target, IPlacementCo return placementHint; } - // If the cache was not populated, just place locally. + // If the cache was not populated, place locally only if this silo is compatible. if (_localCache.IsEmpty) { - return _localAddress; + foreach (var silo in compatibleSilos) + { + if (silo.Equals(_localAddress)) + { + return _localAddress; + } + } + + return SelectSiloPowerOfK(compatibleSilos); } return SelectSiloPowerOfK(compatibleSilos); diff --git a/test/NonSilo.Tests/Runtime/ActivationCountPlacementDirectorTests.cs b/test/NonSilo.Tests/Runtime/ActivationCountPlacementDirectorTests.cs new file mode 100644 index 00000000000..26344f1e0a1 --- /dev/null +++ b/test/NonSilo.Tests/Runtime/ActivationCountPlacementDirectorTests.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.Options; +using NSubstitute; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Placement; +using Xunit; + +namespace UnitTests.Runtime +{ + [TestCategory("BVT"), TestCategory("Placement")] + public class ActivationCountPlacementDirectorTests + { + [Fact] + public async Task OnAddActivation_WhenCacheIsEmptyAndLocalSiloIsIncompatible_PlacesOnCompatibleSilo() + { + var localSilo = Silo("127.0.0.1:100@1"); + var compatibleSilo = Silo("127.0.0.1:101@1"); + var director = CreateDirector(localSilo); + var placementContext = Substitute.For(); + placementContext.GetCompatibleSilos(Arg.Any()).Returns([compatibleSilo]); + + var result = await director.OnAddActivation(strategy: null!, target: default, placementContext); + + Assert.Equal(compatibleSilo, result); + } + + [Fact] + public async Task OnAddActivation_WhenCacheIsEmptyAndLocalSiloIsCompatible_PlacesLocally() + { + var localSilo = Silo("127.0.0.1:100@1"); + var compatibleSilo = Silo("127.0.0.1:101@1"); + var director = CreateDirector(localSilo); + var placementContext = Substitute.For(); + placementContext.GetCompatibleSilos(Arg.Any()).Returns([compatibleSilo, localSilo]); + + var result = await director.OnAddActivation(strategy: null!, target: default, placementContext); + + Assert.Equal(localSilo, result); + } + + private static ActivationCountPlacementDirector CreateDirector(SiloAddress localSilo) + { + var localSiloDetails = Substitute.For(); + localSiloDetails.SiloAddress.Returns(localSilo); + + return new ActivationCountPlacementDirector( + localSiloDetails, + deploymentLoadPublisher: null!, + Options.Create(new ActivationCountBasedPlacementOptions())); + } + + private static SiloAddress Silo(string value) => SiloAddress.FromParsableString(value); + } +} From 14c71157c18c88729bc7c2c821dae938cb8bde8e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 13 Feb 2026 13:15:36 -0800 Subject: [PATCH 5/5] Address PR review feedback --- .../ActivationCountPlacementDirector.cs | 12 ++- .../ActivationCountPlacementDirectorTests.cs | 79 +++++++++++++++++++ 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs b/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs index b80815650db..43b96f2db7e 100644 --- a/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs +++ b/src/Orleans.Runtime/Placement/ActivationCountPlacementDirector.cs @@ -108,8 +108,9 @@ private SiloAddress SelectSiloPowerOfK(SiloAddress[] silos) } // All compatible silos have published stats and are overloaded. - var all = _localCache.ToList(); - throw new SiloUnavailableException($"Unable to select a candidate from {silos.Length} compatible silos: {Utils.EnumerableToString(all, kvp => $"SiloAddress = {kvp.Key} -> {kvp.Value}")}"); + var allSiloStats = _localCache.ToList(); + throw new SiloUnavailableException( + $"Unable to select a candidate from {silos.Length} compatible silos (all are overloaded). All silo stats: {Utils.EnumerableToString(allSiloStats, kvp => $"SiloAddress = {kvp.Key} -> IsOverloaded = {kvp.Value.SiloStats.IsOverloaded}, ActivationCount = {kvp.Value.ActivationCount}, RecentlyUsedActivationCount = {kvp.Value.SiloStats.RecentlyUsedActivationCount}")}"); } public override Task OnAddActivation(PlacementStrategy strategy, PlacementTarget target, IPlacementContext context) => Task.FromResult(OnAddActivationInternal(target, context)); @@ -127,12 +128,9 @@ private SiloAddress OnAddActivationInternal(PlacementTarget target, IPlacementCo // If the cache was not populated, place locally only if this silo is compatible. if (_localCache.IsEmpty) { - foreach (var silo in compatibleSilos) + if (compatibleSilos.Contains(_localAddress)) { - if (silo.Equals(_localAddress)) - { - return _localAddress; - } + return _localAddress; } return SelectSiloPowerOfK(compatibleSilos); diff --git a/test/NonSilo.Tests/Runtime/ActivationCountPlacementDirectorTests.cs b/test/NonSilo.Tests/Runtime/ActivationCountPlacementDirectorTests.cs index 26344f1e0a1..6892c0ab08a 100644 --- a/test/NonSilo.Tests/Runtime/ActivationCountPlacementDirectorTests.cs +++ b/test/NonSilo.Tests/Runtime/ActivationCountPlacementDirectorTests.cs @@ -1,8 +1,10 @@ +using System; using Microsoft.Extensions.Options; using NSubstitute; using Orleans.Configuration; using Orleans.Runtime; using Orleans.Runtime.Placement; +using Orleans.Statistics; using Xunit; namespace UnitTests.Runtime @@ -38,6 +40,50 @@ public async Task OnAddActivation_WhenCacheIsEmptyAndLocalSiloIsCompatible_Place Assert.Equal(localSilo, result); } + [Fact] + public async Task OnAddActivation_WhenSomeCompatibleSilosHaveNoStats_PrefersSilosWithStats() + { + var localSilo = Silo("127.0.0.1:100@1"); + var siloWithStats = Silo("127.0.0.1:101@1"); + var siloWithoutStats = Silo("127.0.0.1:102@1"); + var director = CreateDirector(localSilo); + var placementContext = CreatePlacementContext(siloWithStats, siloWithoutStats); + director.SiloStatisticsChangeNotification(siloWithStats, CreateSiloRuntimeStatistics(overloaded: false, recentlyUsedActivationCount: 10)); + + var result = await director.OnAddActivation(strategy: null!, target: default, placementContext); + + Assert.Equal(siloWithStats, result); + } + + [Fact] + public async Task OnAddActivation_WhenAllCompatibleSilosWithStatsAreOverloaded_Throws() + { + var localSilo = Silo("127.0.0.1:100@1"); + var overloadedSilo1 = Silo("127.0.0.1:101@1"); + var overloadedSilo2 = Silo("127.0.0.1:102@1"); + var director = CreateDirector(localSilo); + var placementContext = CreatePlacementContext(overloadedSilo1, overloadedSilo2); + director.SiloStatisticsChangeNotification(overloadedSilo1, CreateSiloRuntimeStatistics(overloaded: true)); + director.SiloStatisticsChangeNotification(overloadedSilo2, CreateSiloRuntimeStatistics(overloaded: true)); + + await Assert.ThrowsAsync(() => director.OnAddActivation(strategy: null!, target: default, placementContext)); + } + + [Fact] + public async Task OnAddActivation_WhenSilosWithStatsAreOverloadedAndWithoutStatsExist_FallsBackToWithoutStats() + { + var localSilo = Silo("127.0.0.1:100@1"); + var overloadedSilo = Silo("127.0.0.1:101@1"); + var siloWithoutStats = Silo("127.0.0.1:102@1"); + var director = CreateDirector(localSilo); + var placementContext = CreatePlacementContext(overloadedSilo, siloWithoutStats); + director.SiloStatisticsChangeNotification(overloadedSilo, CreateSiloRuntimeStatistics(overloaded: true)); + + var result = await director.OnAddActivation(strategy: null!, target: default, placementContext); + + Assert.Equal(siloWithoutStats, result); + } + private static ActivationCountPlacementDirector CreateDirector(SiloAddress localSilo) { var localSiloDetails = Substitute.For(); @@ -49,6 +95,39 @@ private static ActivationCountPlacementDirector CreateDirector(SiloAddress local Options.Create(new ActivationCountBasedPlacementOptions())); } + private static IPlacementContext CreatePlacementContext(params SiloAddress[] compatibleSilos) + { + var placementContext = Substitute.For(); + placementContext.GetCompatibleSilos(Arg.Any()).Returns(compatibleSilos); + return placementContext; + } + + private static SiloRuntimeStatistics CreateSiloRuntimeStatistics(bool overloaded, int recentlyUsedActivationCount = 0) + { + var environmentStatisticsProvider = Substitute.For(); + var maxMemoryBytes = 1000L; + var memoryUsageBytes = overloaded ? 950L : 100L; + var availableMemoryBytes = maxMemoryBytes - memoryUsageBytes; + var cpuUsagePercentage = overloaded ? 100f : 0f; + + environmentStatisticsProvider.GetEnvironmentStatistics().Returns( + new EnvironmentStatistics( + cpuUsagePercentage: cpuUsagePercentage, + rawCpuUsagePercentage: cpuUsagePercentage, + memoryUsageBytes: memoryUsageBytes, + rawMemoryUsageBytes: memoryUsageBytes, + availableMemoryBytes: availableMemoryBytes, + rawAvailableMemoryBytes: availableMemoryBytes, + maximumAvailableMemoryBytes: maxMemoryBytes)); + + return new SiloRuntimeStatistics( + activationCount: 0, + recentlyUsedActivationCount: recentlyUsedActivationCount, + environmentStatisticsProvider, + Options.Create(new LoadSheddingOptions { LoadSheddingEnabled = overloaded }), + DateTime.UtcNow); + } + private static SiloAddress Silo(string value) => SiloAddress.FromParsableString(value); } }