Fixed: Akka.Cluster.Singleton no longer moves when node with matching role and higher AppVersion joins cluster

src/contrib/cluster/Akka.Cluster.Tools.Tests/Singleton/BugFix7196Specs.cs

@@ -0,0 +1,164 @@
+// -----------------------------------------------------------------------
+//  <copyright file="BugFix7196Specs.cs" company="Akka.NET Project">
+//      Copyright (C) 2009-2024 Lightbend Inc. <http://www.lightbend.com>
+//      Copyright (C) 2013-2024 .NET Foundation <https://github.com/akkadotnet/akka.net>
+//  </copyright>
+// -----------------------------------------------------------------------
+
+#nullable enable
+using System;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Threading.Tasks;
+using Akka.Actor;
+using Akka.Cluster.Tools.Singleton;
+using Akka.Configuration;
+using Akka.TestKit;
+using FluentAssertions;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Akka.Cluster.Tools.Tests.Singleton;
+
+/// <summary>
+/// Reproduction for https://github.com/akkadotnet/akka.net/issues/7196 - clearly, what we did
+/// </summary>
+public class BugFix7196Specs : AkkaSpec
+{
+    private readonly ActorSystem _hostNodeV1;
+    private readonly ActorSystem _hostNode2V1;
+    private readonly ActorSystem _hostNodeV2;
+
+    private static Config OriginalNodeConfig() => """
+
+                                                                akka.loglevel = INFO
+                                                                akka.actor.provider = "cluster"
+                                                                akka.cluster.roles = [non-singleton]
+                                                                akka.cluster.singleton.min-number-of-hand-over-retries = 5
+                                                                akka.cluster.app-version = "1.0.0"
+                                                                akka.remote {
+                                                                  dot-netty.tcp {
+                                                                    hostname = "127.0.0.1"
+                                                                    port = 0
+                                                                  }
+                                                                }
+                                                  """;
+
+    private static Config V2NodeConfig(ActorSystem originalSys) => ConfigurationFactory.ParseString(
+        "akka.cluster.app-version = \"1.0.2\"").WithFallback(originalSys.Settings.Config);
+
+    public BugFix7196Specs(ITestOutputHelper output) : base(OriginalNodeConfig(), output)
+    {
+        _hostNodeV1 = ActorSystem.Create(Sys.Name,
+            ConfigurationFactory.ParseString("akka.cluster.roles = [singleton]").WithFallback(Sys.Settings.Config));
+        InitializeLogger(_hostNodeV1);
+        _hostNode2V1 = ActorSystem.Create(Sys.Name,
+            ConfigurationFactory.ParseString("akka.cluster.roles = [singleton]").WithFallback(Sys.Settings.Config));
+        InitializeLogger(_hostNode2V1);
+        _hostNodeV2 = ActorSystem.Create(Sys.Name,
+            ConfigurationFactory.ParseString("akka.cluster.roles = [singleton]").WithFallback(V2NodeConfig(Sys)));
+        InitializeLogger(_hostNodeV2);
+    }
+
+    [Fact(DisplayName =
+        "Singletons should not move to higher AppVersion nodes until after older incarnation is downed")]
+    public async Task Bugfix7196Spec()
+    {
+        await JoinAsync(Sys, Sys); // have to do a self join first
+        await JoinAsync(_hostNodeV1, Sys);
+        await JoinAsync(_hostNode2V1, Sys);
+
+        var proxy = Sys.ActorOf(
+            ClusterSingletonProxy.Props("user/echo",
+                ClusterSingletonProxySettings.Create(Sys).WithRole("singleton")), "proxy3");
+
+        // confirm that singleton is on _hostNodeV1
+        await AssertSingletonHostedOn(proxy, _hostNodeV1);
+
+        // have _hostNodeV2 join the cluster
+        await JoinAsync(_hostNodeV2, Sys);
+
+        // confirm that singleton is STILL on _hostNodeV1
+        await AssertSingletonHostedOn(proxy, _hostNodeV1);
+
+        // now, down the original node
+        Cluster.Get(Sys).Leave(Cluster.Get(_hostNodeV1).SelfAddress);
+
+        // validate that _hostNodeV1 is no longer in the cluster
+        await WithinAsync(TimeSpan.FromSeconds(5), () =>
+        {
+            return AwaitAssertAsync(() =>
+            {
+                Cluster.Get(Sys).State.Members.Select(x => x.UniqueAddress).Should()
+                    .NotContain(Cluster.Get(_hostNodeV1).SelfUniqueAddress);
+            });
+        });
+
+        // validate that the singleton has moved to _hostNodeV2
+        await AssertSingletonHostedOn(proxy, _hostNodeV2);
+
+        /*
+         * NOTE: an important detail here: _hostNode2V1 is actually OLDER than _hostNodeV2,
+         * but when selecting a new "oldest" node after the previous one dies Akka.Cluster.Tools.Singleton
+         * will always prefer to move onto the new version of the software.
+         */
+    }
+
+    private async Task AssertSingletonHostedOn(IActorRef proxy, ActorSystem targetNode)
+    {
+        await WithinAsync(TimeSpan.FromSeconds(5), () =>
+        {
+            return AwaitAssertAsync(() =>
+            {
+                var probe = CreateTestProbe(Sys);
+                proxy.Tell("hello", probe.Ref);
+                probe.ExpectMsg<UniqueAddress>(TimeSpan.FromSeconds(1))
+                    .Should()
+                    .Be(Cluster.Get(targetNode).SelfUniqueAddress);
+            });
+        });
+    }
+
+    public async Task JoinAsync(ActorSystem from, ActorSystem to)
+    {
+        if (Cluster.Get(from).SelfRoles.Contains("singleton"))
+        {
+            from.ActorOf(ClusterSingletonManager.Props(Props.Create(() => new Singleton()),
+                PoisonPill.Instance,
+                ClusterSingletonManagerSettings.Create(from).WithRole("singleton")), "echo");
+        }
+
+
+        await WithinAsync(TimeSpan.FromSeconds(45), () =>
+        {
+            AwaitAssert(() =>
+            {
+                Cluster.Get(from).Join(Cluster.Get(to).SelfAddress);
+                Cluster.Get(from).State.Members.Select(x => x.UniqueAddress).Should()
+                    .Contain(Cluster.Get(from).SelfUniqueAddress);
+                Cluster.Get(from)
+                    .State.Members.Select(x => x.Status)
+                    .ToImmutableHashSet()
+                    .Should()
+                    .Equal(ImmutableHashSet<MemberStatus>.Empty.Add(MemberStatus.Up));
+            });
+            return Task.CompletedTask;
+        });
+    }
+
+    public class Singleton : ReceiveActor
+    {
+        public Singleton()
+        {
+            ReceiveAny(_ => { Sender.Tell(Cluster.Get(Context.System).SelfUniqueAddress); });
+        }
+    }
+
+    protected override void AfterAll()
+    {
+        Shutdown(_hostNodeV1);
+        Shutdown(_hostNode2V1);
+        Shutdown(_hostNodeV2);
+        base.AfterAll();
+    }
+}

src/contrib/cluster/Akka.Cluster.Tools.Tests/Singleton/ClusterSingletonConfigSpec.cs

@@ -42,6 +42,7 @@ public void ClusterSingletonManagerSettings_must_have_default_config()
             var config = Sys.Settings.Config.GetConfig("akka.cluster.singleton");
             Assert.False(config.IsNullOrEmpty());
             config.GetInt("min-number-of-hand-over-retries", 0).ShouldBe(15);
+            clusterSingletonManagerSettings.ConsiderAppVersion.ShouldBeTrue();
         }
 
         [Fact]
@@ -54,6 +55,7 @@ public void ClusterSingletonProxySettings_must_have_default_config()
             clusterSingletonProxySettings.Role.ShouldBe(null);
             clusterSingletonProxySettings.SingletonIdentificationInterval.TotalSeconds.ShouldBe(1);
             clusterSingletonProxySettings.BufferSize.ShouldBe(1000);
+            clusterSingletonProxySettings.ConsiderAppVersion.ShouldBeTrue();
         }
     }
 }

src/contrib/cluster/Akka.Cluster.Tools.Tests/Singleton/ClusterSingletonRestart2Spec.cs

@@ -29,7 +29,6 @@ public ClusterSingletonRestart2Spec() : base(@"
               akka.loglevel = INFO
               akka.actor.provider = ""cluster""
               akka.cluster.roles = [singleton]
-              akka.cluster.auto-down-unreachable-after = 2s
               akka.cluster.singleton.min-number-of-hand-over-retries = 5
               akka.remote {
                 dot-netty.tcp {

src/contrib/cluster/Akka.Cluster.Tools.Tests/Singleton/ClusterSingletonRestart3Spec.cs

@@ -29,7 +29,6 @@ public ClusterSingletonRestart3Spec(ITestOutputHelper output) : base(@"
               akka.loglevel = DEBUG
               akka.actor.provider = ""cluster""
               akka.cluster.app-version = ""1.0.0""
-              akka.cluster.auto-down-unreachable-after = 2s
               akka.cluster.singleton.min-number-of-hand-over-retries = 5
               akka.cluster.singleton.consider-app-version = true
               akka.remote {

src/contrib/cluster/Akka.Cluster.Tools.Tests/Singleton/ClusterSingletonRestartSpec.cs

@@ -28,7 +28,6 @@ public class ClusterSingletonRestartSpec : AkkaSpec
         public ClusterSingletonRestartSpec() : base(@"
               akka.loglevel = INFO
               akka.actor.provider = ""cluster""
-              akka.cluster.auto-down-unreachable-after = 2s
               akka.remote {
                 dot-netty.tcp {
                   hostname = ""127.0.0.1""

src/contrib/cluster/Akka.Cluster.Tools.Tests/Singleton/MemberAgeOrderingSpec.cs

@@ -21,7 +21,24 @@ public class MemberAgeOrderingSpec
         [Fact(DisplayName = "MemberAgeOrdering should sort based on UpNumber")]
         public void SortByUpNumberTest()
         {
-            var members = new SortedSet<Member>(MemberAgeOrdering.DescendingWithAppVersion)
+            var members = new SortedSet<Member>(MemberAgeOrdering.OldestToYoungest)
+            {
+                Create(Address.Parse("akka://sys@darkstar:1112"), upNumber: 3),
+                Create(Address.Parse("akka://sys@darkstar:1113"), upNumber: 1),
+                Create(Address.Parse("akka://sys@darkstar:1111"), upNumber: 9),
+            };
+
+            var seq = members.ToList();
+            seq.Count.Should().Be(3);
+            seq[0].Should().Be(Create(Address.Parse("akka://sys@darkstar:1113"), upNumber: 1));
+            seq[1].Should().Be(Create(Address.Parse("akka://sys@darkstar:1112"), upNumber: 3));
+            seq[2].Should().Be(Create(Address.Parse("akka://sys@darkstar:1111"), upNumber: 9));
+        }
+
+        [Fact(DisplayName = "MemberAgeOrdering should sort based on UpNumber and AppVersion")]
+        public void SortByUpNumberAndAppVersionTest()
+        {
+            var members = new SortedSet<Member>(MemberAgeOrdering.OldestToYoungestWithAppVersion)
             {
                 Create(Address.Parse("akka://sys@darkstar:1112"), upNumber: 3),
                 Create(Address.Parse("akka://sys@darkstar:1113"), upNumber: 1),
@@ -38,7 +55,7 @@ public void SortByUpNumberTest()
         [Fact(DisplayName = "MemberAgeOrdering should sort based on Address if UpNumber is the same")]
         public void SortByAddressTest()
         {
-            var members = new SortedSet<Member>(MemberAgeOrdering.DescendingWithAppVersion)
+            var members = new SortedSet<Member>(MemberAgeOrdering.OldestToYoungestWithAppVersion)
             {
                 Create(Address.Parse("akka://sys@darkstar:1112"), upNumber: 1),
                 Create(Address.Parse("akka://sys@darkstar:1113"), upNumber: 1),
@@ -55,7 +72,7 @@ public void SortByAddressTest()
         [Fact(DisplayName = "MemberAgeOrdering should prefer AppVersion over UpNumber")]
         public void SortByAppVersionTest()
         {
-            var members = new SortedSet<Member>(MemberAgeOrdering.DescendingWithAppVersion)
+            var members = new SortedSet<Member>(MemberAgeOrdering.OldestToYoungestWithAppVersion)
             {
                 Create(Address.Parse("akka://sys@darkstar:1112"), upNumber: 3, appVersion: AppVersion.Create("1.0.0")),
                 Create(Address.Parse("akka://sys@darkstar:1113"), upNumber: 1, appVersion: AppVersion.Create("1.0.0")),