Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions build-system/pr-validation.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,10 @@ jobs:
artifactName: "net_mntr_windows-$(Build.BuildId)"
mntrFailuresDir: 'TestResults\\multinode'
mntrFailuresArtifactName: "net_7_mntr_FAILED_windows-$(Build.BuildId)"
env:
# 2-vCPU hosted agents can't comfortably run StressSpec's 10-node default; 7 is the
# smallest count StressSpecConfig.BuildConfig can still fit (see its doc comment).
MNTR_STRESSSPEC_NODECOUNT: "7"

# First-ever Linux MNTR lane: same suite, same incrementalist scoping and
# command as "net_mntr_windows" above, on ubuntu-latest (matching the Linux
Expand All @@ -129,6 +133,10 @@ jobs:
mntrFailuresDir: 'TestResults/multinode'
mntrFailuresArtifactName: "net_mntr_FAILED_linux-$(Build.BuildId)"
continueOnError: true
env:
# 2-vCPU hosted agents can't comfortably run StressSpec's 10-node default; 7 is the
# smallest count StressSpecConfig.BuildConfig can still fit (see its doc comment).
MNTR_STRESSSPEC_NODECOUNT: "7"

# Independent, parallel, NON-BLOCKING Artery MNTR stage. Runs the exact same
# incrementally-scoped multi-node suite as "net_mntr_windows" above, but
Expand Down Expand Up @@ -167,6 +175,9 @@ jobs:
continueOnError: true
env:
AKKA_MNTR_TRANSPORT: artery
# 2-vCPU hosted agents can't comfortably run StressSpec's 10-node default; 7 is the
# smallest count StressSpecConfig.BuildConfig can still fit (see its doc comment).
MNTR_STRESSSPEC_NODECOUNT: "7"

# Fourth lane, completing the classic/Artery x Windows/Linux MNTR matrix
# (only classic Windows is blocking). Same rationale and known/accepted
Expand All @@ -184,6 +195,9 @@ jobs:
continueOnError: true
env:
AKKA_MNTR_TRANSPORT: artery
# 2-vCPU hosted agents can't comfortably run StressSpec's 10-node default; 7 is the
# smallest count StressSpecConfig.BuildConfig can still fit (see its doc comment).
MNTR_STRESSSPEC_NODECOUNT: "7"

- template: azure-pipeline.template.yaml
parameters:
Expand Down
42 changes: 34 additions & 8 deletions src/core/Akka.Cluster.Tests.MultiNode/StressSpec.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,35 @@ public StressSpecConfig()
foreach (var i in Enumerable.Range(1, TotalNumberOfNodes))
Role("node-" + i);

CommonConfig = ConfigurationFactory.ParseString(@"
CommonConfig = ConfigurationFactory.ParseString(BuildConfig(TotalNumberOfNodes));

TestTransport = true;
}

/// <summary>
/// Builds the `akka.test.cluster-stress-spec` (plus supporting actor/remote) config for a run
/// of <paramref name="totalNumberOfNodes"/> nodes.
///
/// The reference node count (matching Pekko's stress spec) is 10, and with the full 10-node
/// `nr-of-nodes-*` defaults below, 10 also happens to be the *smallest* count that fits every
/// phase -- see the arithmetic in <see cref="Settings"/>'s constructor. The joining phases need
/// >= 7 nodes on their own (3 seed nodes + 4 singleton join phases), and the leaving/shutdown
/// phases together remove 7 nodes' worth of `nr-of-nodes-*`, which requires
/// `totalNumberOfNodes - 3 >= 7`, i.e. `totalNumberOfNodes >= 10`. So on the 2-vCPU hosted CI
/// agents, lowering `MNTR_STRESSSPEC_NODECOUNT` below 10 by itself is not enough -- <see cref="Settings"/>
/// throws unless the phase counts shrink too. Below 10 nodes, this method drops the two
/// "-large" one-by-one leave/shutdown phases (each mostly redundant with the "-small" one-by-one
/// phase that already covers that code path, just at a different point in the cluster's
/// lifecycle) and halves the simultaneous "shutdown" phase from 2 to 1, freeing exactly the 3
/// nodes needed to fit a 7-node run.
/// </summary>
internal static string BuildConfig(int totalNumberOfNodes)
{
var leavingOneByOneLarge = totalNumberOfNodes < 10 ? 0 : 1;
var shutdownOneByOneLarge = totalNumberOfNodes < 10 ? 0 : 1;
var shutdown = totalNumberOfNodes < 10 ? 1 : 2;

return @"
akka.test.cluster-stress-spec {
infolog = on
# scale the nr-of-nodes* settings with this factor
Expand All @@ -59,12 +87,12 @@ public StressSpecConfig()
nr-of-nodes-joining-one-by-one-large = 1
nr-of-nodes-joining-to-one = 1
nr-of-nodes-leaving-one-by-one-small = 1
nr-of-nodes-leaving-one-by-one-large = 1
nr-of-nodes-leaving-one-by-one-large = " + leavingOneByOneLarge + @"
nr-of-nodes-leaving = 1
nr-of-nodes-shutdown-one-by-one-small = 1
nr-of-nodes-shutdown-one-by-one-large = 1
nr-of-nodes-shutdown-one-by-one-large = " + shutdownOneByOneLarge + @"
nr-of-nodes-partition = 2
nr-of-nodes-shutdown = 2
nr-of-nodes-shutdown = " + shutdown + @"
nr-of-nodes-join-remove = 2
# not scaled
# scale the *-duration settings with this factor
Expand All @@ -76,7 +104,7 @@ public StressSpecConfig()
convergence-within-factor = 1.0
}
akka.actor.provider = cluster

akka.cluster {
# akka.test.timefactor does NOT reach cluster settings. TestKitBase.Dilated scales TestKit
# waits only; ClusterSettings reads this value raw. So a lane that declares ""this box is 3x
Expand Down Expand Up @@ -134,9 +162,7 @@ public StressSpecConfig()
parallelism-factor = 0.5
parallelism-max = 16
}
}");

TestTransport = true;
}";
}

public class Settings
Expand Down
86 changes: 86 additions & 0 deletions src/core/Akka.Cluster.Tests.MultiNode/StressSpecConfigSpec.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
//-----------------------------------------------------------------------
// <copyright file="StressSpecConfigSpec.cs" company="Akka.NET Project">
// Copyright (C) 2009-2022 Lightbend Inc. <http://www.lightbend.com>
// Copyright (C) 2013-2026 .NET Foundation <https://github.com/akkadotnet/akka.net>
// </copyright>
//-----------------------------------------------------------------------

using System;
using Akka.Configuration;
using Xunit;

namespace Akka.Cluster.Tests.MultiNode;

/// <summary>
/// Plain (non multi-node) unit tests for <see cref="StressSpecConfig"/>'s per-phase node-count
/// arithmetic -- in particular, that <see cref="StressSpecConfig.BuildConfig"/> automatically
/// shrinks the `nr-of-nodes-*` phase counts so a reduced `MNTR_STRESSSPEC_NODECOUNT` (used on the
/// 2-vCPU hosted CI agents) still produces a consistent <see cref="StressSpecConfig.Settings"/>
/// instead of throwing.
///
/// These tests do not spin up an actor system or a multi-node run --
/// <see cref="StressSpecConfig.Settings"/> only reads plain HOCON values.
/// </summary>
public class StressSpecConfigSpec
{
private static StressSpecConfig.Settings BuildSettings(int totalNumberOfNodes)
{
var config = ConfigurationFactory.ParseString(StressSpecConfig.BuildConfig(totalNumberOfNodes));
return new StressSpecConfig.Settings(config, totalNumberOfNodes);
}

[Fact]
public void Settings_at_the_10_node_reference_count_uses_the_full_defaults()
{
var settings = BuildSettings(10);

Assert.Equal(1, settings.NumberOfNodesLeavingOneByOneLarge);
Assert.Equal(1, settings.NumberOfNodesShutdownOneByOneLarge);
Assert.Equal(2, settings.NumberOfNodesShutdown);
Assert.Equal(3, settings.NumberOfNodesJoiningToSeedNodes);
}

[Theory]
[InlineData(7)]
[InlineData(8)]
[InlineData(9)]
public void Settings_below_10_nodes_does_not_throw(int totalNumberOfNodes)
{
var settings = BuildSettings(totalNumberOfNodes);
Assert.Equal(totalNumberOfNodes, settings.TotalNumberOfNodes);
}

[Fact]
public void Settings_at_7_nodes_shrinks_the_leaving_and_shutdown_phases_to_exactly_fit()
{
var settings = BuildSettings(7);

// the two "-large" one-by-one phases are dropped, and the simultaneous "shutdown" phase
// is halved from 2 to 1, freeing exactly the 3 nodes a 7-node run needs
Assert.Equal(0, settings.NumberOfNodesLeavingOneByOneLarge);
Assert.Equal(0, settings.NumberOfNodesShutdownOneByOneLarge);
Assert.Equal(1, settings.NumberOfNodesShutdown);

// the joining phases exactly consume all 7 nodes (3 seed + 4 singleton joins), so
// there are none left over to join to the seed nodes as a batch
Assert.Equal(0, settings.NumberOfNodesJoiningToSeedNodes);

// the leaving/shutdown phases must still leave the 3 master-hosting nodes alone
var leavingAndShutdown = settings.NumberOfNodesLeavingOneByOneSmall +
settings.NumberOfNodesLeavingOneByOneLarge +
settings.NumberOfNodesLeaving +
settings.NumberOfNodesShutdownOneByOneSmall +
settings.NumberOfNodesShutdownOneByOneLarge +
settings.NumberOfNodesShutdown;
Assert.True(leavingAndShutdown <= settings.TotalNumberOfNodes - 3);
}

[Fact]
public void Settings_below_7_nodes_still_throws_because_the_joining_phases_alone_need_7()
{
// 3 seed nodes + 4 singleton join phases (joining-to-seed-initially, one-by-one-small,
// one-by-one-large, joining-to-one) need >= 7 nodes regardless of how far the
// leaving/shutdown phases are shrunk, so 7 is the practical floor.
Assert.Throws<ArgumentOutOfRangeException>(() => BuildSettings(6));
}
}
Loading