diff --git a/src/Extensions/Wolverine.DataAnnotationsValidation.Tests/Wolverine.DataAnnotationsValidation.Tests.csproj b/src/Extensions/Wolverine.DataAnnotationsValidation.Tests/Wolverine.DataAnnotationsValidation.Tests.csproj index 0aebb1feb..0b1e5de4b 100644 --- a/src/Extensions/Wolverine.DataAnnotationsValidation.Tests/Wolverine.DataAnnotationsValidation.Tests.csproj +++ b/src/Extensions/Wolverine.DataAnnotationsValidation.Tests/Wolverine.DataAnnotationsValidation.Tests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Extensions/Wolverine.FluentValidation.Tests/Wolverine.FluentValidation.Tests.csproj b/src/Extensions/Wolverine.FluentValidation.Tests/Wolverine.FluentValidation.Tests.csproj index 0284f16ea..9d01fae69 100644 --- a/src/Extensions/Wolverine.FluentValidation.Tests/Wolverine.FluentValidation.Tests.csproj +++ b/src/Extensions/Wolverine.FluentValidation.Tests/Wolverine.FluentValidation.Tests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Extensions/Wolverine.MemoryPack.Tests/Wolverine.MemoryPack.Tests.csproj b/src/Extensions/Wolverine.MemoryPack.Tests/Wolverine.MemoryPack.Tests.csproj index 4b13c46a5..789abc94e 100644 --- a/src/Extensions/Wolverine.MemoryPack.Tests/Wolverine.MemoryPack.Tests.csproj +++ b/src/Extensions/Wolverine.MemoryPack.Tests/Wolverine.MemoryPack.Tests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Extensions/Wolverine.MessagePack.Tests/Wolverine.MessagePack.Tests.csproj b/src/Extensions/Wolverine.MessagePack.Tests/Wolverine.MessagePack.Tests.csproj index b61118cb7..1cbc36506 100644 --- a/src/Extensions/Wolverine.MessagePack.Tests/Wolverine.MessagePack.Tests.csproj +++ b/src/Extensions/Wolverine.MessagePack.Tests/Wolverine.MessagePack.Tests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Extensions/Wolverine.Protobuf.Tests/Wolverine.Protobuf.Tests.csproj b/src/Extensions/Wolverine.Protobuf.Tests/Wolverine.Protobuf.Tests.csproj index fce8b1182..b8f92baf0 100644 --- a/src/Extensions/Wolverine.Protobuf.Tests/Wolverine.Protobuf.Tests.csproj +++ b/src/Extensions/Wolverine.Protobuf.Tests/Wolverine.Protobuf.Tests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Http/Wolverine.Http.AspVersioning.Tests/Wolverine.Http.AspVersioning.Tests.csproj b/src/Http/Wolverine.Http.AspVersioning.Tests/Wolverine.Http.AspVersioning.Tests.csproj index 7bcc4ebdc..a6eb9f1dd 100644 --- a/src/Http/Wolverine.Http.AspVersioning.Tests/Wolverine.Http.AspVersioning.Tests.csproj +++ b/src/Http/Wolverine.Http.AspVersioning.Tests/Wolverine.Http.AspVersioning.Tests.csproj @@ -1,7 +1,6 @@  - Exe net10.0 false diff --git a/src/Http/Wolverine.Http.Tests/Wolverine.Http.Tests.csproj b/src/Http/Wolverine.Http.Tests/Wolverine.Http.Tests.csproj index 924f055b6..dea0721a5 100644 --- a/src/Http/Wolverine.Http.Tests/Wolverine.Http.Tests.csproj +++ b/src/Http/Wolverine.Http.Tests/Wolverine.Http.Tests.csproj @@ -1,7 +1,6 @@ - Exe false net9.0 diff --git a/src/Persistence/CosmosDbTests/CosmosDbTests.csproj b/src/Persistence/CosmosDbTests/CosmosDbTests.csproj index 2130333e9..883522e96 100644 --- a/src/Persistence/CosmosDbTests/CosmosDbTests.csproj +++ b/src/Persistence/CosmosDbTests/CosmosDbTests.csproj @@ -1,7 +1,6 @@ - Exe net9.0 enable diff --git a/src/Persistence/EfCoreTests.MultiTenancy/EfCoreTests.MultiTenancy.csproj b/src/Persistence/EfCoreTests.MultiTenancy/EfCoreTests.MultiTenancy.csproj index 4d35eb944..b09ce498b 100644 --- a/src/Persistence/EfCoreTests.MultiTenancy/EfCoreTests.MultiTenancy.csproj +++ b/src/Persistence/EfCoreTests.MultiTenancy/EfCoreTests.MultiTenancy.csproj @@ -1,7 +1,6 @@ - Exe false true diff --git a/src/Persistence/EfCoreTests/EfCoreTests.csproj b/src/Persistence/EfCoreTests/EfCoreTests.csproj index b6b9ce74d..8d4b5feb6 100644 --- a/src/Persistence/EfCoreTests/EfCoreTests.csproj +++ b/src/Persistence/EfCoreTests/EfCoreTests.csproj @@ -1,7 +1,6 @@ - Exe false true diff --git a/src/Persistence/LeaderElection/CosmosDbTests.LeaderElection/CosmosDbTests.LeaderElection.csproj b/src/Persistence/LeaderElection/CosmosDbTests.LeaderElection/CosmosDbTests.LeaderElection.csproj index 8dfc4bb2c..d0a1ee40f 100644 --- a/src/Persistence/LeaderElection/CosmosDbTests.LeaderElection/CosmosDbTests.LeaderElection.csproj +++ b/src/Persistence/LeaderElection/CosmosDbTests.LeaderElection/CosmosDbTests.LeaderElection.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Persistence/LeaderElection/MySqlTests.LeaderElection/MySqlTests.LeaderElection.csproj b/src/Persistence/LeaderElection/MySqlTests.LeaderElection/MySqlTests.LeaderElection.csproj index 6864e5706..e61185419 100644 --- a/src/Persistence/LeaderElection/MySqlTests.LeaderElection/MySqlTests.LeaderElection.csproj +++ b/src/Persistence/LeaderElection/MySqlTests.LeaderElection/MySqlTests.LeaderElection.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Persistence/LeaderElection/OracleTests.LeaderElection/OracleTests.LeaderElection.csproj b/src/Persistence/LeaderElection/OracleTests.LeaderElection/OracleTests.LeaderElection.csproj index c43010e37..f7ad2dbec 100644 --- a/src/Persistence/LeaderElection/OracleTests.LeaderElection/OracleTests.LeaderElection.csproj +++ b/src/Persistence/LeaderElection/OracleTests.LeaderElection/OracleTests.LeaderElection.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Persistence/LeaderElection/PostgresqlTests.LeaderElection/PostgresqlTests.LeaderElection.csproj b/src/Persistence/LeaderElection/PostgresqlTests.LeaderElection/PostgresqlTests.LeaderElection.csproj index dd8f6b194..87721a731 100644 --- a/src/Persistence/LeaderElection/PostgresqlTests.LeaderElection/PostgresqlTests.LeaderElection.csproj +++ b/src/Persistence/LeaderElection/PostgresqlTests.LeaderElection/PostgresqlTests.LeaderElection.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Persistence/LeaderElection/RavenDbTests.LeaderElection/RavenDbTests.LeaderElection.csproj b/src/Persistence/LeaderElection/RavenDbTests.LeaderElection/RavenDbTests.LeaderElection.csproj index 2b4657c5a..176b08420 100644 --- a/src/Persistence/LeaderElection/RavenDbTests.LeaderElection/RavenDbTests.LeaderElection.csproj +++ b/src/Persistence/LeaderElection/RavenDbTests.LeaderElection/RavenDbTests.LeaderElection.csproj @@ -1,7 +1,6 @@ - Exe net9.0 enable diff --git a/src/Persistence/LeaderElection/SqlServerTests.LeaderElection/SqlServerTests.LeaderElection.csproj b/src/Persistence/LeaderElection/SqlServerTests.LeaderElection/SqlServerTests.LeaderElection.csproj index 74c02afad..936676958 100644 --- a/src/Persistence/LeaderElection/SqlServerTests.LeaderElection/SqlServerTests.LeaderElection.csproj +++ b/src/Persistence/LeaderElection/SqlServerTests.LeaderElection/SqlServerTests.LeaderElection.csproj @@ -1,7 +1,6 @@ - Exe false true diff --git a/src/Persistence/MartenSubscriptionTests/MartenSubscriptionTests.csproj b/src/Persistence/MartenSubscriptionTests/MartenSubscriptionTests.csproj index 04c732191..113babc10 100644 --- a/src/Persistence/MartenSubscriptionTests/MartenSubscriptionTests.csproj +++ b/src/Persistence/MartenSubscriptionTests/MartenSubscriptionTests.csproj @@ -1,7 +1,6 @@ - Exe false true diff --git a/src/Persistence/MartenTests/MartenTests.csproj b/src/Persistence/MartenTests/MartenTests.csproj index d8aed9718..598f3a542 100644 --- a/src/Persistence/MartenTests/MartenTests.csproj +++ b/src/Persistence/MartenTests/MartenTests.csproj @@ -1,7 +1,6 @@ - Exe net9.0;net10.0 false diff --git a/src/Persistence/MySql/MySqlTests/MySqlTests.csproj b/src/Persistence/MySql/MySqlTests/MySqlTests.csproj index a5968d00e..ec8719060 100644 --- a/src/Persistence/MySql/MySqlTests/MySqlTests.csproj +++ b/src/Persistence/MySql/MySqlTests/MySqlTests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Persistence/Oracle/OracleTests/OracleTests.csproj b/src/Persistence/Oracle/OracleTests/OracleTests.csproj index f8631fe20..f051b62a1 100644 --- a/src/Persistence/Oracle/OracleTests/OracleTests.csproj +++ b/src/Persistence/Oracle/OracleTests/OracleTests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Persistence/PersistenceTests/PersistenceTests.csproj b/src/Persistence/PersistenceTests/PersistenceTests.csproj index b8d82e73b..4b43af1a0 100644 --- a/src/Persistence/PersistenceTests/PersistenceTests.csproj +++ b/src/Persistence/PersistenceTests/PersistenceTests.csproj @@ -1,7 +1,6 @@  - Exe false net9.0;net10.0 diff --git a/src/Persistence/Polecat/PolecatIncidentService.Tests/PolecatIncidentService.Tests.csproj b/src/Persistence/Polecat/PolecatIncidentService.Tests/PolecatIncidentService.Tests.csproj index 9cd507757..e619bbdf3 100644 --- a/src/Persistence/Polecat/PolecatIncidentService.Tests/PolecatIncidentService.Tests.csproj +++ b/src/Persistence/Polecat/PolecatIncidentService.Tests/PolecatIncidentService.Tests.csproj @@ -1,7 +1,6 @@ - Exe false net10.0 diff --git a/src/Persistence/PolecatTests/PolecatTests.csproj b/src/Persistence/PolecatTests/PolecatTests.csproj index 3e390d666..e2fde0b59 100644 --- a/src/Persistence/PolecatTests/PolecatTests.csproj +++ b/src/Persistence/PolecatTests/PolecatTests.csproj @@ -1,7 +1,6 @@ - Exe false true diff --git a/src/Persistence/PostgresqlTests/PostgresqlTests.csproj b/src/Persistence/PostgresqlTests/PostgresqlTests.csproj index c6c952dea..1cb70ad93 100644 --- a/src/Persistence/PostgresqlTests/PostgresqlTests.csproj +++ b/src/Persistence/PostgresqlTests/PostgresqlTests.csproj @@ -1,7 +1,6 @@ - Exe net9.0;net10.0 false diff --git a/src/Persistence/RavenDbTests/RavenDbTests.csproj b/src/Persistence/RavenDbTests/RavenDbTests.csproj index ec8e06f5b..f2875a469 100644 --- a/src/Persistence/RavenDbTests/RavenDbTests.csproj +++ b/src/Persistence/RavenDbTests/RavenDbTests.csproj @@ -1,7 +1,6 @@ - Exe net9.0 enable diff --git a/src/Persistence/SqlServerTests/SqlServerTests.csproj b/src/Persistence/SqlServerTests/SqlServerTests.csproj index 7db971991..ae874df2e 100644 --- a/src/Persistence/SqlServerTests/SqlServerTests.csproj +++ b/src/Persistence/SqlServerTests/SqlServerTests.csproj @@ -1,7 +1,6 @@ - Exe false true diff --git a/src/Persistence/SqliteTests/SqliteTests.csproj b/src/Persistence/SqliteTests/SqliteTests.csproj index f464d7f25..9cb9f26e9 100644 --- a/src/Persistence/SqliteTests/SqliteTests.csproj +++ b/src/Persistence/SqliteTests/SqliteTests.csproj @@ -1,7 +1,6 @@ - false Exe diff --git a/src/Persistence/Wolverine.ClaimCheck.AmazonS3.Tests/Wolverine.ClaimCheck.AmazonS3.Tests.csproj b/src/Persistence/Wolverine.ClaimCheck.AmazonS3.Tests/Wolverine.ClaimCheck.AmazonS3.Tests.csproj index 91e1c2824..47033869a 100644 --- a/src/Persistence/Wolverine.ClaimCheck.AmazonS3.Tests/Wolverine.ClaimCheck.AmazonS3.Tests.csproj +++ b/src/Persistence/Wolverine.ClaimCheck.AmazonS3.Tests/Wolverine.ClaimCheck.AmazonS3.Tests.csproj @@ -1,7 +1,6 @@ - Exe net9.0;net10.0 enable diff --git a/src/Persistence/Wolverine.ClaimCheck.AzureBlobStorage.Tests/Wolverine.ClaimCheck.AzureBlobStorage.Tests.csproj b/src/Persistence/Wolverine.ClaimCheck.AzureBlobStorage.Tests/Wolverine.ClaimCheck.AzureBlobStorage.Tests.csproj index 0a7de28be..1ceb75a33 100644 --- a/src/Persistence/Wolverine.ClaimCheck.AzureBlobStorage.Tests/Wolverine.ClaimCheck.AzureBlobStorage.Tests.csproj +++ b/src/Persistence/Wolverine.ClaimCheck.AzureBlobStorage.Tests/Wolverine.ClaimCheck.AzureBlobStorage.Tests.csproj @@ -1,7 +1,6 @@ - Exe net9.0;net10.0 enable diff --git a/src/Persistence/Wolverine.ClaimCheck.GoogleCloudStorage.Tests/Wolverine.ClaimCheck.GoogleCloudStorage.Tests.csproj b/src/Persistence/Wolverine.ClaimCheck.GoogleCloudStorage.Tests/Wolverine.ClaimCheck.GoogleCloudStorage.Tests.csproj index cac418695..1411525f4 100644 --- a/src/Persistence/Wolverine.ClaimCheck.GoogleCloudStorage.Tests/Wolverine.ClaimCheck.GoogleCloudStorage.Tests.csproj +++ b/src/Persistence/Wolverine.ClaimCheck.GoogleCloudStorage.Tests/Wolverine.ClaimCheck.GoogleCloudStorage.Tests.csproj @@ -1,7 +1,6 @@ - Exe net9.0;net10.0 enable diff --git a/src/Persistence/Wolverine.ClaimCheck.Nats.Tests/Wolverine.ClaimCheck.Nats.Tests.csproj b/src/Persistence/Wolverine.ClaimCheck.Nats.Tests/Wolverine.ClaimCheck.Nats.Tests.csproj index 1b3504546..fc887b45f 100644 --- a/src/Persistence/Wolverine.ClaimCheck.Nats.Tests/Wolverine.ClaimCheck.Nats.Tests.csproj +++ b/src/Persistence/Wolverine.ClaimCheck.Nats.Tests/Wolverine.ClaimCheck.Nats.Tests.csproj @@ -1,7 +1,6 @@ - Exe net9.0;net10.0 enable diff --git a/src/Persistence/Wolverine.ClaimCheck.Postgresql.Tests/Wolverine.ClaimCheck.Postgresql.Tests.csproj b/src/Persistence/Wolverine.ClaimCheck.Postgresql.Tests/Wolverine.ClaimCheck.Postgresql.Tests.csproj index 1f1bc5ca5..86e2d8abd 100644 --- a/src/Persistence/Wolverine.ClaimCheck.Postgresql.Tests/Wolverine.ClaimCheck.Postgresql.Tests.csproj +++ b/src/Persistence/Wolverine.ClaimCheck.Postgresql.Tests/Wolverine.ClaimCheck.Postgresql.Tests.csproj @@ -1,7 +1,6 @@ - Exe net9.0;net10.0 enable diff --git a/src/Samples/CQRSWithMarten/TeleHealth.Tests/TeleHealth.Tests.csproj b/src/Samples/CQRSWithMarten/TeleHealth.Tests/TeleHealth.Tests.csproj index 34936a790..9185b0002 100644 --- a/src/Samples/CQRSWithMarten/TeleHealth.Tests/TeleHealth.Tests.csproj +++ b/src/Samples/CQRSWithMarten/TeleHealth.Tests/TeleHealth.Tests.csproj @@ -1,7 +1,6 @@ - Exe net9.0;net10.0 false diff --git a/src/Samples/Diagnostics/DiagnosticsTests/DiagnosticsTests.csproj b/src/Samples/Diagnostics/DiagnosticsTests/DiagnosticsTests.csproj index bf657e63a..1e8f888f8 100644 --- a/src/Samples/Diagnostics/DiagnosticsTests/DiagnosticsTests.csproj +++ b/src/Samples/Diagnostics/DiagnosticsTests/DiagnosticsTests.csproj @@ -1,7 +1,6 @@ - Exe net9.0;net10.0 false diff --git a/src/Samples/EFCoreSample/ItemService.Tests/ItemService.Tests.csproj b/src/Samples/EFCoreSample/ItemService.Tests/ItemService.Tests.csproj index a7ae7ea32..08815483a 100644 --- a/src/Samples/EFCoreSample/ItemService.Tests/ItemService.Tests.csproj +++ b/src/Samples/EFCoreSample/ItemService.Tests/ItemService.Tests.csproj @@ -1,6 +1,5 @@ - Exe net9.0;net10.0 diff --git a/src/Samples/IncidentService/IncidentService.Tests/IncidentService.Tests.csproj b/src/Samples/IncidentService/IncidentService.Tests/IncidentService.Tests.csproj index 59abbcb1b..745b5db25 100644 --- a/src/Samples/IncidentService/IncidentService.Tests/IncidentService.Tests.csproj +++ b/src/Samples/IncidentService/IncidentService.Tests/IncidentService.Tests.csproj @@ -1,7 +1,6 @@ - Exe false net9.0 diff --git a/src/Samples/Middleware/AppWithMiddleware.Tests/AppWithMiddleware.Tests.csproj b/src/Samples/Middleware/AppWithMiddleware.Tests/AppWithMiddleware.Tests.csproj index b41376ff1..79a79a63e 100644 --- a/src/Samples/Middleware/AppWithMiddleware.Tests/AppWithMiddleware.Tests.csproj +++ b/src/Samples/Middleware/AppWithMiddleware.Tests/AppWithMiddleware.Tests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Samples/MultiTenantedTodoService/MultiTenantedTodoWebService.Tests/MultiTenantedTodoWebService.Tests.csproj b/src/Samples/MultiTenantedTodoService/MultiTenantedTodoWebService.Tests/MultiTenantedTodoWebService.Tests.csproj index 87e7e6c49..b3e3dfa22 100644 --- a/src/Samples/MultiTenantedTodoService/MultiTenantedTodoWebService.Tests/MultiTenantedTodoWebService.Tests.csproj +++ b/src/Samples/MultiTenantedTodoService/MultiTenantedTodoWebService.Tests/MultiTenantedTodoWebService.Tests.csproj @@ -1,7 +1,6 @@ - Exe false true diff --git a/src/Samples/ProcessManagerViaHandlers/ProcessManagerViaHandlers.Tests/ProcessManagerViaHandlers.Tests.csproj b/src/Samples/ProcessManagerViaHandlers/ProcessManagerViaHandlers.Tests/ProcessManagerViaHandlers.Tests.csproj index 2e49c09b3..206963e24 100644 --- a/src/Samples/ProcessManagerViaHandlers/ProcessManagerViaHandlers.Tests/ProcessManagerViaHandlers.Tests.csproj +++ b/src/Samples/ProcessManagerViaHandlers/ProcessManagerViaHandlers.Tests/ProcessManagerViaHandlers.Tests.csproj @@ -1,7 +1,6 @@ - Exe false net9.0 diff --git a/src/Samples/TestHarness/BankingService.Tests/BankingService.Tests.csproj b/src/Samples/TestHarness/BankingService.Tests/BankingService.Tests.csproj index 03e43e82e..f55740f95 100644 --- a/src/Samples/TestHarness/BankingService.Tests/BankingService.Tests.csproj +++ b/src/Samples/TestHarness/BankingService.Tests/BankingService.Tests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Samples/TodoWebService/TodoWebServiceTests/TodoWebServiceTests.csproj b/src/Samples/TodoWebService/TodoWebServiceTests/TodoWebServiceTests.csproj index 894555453..ae91c5001 100644 --- a/src/Samples/TodoWebService/TodoWebServiceTests/TodoWebServiceTests.csproj +++ b/src/Samples/TodoWebService/TodoWebServiceTests/TodoWebServiceTests.csproj @@ -1,7 +1,6 @@ - Exe net9.0;net10.0 false diff --git a/src/Testing/BackPressureTests/BackPressureTests.csproj b/src/Testing/BackPressureTests/BackPressureTests.csproj index 596551e2c..f7f5038f4 100644 --- a/src/Testing/BackPressureTests/BackPressureTests.csproj +++ b/src/Testing/BackPressureTests/BackPressureTests.csproj @@ -1,7 +1,6 @@ - Exe enable false diff --git a/src/Testing/CoreTests/CoreTests.csproj b/src/Testing/CoreTests/CoreTests.csproj index f36339838..d8e4dfb1f 100644 --- a/src/Testing/CoreTests/CoreTests.csproj +++ b/src/Testing/CoreTests/CoreTests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Testing/CoreTests/Runtime/Agents/slow_agent_start_convergence.cs b/src/Testing/CoreTests/Runtime/Agents/slow_agent_start_convergence.cs new file mode 100644 index 000000000..a423fab38 --- /dev/null +++ b/src/Testing/CoreTests/Runtime/Agents/slow_agent_start_convergence.cs @@ -0,0 +1,440 @@ +using CoreTests.Transports; +using JasperFx.Core; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using Shouldly; +using Wolverine.ComplianceTests; +using Wolverine.Configuration; +using Wolverine.Runtime; +using Wolverine.Runtime.Agents; +using Xunit; + +namespace CoreTests.Runtime.Agents; + +/// +/// GH-3779. Every defect in the GH-3753 chain — GH-3748, GH-3749, GH-3750, jasperfx#594, jasperfx#598 — +/// takes a slow agent start as its precondition, and none of them had a dev-scale reproduction: +/// FakeAgent.StartAsync returned Task.CompletedTask, which is precisely the assumption the +/// whole chain violates. The only verification site was a customer's restored-production canary (512 tenant +/// databases, ~6,500 agents, 5 nodes), so every candidate fix shipped as a pinned prerelease and was measured +/// by someone else, and three closed defects carried no regression coverage at all. +/// +/// This drives the leader's real round after +/// round against a simulated cluster whose agent starts cost time, and asserts the emergent properties +/// the field reported losing. +/// +/// Time is measured in evaluation rounds, not milliseconds. The field distribution is p50 27s / +/// p95 82s / a 215s tail against a ~5s health-check cadence, so a wall-clock simulation would take hours and +/// would trade a deterministic assertion for a timing race. A round is one health-check tick; the costs below +/// are that distribution divided by it. Everything here is seeded and deterministic — a failure reproduces. +/// +/// What this is not. A simulation is not the canary. The pathologies in GH-3753 are emergent from +/// real database contention, real node-table timing and real chunk sizes, so this reproduces the +/// shape of the problem and not its magnitude. Its value is the other direction: a bad fix can be +/// rejected in milliseconds instead of round-tripping through someone else's production restore. +/// +public class slow_agent_start_convergence +{ + /// + /// Start cost in rounds, calibrated against the field distribution behind GH-3753 at a ~5s health-check + /// cadence: p50 27s (~5 rounds), p95 82s (~16 rounds), tail 215s (~43 rounds). Deliberately a long tail + /// rather than a uniform cost — a uniformly slow family converges late but converges, while it is the + /// handful of very slow starts that outlive a reply window and get re-decided. + /// + private static Func fieldStartCost(int seed, IReadOnlyList agents) + { + var random = new Random(seed); + var costs = agents.ToDictionary(x => x, _ => + { + var roll = random.NextDouble(); + if (roll < 0.50) return random.Next(1, 6); + if (roll < 0.95) return random.Next(6, 17); + return random.Next(17, 44); + }); + + return uri => costs.TryGetValue(uri, out var cost) ? cost : 1; + } + + [Fact] + public async Task every_agent_converges_despite_a_long_tailed_start_distribution() + { + var cluster = new SlowStartCluster(nodeCount: 5, agentCount: 603, seed: 3753); + cluster.StartCost = fieldStartCost(3753, cluster.AllAgents); + + // The slowest single start is 43 rounds; anything much past that is the leader failing to converge + // rather than the cluster merely being slow. + var rounds = await cluster.RunUntilConvergedAsync(maxRounds: 60); + + cluster.RunningAgents.Count.ShouldBe(cluster.AllAgents.Length); + rounds.ShouldBeLessThan(60); + } + + /// + /// GH-3698's failure mode. A dispatched-but-unstarted agent has no persisted assignment row, so it looks + /// completely unplaced to the next evaluation; without the pending-assignment ledger the leader re-decides + /// its placement from scratch and sends the same agent to a SECOND node with no stop for the copy already + /// coming up on the first. Two live copies of a projection agent is the worst outcome in the whole chain. + /// + [Fact] + public async Task an_agent_is_never_started_on_two_nodes_at_once() + { + var cluster = new SlowStartCluster(nodeCount: 5, agentCount: 603, seed: 3753); + cluster.StartCost = fieldStartCost(3753, cluster.AllAgents); + + await cluster.RunUntilConvergedAsync(maxRounds: 60); + + // The cluster asserts the single-copy invariant on every mutation as it runs; this is the end-state + // restatement of it. + cluster.DoubleStartReports.ShouldBeEmpty(); + } + + /// + /// The other field shape: five nodes holding 799 / 0 / 0 / 0 / 1. A slow wave must not leave the + /// distribution lopsided once it has converged. + /// + [Fact] + public async Task no_node_is_starved_while_another_holds_everything() + { + var cluster = new SlowStartCluster(nodeCount: 5, agentCount: 603, seed: 3753); + cluster.StartCost = fieldStartCost(3753, cluster.AllAgents); + + await cluster.RunUntilConvergedAsync(maxRounds: 60); + + var counts = cluster.RunningCountsByNode; + + // 603 across 5 nodes is 120 with a remainder of 3, so an even spread is 120 or 121 everywhere. + counts.Max().ShouldBeLessThanOrEqualTo(counts.Min() + 1); + } + + /// + /// A wave of slow starts must not itself generate churn. Once the leader has placed an agent it should + /// stay placed: every stop or reassignment emitted during a first-time placement wave against a stable + /// cluster is work the leader is undoing, and it is what turned the field's convergence into a livelock. + /// + [Fact] + public async Task a_first_placement_wave_emits_no_stops_and_no_reassignments() + { + var cluster = new SlowStartCluster(nodeCount: 5, agentCount: 603, seed: 3753); + cluster.StartCost = fieldStartCost(3753, cluster.AllAgents); + + await cluster.RunUntilConvergedAsync(maxRounds: 60); + + cluster.StopsEmitted.ShouldBe(0); + cluster.ReassignmentsEmitted.ShouldBe(0); + + // And each agent was asked to start exactly once — no duplicate dispatch of work already in flight. + cluster.DispatchCounts.Values.ShouldAllBe(x => x == 1); + + // The clearest signal from the field that something was wrong: over one four-minute window the + // cluster's total running-agent count went 2,401 -> 1,885. Against a stable node set that number can + // only fall if agents that had come up were stopped, so this is a restatement of the two assertions + // above rather than an independent one — it is here because it is the shape an operator actually + // sees, and because it is the assertion that still holds if a node-churn scenario is added later. + cluster.RunningCountByRound.ShouldBe(cluster.RunningCountByRound.OrderBy(x => x).ToArray()); + } + + /// + /// GH-3698, sharpened. The pending-assignment ledger has two ways to hold an agent: a TTL backstop of + /// 2 x CheckAssignmentPeriod, and the dispatcher's own answer to "is this start still outstanding?". + /// A start slower than the TTL is the normal field case (27s p50 against a 60s TTL is close; the 215s tail + /// is not close at all), so the TTL is squeezed to nothing here and the probe is left as the only thing + /// holding the agent. It must be enough on its own. + /// + [Fact] + public async Task a_start_slower_than_the_ledger_ttl_is_still_held_by_the_outstanding_dispatch() + { + var cluster = new SlowStartCluster(nodeCount: 3, agentCount: 12, seed: 3698); + + // 1ms period => a 2ms TTL. Every round below sleeps far past it, so nothing is held by the clock. + cluster.Options.Durability.CheckAssignmentPeriod = 1.Milliseconds(); + + // No start ever lands during this test. + cluster.StartCost = _ => 1000; + + var first = await cluster.RunRoundAsync(); + cluster.AssignedIn(first).Count().ShouldBe(12); + + for (var i = 0; i < 4; i++) + { + // Comfortably past the 2ms TTL, so a re-emission here can only come from the ledger having + // released an agent whose start is still outstanding. + await Task.Delay(25.Milliseconds(), TestContext.Current.CancellationToken); + + var round = await cluster.RunRoundAsync(); + round.ShouldBeEmpty(); + } + + cluster.DoubleStartReports.ShouldBeEmpty(); + cluster.DispatchCounts.Values.ShouldAllBe(x => x == 1); + } + + /// + /// GH-3750: a PARTIAL confirmation is the normal case whenever starts are slow — StartAgents only + /// bags an agent once its start returns, so a chunk of 50 agents whose costs straddle the reply window + /// answers with a subset. The remainder must still be driven to completion rather than being counted as + /// started and forgotten. + /// + [Fact] + public async Task the_unconfirmed_remainder_of_a_partially_started_chunk_still_converges() + { + var cluster = new SlowStartCluster(nodeCount: 2, agentCount: 100, seed: 3750); + + // Half the chunk lands almost immediately, half of it long after any reply window — the straddle + // that produces a partial AgentsStarted reply. + var slow = cluster.AllAgents.Where((_, i) => i % 2 == 1).ToHashSet(); + cluster.StartCost = uri => slow.Contains(uri) ? 30 : 1; + + var rounds = await cluster.RunUntilConvergedAsync(maxRounds: 50); + + cluster.RunningAgents.Count.ShouldBe(100); + rounds.ShouldBeLessThan(50); + + // The fast half must not have dragged the slow half into being re-placed somewhere else. + cluster.DoubleStartReports.ShouldBeEmpty(); + cluster.StopsEmitted.ShouldBe(0); + } + + /// + /// A simulated multi-node cluster driving the leader's real assignment evaluation. One + /// is one health-check tick: in-flight starts advance, the leader evaluates, and the commands it emits are + /// applied to the cluster's state the way the corresponding agent commands would apply them for real. + /// + private sealed class SlowStartCluster + { + private readonly IWolverineRuntime _runtime; + private readonly FakeAgentFamily _family; + private readonly NodeAgentController _controller; + private readonly List _nodes = []; + private readonly Dictionary _inFlight = new(); + + // Ground truth: where each agent is actually running. Deliberately NOT the same thing as the leader's + // view of it — a node persists its assignment row only after the agent is up, and the leader reads + // that row on a later snapshot, so there is a window in which an agent is genuinely running and looks + // completely unplaced. That window is what GH-3750 is about, and a simulation that closes it + // instantly cannot reproduce the field's falling assigned-agent count. + private readonly Dictionary _running = new(); + private readonly Dictionary _awaitingVisibility = new(); + + private readonly List _runningCountByRound = []; + private readonly List _doubleStartReports = []; + private readonly Dictionary _dispatchCounts = new(); + + private record struct InFlightStart(Guid NodeId, int RoundsRemaining); + + public SlowStartCluster(int nodeCount, int agentCount, int seed) + { + Options = new WolverineOptions { ApplicationAssembly = GetType().Assembly }; + Options.Transports.NodeControlEndpoint = new FakeEndpoint("fake://self".ToUri(), EndpointRole.System); + Options.Durability.DurabilityAgentEnabled = false; + + _runtime = Substitute.For(); + _runtime.Options.Returns(Options); + _runtime.DurabilitySettings.Returns(Options.Durability); + _runtime.Observer.Returns(Substitute.For()); + + _family = new FakeAgentFamily("fake", agentCount); + AllAgents = _family.AllAgentUris(); + + _controller = new NodeAgentController(_runtime, Substitute.For(), [_family], + NullLogger.Instance, CancellationToken.None); + + // The dispatcher holds a command from the moment it is queued until its lane is done with it, + // whatever the outcome. An in-flight start here is exactly that hold. + _controller.PendingDispatches = (Uri agentUri, out Guid nodeId) => + { + if (_inFlight.TryGetValue(agentUri, out var pending)) + { + nodeId = pending.NodeId; + return true; + } + + nodeId = Guid.Empty; + return false; + }; + + // Node 0 is this process — the controller injects self into any node list that omits it, so the + // leader has to BE one of the simulated nodes rather than a sixth observer. + for (var i = 0; i < nodeCount; i++) + { + var node = new WolverineNode + { + NodeId = i == 0 ? Options.UniqueNodeId : Guid.NewGuid(), + AssignedNodeNumber = i + 1, + ControlUri = new Uri($"fake://node{i}") + }; + + node.Capabilities.AddRange(AllAgents); + _nodes.Add(node); + } + + StartCost = _ => 1; + Seed = seed; + } + + public WolverineOptions Options { get; } + public int Seed { get; } + public Uri[] AllAgents { get; } + + /// How many rounds each agent's start takes to complete on its destination node. + public Func StartCost { get; set; } + + public IReadOnlyList RunningCountByRound => _runningCountByRound; + public IReadOnlyList DoubleStartReports => _doubleStartReports; + public IReadOnlyDictionary DispatchCounts => _dispatchCounts; + public int StopsEmitted { get; private set; } + public int ReassignmentsEmitted { get; private set; } + + public IReadOnlyList RunningAgents => _running.Keys.ToList(); + + public int[] RunningCountsByNode + => _nodes.Select(node => _running.Count(x => x.Value == node.NodeId)).ToArray(); + + public IEnumerable AssignedIn(AgentCommands commands) + => commands.OfType().SelectMany(x => x.AgentIds) + .Concat(commands.OfType().Select(x => x.AgentUri)); + + /// + /// One health-check tick: land any starts whose cost has run out, evaluate, and apply the result. + /// + public async Task RunRoundAsync() + { + landCompletedStarts(); + + var commands = await _controller.EvaluateAssignmentsAsync(_nodes, new AgentRestrictions()); + + foreach (var command in commands) + { + switch (command) + { + case AssignAgent assign: + dispatchStart(assign.AgentUri, assign.Destination.NodeId); + break; + + case AssignAgents assigns: + foreach (var uri in assigns.AgentIds) dispatchStart(uri, assigns.Destination.NodeId); + break; + + case ReassignAgent reassign: + ReassignmentsEmitted++; + stop(reassign.AgentUri, reassign.OriginalNode.NodeId); + dispatchStart(reassign.AgentUri, reassign.ActiveNode.NodeId); + break; + + case ReassignAgents reassigns: + ReassignmentsEmitted += reassigns.AgentUris.Length; + foreach (var uri in reassigns.AgentUris) + { + stop(uri, reassigns.OriginalNode.NodeId); + dispatchStart(uri, reassigns.ActiveNode.NodeId); + } + + break; + + case StopRemoteAgent stopOne: + StopsEmitted++; + stop(stopOne.AgentUri, stopOne.Destination.NodeId); + break; + + case StopRemoteAgents stopMany: + StopsEmitted += stopMany.AgentIds.Length; + foreach (var uri in stopMany.AgentIds) stop(uri, stopMany.Destination.NodeId); + break; + } + } + + _runningCountByRound.Add(_running.Count); + + return commands; + } + + /// Runs rounds until every agent is running and the leader has nothing left to say. + public async Task RunUntilConvergedAsync(int maxRounds) + { + for (var round = 1; round <= maxRounds; round++) + { + var commands = await RunRoundAsync(); + + if (commands.Count == 0 && _inFlight.Count == 0 && _awaitingVisibility.Count == 0 + && _running.Count == AllAgents.Length) + { + return round; + } + } + + return maxRounds; + } + + private void landCompletedStarts() + { + // Agents that came up last round: their assignment row is now visible to the leader's snapshot. + foreach (var (uri, nodeId) in _awaitingVisibility.ToArray()) + { + _nodes.Single(x => x.NodeId == nodeId).ActiveAgents.Fill(uri); + } + + _awaitingVisibility.Clear(); + + foreach (var uri in _inFlight.Keys.ToArray()) + { + var pending = _inFlight[uri]; + if (pending.RoundsRemaining > 1) + { + _inFlight[uri] = pending with { RoundsRemaining = pending.RoundsRemaining - 1 }; + continue; + } + + _inFlight.Remove(uri); + + // Running now — but invisible to the leader until the promotion above runs next round. + _running[uri] = pending.NodeId; + _awaitingVisibility[uri] = pending.NodeId; + + // What AssignAgent/AssignAgents do the moment a start confirms, and the reason the agent stays + // held for the one snapshot cycle it takes the persisted assignment row to become visible. + _controller.ConfirmDispatched([uri], pending.NodeId); + } + } + + private void dispatchStart(Uri agentUri, Guid nodeId) + { + _dispatchCounts.TryGetValue(agentUri, out var count); + _dispatchCounts[agentUri] = count + 1; + + // The single-copy invariant, asserted at the moment it would be violated rather than inferred from + // the end state — a second copy that is later stopped still ran twice. + if (_inFlight.TryGetValue(agentUri, out var already) && already.NodeId != nodeId) + { + _doubleStartReports.Add( + $"{agentUri} dispatched to node {nodeId} while a start was still in flight to node {already.NodeId}"); + } + + if (_running.TryGetValue(agentUri, out var runningOn) && runningOn != nodeId) + { + _doubleStartReports.Add( + $"{agentUri} dispatched to node {nodeId} while already running on node {runningOn}"); + } + + _inFlight[agentUri] = new InFlightStart(nodeId, Math.Max(1, StartCost(agentUri))); + } + + private void stop(Uri agentUri, Guid nodeId) + { + _nodes.Single(x => x.NodeId == nodeId).ActiveAgents.Remove(agentUri); + + if (_running.TryGetValue(agentUri, out var runningOn) && runningOn == nodeId) + { + _running.Remove(agentUri); + } + + if (_awaitingVisibility.TryGetValue(agentUri, out var pendingVisible) && pendingVisible == nodeId) + { + _awaitingVisibility.Remove(agentUri); + } + + if (_inFlight.TryGetValue(agentUri, out var pending) && pending.NodeId == nodeId) + { + _inFlight.Remove(agentUri); + } + } + } +} diff --git a/src/Testing/MessageRoutingTests/MessageRoutingTests.csproj b/src/Testing/MessageRoutingTests/MessageRoutingTests.csproj index 2600da021..98194edb2 100644 --- a/src/Testing/MessageRoutingTests/MessageRoutingTests.csproj +++ b/src/Testing/MessageRoutingTests/MessageRoutingTests.csproj @@ -1,7 +1,6 @@ - Exe enable false diff --git a/src/Testing/MetricsTests/MetricsTests.csproj b/src/Testing/MetricsTests/MetricsTests.csproj index 34dc36003..63e948479 100644 --- a/src/Testing/MetricsTests/MetricsTests.csproj +++ b/src/Testing/MetricsTests/MetricsTests.csproj @@ -1,7 +1,6 @@ - Exe net9.0 enable diff --git a/src/Testing/PolicyTests/PolicyTests.csproj b/src/Testing/PolicyTests/PolicyTests.csproj index 0bfbb1305..46bfd72f3 100644 --- a/src/Testing/PolicyTests/PolicyTests.csproj +++ b/src/Testing/PolicyTests/PolicyTests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Testing/SlowTests/SlowTests.csproj b/src/Testing/SlowTests/SlowTests.csproj index 2c23bee48..30160e90f 100644 --- a/src/Testing/SlowTests/SlowTests.csproj +++ b/src/Testing/SlowTests/SlowTests.csproj @@ -1,9 +1,5 @@ - - - - diff --git a/src/Testing/Wolverine.Behavioural.FSharpTests/Wolverine.Behavioural.FSharpTests.csproj b/src/Testing/Wolverine.Behavioural.FSharpTests/Wolverine.Behavioural.FSharpTests.csproj index 38a283fa8..5370ca8a2 100644 --- a/src/Testing/Wolverine.Behavioural.FSharpTests/Wolverine.Behavioural.FSharpTests.csproj +++ b/src/Testing/Wolverine.Behavioural.FSharpTests/Wolverine.Behavioural.FSharpTests.csproj @@ -9,7 +9,6 @@ --> - Exe net9.0 false diff --git a/src/Testing/Wolverine.ComplianceTests/FakeAgent.cs b/src/Testing/Wolverine.ComplianceTests/FakeAgent.cs index 261f40a3a..344d9b3cd 100644 --- a/src/Testing/Wolverine.ComplianceTests/FakeAgent.cs +++ b/src/Testing/Wolverine.ComplianceTests/FakeAgent.cs @@ -12,21 +12,55 @@ public FakeAgent(Uri uri) public bool IsRunning { get; private set; } - public Task StartAsync(CancellationToken cancellationToken) + /// + /// GH-3779: how long this agent takes to start. Defaults to zero, which is what every test written + /// before the slow-start work assumed. A real daemon agent is nothing like free — a Marten shard + /// replaying behind a projection version bump was measured in the field at p50 27s / p95 82s with a + /// 215s tail — and every defect in the GH-3753 chain takes a slow start as its precondition, so a + /// harness that cannot express one cannot reproduce any of them. + /// + public TimeSpan StartDelay { get; set; } = TimeSpan.Zero; + + /// + /// GH-3779: how long this agent takes to stop. A stop can be as slow as a start — a shard mid-replay + /// has to let go of its work — which is why StopRemoteAgents got the same scaled reply window as + /// AssignAgents in GH-3748. + /// + public TimeSpan StopDelay { get; set; } = TimeSpan.Zero; + + /// + /// Number of times StartAsync has been entered, whether or not it completed. Distinguishes a start + /// that is merely slow from one the leader has re-driven. + /// + public int StartCount => _startCount; + + private int _startCount; + + public async Task StartAsync(CancellationToken cancellationToken) { + Interlocked.Increment(ref _startCount); + + if (StartDelay > TimeSpan.Zero) + { + await Task.Delay(StartDelay, cancellationToken); + } + IsRunning = true; Status = AgentStatus.Running; - return Task.CompletedTask; } - public Task StopAsync(CancellationToken cancellationToken) + public async Task StopAsync(CancellationToken cancellationToken) { + if (StopDelay > TimeSpan.Zero) + { + await Task.Delay(StopDelay, cancellationToken); + } + IsRunning = false; Status = AgentStatus.Stopped; - return Task.CompletedTask; } - + public AgentStatus Status { get; private set; } = AgentStatus.Running; public Uri Uri { get; } -} \ No newline at end of file +} diff --git a/src/Testing/Wolverine.ComplianceTests/FakeAgentFamily.cs b/src/Testing/Wolverine.ComplianceTests/FakeAgentFamily.cs index 7d289a4df..44a8a6105 100644 --- a/src/Testing/Wolverine.ComplianceTests/FakeAgentFamily.cs +++ b/src/Testing/Wolverine.ComplianceTests/FakeAgentFamily.cs @@ -15,6 +15,28 @@ public FakeAgentFamily(string scheme) Scheme = scheme; } + /// + /// GH-3779: a family of an arbitrary size. The twelve hard-coded are fine for + /// asserting the shape of a distribution, but GH-3753 is about ~6,500 agents across five nodes, and + /// the pathologies there — a node left holding everything while its peers sit at zero, an assigned + /// count that goes backwards — only emerge at a scale where one assignment wave cannot complete + /// inside one evaluation cycle. + /// + public FakeAgentFamily(string scheme, int agentCount) + { + if (agentCount < 1) + { + throw new ArgumentOutOfRangeException(nameof(agentCount), agentCount, + "An agent family needs at least one agent"); + } + + Scheme = scheme; + + // Zero-padded so the lexical order matches the numeric one; several assertions read a lot better + // against a sorted agent list. + AgentNames = Enumerable.Range(0, agentCount).Select(i => $"agent-{i:D5}").ToArray(); + } + public string Scheme { get; } = "fake"; public static string[] Names = @@ -33,6 +55,20 @@ public FakeAgentFamily(string scheme) "twelve" ]; + /// + /// The agent names this particular family exposes. Defaults to the shared so that + /// every test predating the count-taking constructor is unaffected. + /// + public IReadOnlyList AgentNames { get; } = Names; + + /// + /// GH-3779: per-agent start cost, so a long TAIL can be simulated rather than a uniform delay. The + /// field distribution that broke GH-3753 was p50 27s / p95 82s / tail 215s, and it is the tail that + /// does the damage — a uniformly slow family converges late but converges, while a handful of very + /// slow starts is what outlives a reply window and gets re-decided. + /// + public Func? StartDelayPolicy { get; set; } + public LightweightCache Agents { get; } = new(x => new FakeAgent(x)); public ValueTask EvaluateAssignmentsAsync(AssignmentGrid assignments) @@ -43,13 +79,20 @@ public ValueTask EvaluateAssignmentsAsync(AssignmentGrid assignments) public ValueTask> AllKnownAgentsAsync() { - var agents = Names.Select(x => new Uri($"{Scheme}://{x}")).ToArray(); + var agents = AllAgentUris(); return ValueTask.FromResult((IReadOnlyList)agents); } public ValueTask BuildAgentAsync(Uri uri, IWolverineRuntime runtime) { - return new ValueTask(Agents[uri]); + var agent = Agents[uri]; + + if (StartDelayPolicy != null) + { + agent.StartDelay = StartDelayPolicy(uri); + } + + return new ValueTask(agent); } public ValueTask> SupportedAgentsAsync() @@ -60,6 +103,6 @@ public ValueTask> SupportedAgentsAsync() public Uri[] AllAgentUris() { - return Names.Select(x => new Uri($"{Scheme}://{x}")).ToArray(); + return AgentNames.Select(x => new Uri($"{Scheme}://{x}")).ToArray(); } -} \ No newline at end of file +} diff --git a/src/Testing/Wolverine.ComplianceTests/Wolverine.ComplianceTests.csproj b/src/Testing/Wolverine.ComplianceTests/Wolverine.ComplianceTests.csproj index 5267e6933..6382e9d4a 100644 --- a/src/Testing/Wolverine.ComplianceTests/Wolverine.ComplianceTests.csproj +++ b/src/Testing/Wolverine.ComplianceTests/Wolverine.ComplianceTests.csproj @@ -1,7 +1,6 @@ - Compliance test harnesses for adding persistence and transport options to Wolverine WolverineFx.ComplianceTests - Exe net9.0 false diff --git a/src/Testing/Wolverine.Cosmos.FSharpTests/Wolverine.Cosmos.FSharpTests.csproj b/src/Testing/Wolverine.Cosmos.FSharpTests/Wolverine.Cosmos.FSharpTests.csproj index 3dd3a5e4b..508364634 100644 --- a/src/Testing/Wolverine.Cosmos.FSharpTests/Wolverine.Cosmos.FSharpTests.csproj +++ b/src/Testing/Wolverine.Cosmos.FSharpTests/Wolverine.Cosmos.FSharpTests.csproj @@ -9,7 +9,6 @@ --> - Exe net9.0 false diff --git a/src/Testing/Wolverine.EfCore.FSharpTests/Wolverine.EfCore.FSharpTests.csproj b/src/Testing/Wolverine.EfCore.FSharpTests/Wolverine.EfCore.FSharpTests.csproj index 2072d56c3..608cf43db 100644 --- a/src/Testing/Wolverine.EfCore.FSharpTests/Wolverine.EfCore.FSharpTests.csproj +++ b/src/Testing/Wolverine.EfCore.FSharpTests/Wolverine.EfCore.FSharpTests.csproj @@ -8,7 +8,6 @@ --> - Exe net9.0 false diff --git a/src/Testing/Wolverine.Http.FSharpTests/Wolverine.Http.FSharpTests.csproj b/src/Testing/Wolverine.Http.FSharpTests/Wolverine.Http.FSharpTests.csproj index 25e5cd7a2..e45ce98f1 100644 --- a/src/Testing/Wolverine.Http.FSharpTests/Wolverine.Http.FSharpTests.csproj +++ b/src/Testing/Wolverine.Http.FSharpTests/Wolverine.Http.FSharpTests.csproj @@ -8,7 +8,6 @@ --> - Exe net9.0 false diff --git a/src/Testing/Wolverine.Marten.FSharpTests/Wolverine.Marten.FSharpTests.csproj b/src/Testing/Wolverine.Marten.FSharpTests/Wolverine.Marten.FSharpTests.csproj index e0c029de3..1e0e6e109 100644 --- a/src/Testing/Wolverine.Marten.FSharpTests/Wolverine.Marten.FSharpTests.csproj +++ b/src/Testing/Wolverine.Marten.FSharpTests/Wolverine.Marten.FSharpTests.csproj @@ -8,7 +8,6 @@ --> - Exe net9.0 false diff --git a/src/Testing/Wolverine.MartenAggregate.FSharpTests/Wolverine.MartenAggregate.FSharpTests.csproj b/src/Testing/Wolverine.MartenAggregate.FSharpTests/Wolverine.MartenAggregate.FSharpTests.csproj index 6aba54097..722ac0287 100644 --- a/src/Testing/Wolverine.MartenAggregate.FSharpTests/Wolverine.MartenAggregate.FSharpTests.csproj +++ b/src/Testing/Wolverine.MartenAggregate.FSharpTests/Wolverine.MartenAggregate.FSharpTests.csproj @@ -8,7 +8,6 @@ --> - Exe net9.0 false diff --git a/src/Transports/AWS/Wolverine.AmazonSns.Tests/Wolverine.AmazonSns.Tests.csproj b/src/Transports/AWS/Wolverine.AmazonSns.Tests/Wolverine.AmazonSns.Tests.csproj index c8875227c..62bb09b57 100644 --- a/src/Transports/AWS/Wolverine.AmazonSns.Tests/Wolverine.AmazonSns.Tests.csproj +++ b/src/Transports/AWS/Wolverine.AmazonSns.Tests/Wolverine.AmazonSns.Tests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Transports/AWS/Wolverine.AmazonSqs.Tests/Wolverine.AmazonSqs.Tests.csproj b/src/Transports/AWS/Wolverine.AmazonSqs.Tests/Wolverine.AmazonSqs.Tests.csproj index d3a8b0d4a..ee63c4b09 100644 --- a/src/Transports/AWS/Wolverine.AmazonSqs.Tests/Wolverine.AmazonSqs.Tests.csproj +++ b/src/Transports/AWS/Wolverine.AmazonSqs.Tests/Wolverine.AmazonSqs.Tests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Transports/Azure/Wolverine.AzureServiceBus.Tests/Wolverine.AzureServiceBus.Tests.csproj b/src/Transports/Azure/Wolverine.AzureServiceBus.Tests/Wolverine.AzureServiceBus.Tests.csproj index 713d29e6c..0bbac1068 100644 --- a/src/Transports/Azure/Wolverine.AzureServiceBus.Tests/Wolverine.AzureServiceBus.Tests.csproj +++ b/src/Transports/Azure/Wolverine.AzureServiceBus.Tests/Wolverine.AzureServiceBus.Tests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Transports/GCP/Wolverine.Pubsub.Tests/Wolverine.Pubsub.Tests.csproj b/src/Transports/GCP/Wolverine.Pubsub.Tests/Wolverine.Pubsub.Tests.csproj index 7535ace0f..913b17345 100644 --- a/src/Transports/GCP/Wolverine.Pubsub.Tests/Wolverine.Pubsub.Tests.csproj +++ b/src/Transports/GCP/Wolverine.Pubsub.Tests/Wolverine.Pubsub.Tests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Transports/Kafka/BatchMessaging/Program.cs b/src/Transports/Kafka/BatchMessaging/Program.cs index 1cb3ba0c8..bbbf725fa 100644 --- a/src/Transports/Kafka/BatchMessaging/Program.cs +++ b/src/Transports/Kafka/BatchMessaging/Program.cs @@ -9,6 +9,13 @@ builder.Host.UseWolverine(opts => { + // GH-3763: this app is booted by Wolverine.Kafka.Tests through AlbaHost.For, and without this + // pin Wolverine resolves the application assembly to the *test* assembly instead of this one. Handler + // discovery then never sees TestMessagesHandler, so BatchMessagesOf() below has no + // TestMessage[] handler to bind to and every batch fails at runtime with "there is no known handler for + // TestMessage[]". Same pin, for the same reason, as WolverineWebApi's Program. See GH-3521. + opts.ApplicationAssembly = typeof(Program).Assembly; + opts.UseKafka("localhost:9092") .AutoProvision() .AutoPurgeOnStartup() @@ -35,8 +42,6 @@ var message = new TestMessage(); await bus.PublishAsync(message); await bus.PublishAsync(message); - // results in: - // No known handler for TestMessage#08dced0c-3834-b4c6-54d7-e075bf020000 from kafka://topic/topic_0 }); return await app.RunJasperFxCommands(args); diff --git a/src/Transports/Kafka/Wolverine.Kafka.Tests/Wolverine.Kafka.Tests.csproj b/src/Transports/Kafka/Wolverine.Kafka.Tests/Wolverine.Kafka.Tests.csproj index 8b5a273de..28e6a45a4 100644 --- a/src/Transports/Kafka/Wolverine.Kafka.Tests/Wolverine.Kafka.Tests.csproj +++ b/src/Transports/Kafka/Wolverine.Kafka.Tests/Wolverine.Kafka.Tests.csproj @@ -1,7 +1,6 @@ - Exe net9.0;net10.0 false diff --git a/src/Transports/Kafka/Wolverine.Kafka.Tests/batch_processing_with_kafka.cs b/src/Transports/Kafka/Wolverine.Kafka.Tests/batch_processing_with_kafka.cs index 8cb027e21..dad792ea4 100644 --- a/src/Transports/Kafka/Wolverine.Kafka.Tests/batch_processing_with_kafka.cs +++ b/src/Transports/Kafka/Wolverine.Kafka.Tests/batch_processing_with_kafka.cs @@ -5,7 +5,6 @@ namespace Wolverine.Kafka.Tests; -[Trait("Category", "Flaky")] public class batch_processing_with_kafka { [Fact] @@ -26,7 +25,14 @@ public async Task end_to_end() .Timeout(60.Seconds()) .ExecuteAndWaitAsync(execute); - tracked.FindSingleTrackedMessageOfType() - .Length.ShouldBe(2); + // Both published messages must arrive at the handler in batch (TestMessage[]) form. Deliberately + // NOT asserting a single batch of two: BatchingOptions triggers on a full batch OR on TriggerTime + // (250ms by default), so whether two messages published back-to-back land in one batch or in two + // depends on how the Kafka consumer happens to slice its polls. Asserting the grouping asserts a + // race — the contract that actually matters is that every message was delivered, batched. + var batched = tracked.Received.MessagesOf().ToArray(); + + batched.ShouldNotBeEmpty(); + batched.Sum(x => x.Length).ShouldBe(2); } } \ No newline at end of file diff --git a/src/Transports/MQTT/Wolverine.MQTT.Tests/Wolverine.MQTT.Tests.csproj b/src/Transports/MQTT/Wolverine.MQTT.Tests/Wolverine.MQTT.Tests.csproj index d69173647..2fe716e5a 100644 --- a/src/Transports/MQTT/Wolverine.MQTT.Tests/Wolverine.MQTT.Tests.csproj +++ b/src/Transports/MQTT/Wolverine.MQTT.Tests/Wolverine.MQTT.Tests.csproj @@ -1,7 +1,6 @@ - Exe false true diff --git a/src/Transports/MQTT/Wolverine.Mqtt5.Tests/Wolverine.Mqtt5.Tests.csproj b/src/Transports/MQTT/Wolverine.Mqtt5.Tests/Wolverine.Mqtt5.Tests.csproj index 74e652e16..dd1c17126 100644 --- a/src/Transports/MQTT/Wolverine.Mqtt5.Tests/Wolverine.Mqtt5.Tests.csproj +++ b/src/Transports/MQTT/Wolverine.Mqtt5.Tests/Wolverine.Mqtt5.Tests.csproj @@ -1,7 +1,6 @@ - Exe false true diff --git a/src/Transports/NATS/Wolverine.Nats.Tests/Wolverine.Nats.Tests.csproj b/src/Transports/NATS/Wolverine.Nats.Tests/Wolverine.Nats.Tests.csproj index 1b2d57a36..e4161bffc 100644 --- a/src/Transports/NATS/Wolverine.Nats.Tests/Wolverine.Nats.Tests.csproj +++ b/src/Transports/NATS/Wolverine.Nats.Tests/Wolverine.Nats.Tests.csproj @@ -1,7 +1,6 @@  - Exe false true diff --git a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/Wolverine.Pulsar.Tests.csproj b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/Wolverine.Pulsar.Tests.csproj index 667ceca18..824973369 100644 --- a/src/Transports/Pulsar/Wolverine.Pulsar.Tests/Wolverine.Pulsar.Tests.csproj +++ b/src/Transports/Pulsar/Wolverine.Pulsar.Tests/Wolverine.Pulsar.Tests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Transports/RabbitMQ/ChaosTesting/ChaosTesting.csproj b/src/Transports/RabbitMQ/ChaosTesting/ChaosTesting.csproj index 1d55bdcd3..1e135cb18 100644 --- a/src/Transports/RabbitMQ/ChaosTesting/ChaosTesting.csproj +++ b/src/Transports/RabbitMQ/ChaosTesting/ChaosTesting.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Transports/RabbitMQ/CircuitBreakingTests/CircuitBreakingTests.csproj b/src/Transports/RabbitMQ/CircuitBreakingTests/CircuitBreakingTests.csproj index 4f4c47af8..f3708b6b6 100644 --- a/src/Transports/RabbitMQ/CircuitBreakingTests/CircuitBreakingTests.csproj +++ b/src/Transports/RabbitMQ/CircuitBreakingTests/CircuitBreakingTests.csproj @@ -1,7 +1,6 @@  - Exe false diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Bugs/Bug_189_fails_if_there_are_many_messages_in_queue_on_startup.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Bugs/Bug_189_fails_if_there_are_many_messages_in_queue_on_startup.cs index 4389f71b7..1fe03baba 100644 --- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Bugs/Bug_189_fails_if_there_are_many_messages_in_queue_on_startup.cs +++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Bugs/Bug_189_fails_if_there_are_many_messages_in_queue_on_startup.cs @@ -6,7 +6,6 @@ namespace Wolverine.RabbitMQ.Tests.Bugs; -[Trait("Category", "Flaky")] public class Bug_189_fails_if_there_are_many_messages_in_queue_on_startup { [Fact] diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/RabbitMqBrokerHealthProbe_tests.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/RabbitMqBrokerHealthProbe_tests.cs index 6c4f5752d..088e61959 100644 --- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/RabbitMqBrokerHealthProbe_tests.cs +++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/RabbitMqBrokerHealthProbe_tests.cs @@ -8,7 +8,6 @@ namespace Wolverine.RabbitMQ.Tests; -[Trait("Category", "Flaky")] public class RabbitMqBrokerHealthProbe_tests { [Fact] diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Wolverine.RabbitMQ.Tests.csproj b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Wolverine.RabbitMQ.Tests.csproj index 586befc76..a98633820 100644 --- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Wolverine.RabbitMQ.Tests.csproj +++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Wolverine.RabbitMQ.Tests.csproj @@ -1,7 +1,6 @@ - Exe false diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/cluster_endpoints.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/cluster_endpoints.cs index e19b66fff..b23538d62 100644 --- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/cluster_endpoints.cs +++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/cluster_endpoints.cs @@ -191,7 +191,7 @@ public void compiling_virtual_host_tenant_twice_does_not_duplicate_cluster_nodes tenant.Transport.AmqpTcpEndpoints.Count.ShouldBe(2); } - [Fact, Trait("Category", "Flaky")] + [Fact] public async Task can_publish_and_receive_through_cluster_code_path() { var queueName = RabbitTesting.NextQueueName(); diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/end_to_end.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/end_to_end.cs index 0f5d7f997..3fd6bf48e 100644 --- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/end_to_end.cs +++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/end_to_end.cs @@ -36,6 +36,15 @@ public static string NextExchangeName() } } +// GH-3763: stays tagged, and the ledger entry now has numbers behind it. Measured 2026-08-02, this class +// alone against a fresh broker, three consecutive runs: 18/20 pass every time and TWO fail every time. +// +// send_message_to_and_receive_through_rabbitmq_with_routing_key fails 3 of 3 (deterministic) +// use_direct_exchange_with_binding_key / use_fan_out_exchange one or the other, every run +// +// So this is one hard failure plus a genuine flake, not one flaky class. Both fail in under 500ms, which +// is the exchange/binding-declaration race described in #2618 rather than anything timing out. Needs the +// deterministic binding-readiness gate that issue calls for before it can come off the list. [Trait("Category", "Flaky")] public class end_to_end { diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/endpoint_health_connection_state_3231.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/endpoint_health_connection_state_3231.cs index acc44e9cb..e9af97bd6 100644 --- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/endpoint_health_connection_state_3231.cs +++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/endpoint_health_connection_state_3231.cs @@ -23,7 +23,6 @@ public void Handle(EndpointHealthConnectionStateMessage message) // GH-3231: EndpointHealthSnapshot must surface the underlying transport channel/agent connection state so external // monitors (CritterWatch) can see a dead-but-"Accepting" listener (or a disconnected sender) directly rather than // inferring it from staleness. -[Trait("Category", "Flaky")] public class endpoint_health_connection_state_3231 { private static string nextQueue() => "conn_state_" + Guid.NewGuid().ToString("N"); diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/force_restart_listener_3232.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/force_restart_listener_3232.cs index 0336c709b..fcae376a8 100644 --- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/force_restart_listener_3232.cs +++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/force_restart_listener_3232.cs @@ -20,7 +20,6 @@ public void Handle(ForceRestartMessage message) // GH-3232: an operator/monitor must be able to force-recover a listener that reports Accepting but isn't actually // consuming (a stuck transport channel the framework can't self-heal) without a process bounce. Bare StartAsync() // is a no-op when Status == Accepting; RestartAsync(force: true) tears down and rebuilds regardless. -[Trait("Category", "Flaky")] public class force_restart_listener_3232 { private static string nextQueue() => "force_restart_" + Guid.NewGuid().ToString("N"); diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/multi_tenancy_through_virtual_hosts.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/multi_tenancy_through_virtual_hosts.cs index 3e91c59f1..4cb395301 100644 --- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/multi_tenancy_through_virtual_hosts.cs +++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/multi_tenancy_through_virtual_hosts.cs @@ -144,6 +144,19 @@ private static async Task declareVirtualHost(string vhname) } } +// GH-3763: kept tagged, but this is NOT flakiness and the tag should not be read as "sometimes fails". +// `send_message_to_a_specific_tenant` is DETERMINISTIC in both directions, measured on 2026-08-02: +// +// this class alone 7/7 pass, 3 runs +// this class alongside the other Rabbit suites fails every time, 3 runs +// +// It fails in ~100ms — far short of its own 15s tracked-session timeout — so nothing is timing out; +// the send is rejected or misrouted immediately. That is cross-class state on a shared broker, which is +// the #1 shape called out in #3763, and it needs the interfering suite identified rather than another +// retry budget. Untagging it would put a guaranteed red in CIRabbitMQ. +// +// The other eight Rabbit classes that carried this tag were untagged in the same pass: 5 consecutive +// clean runs, 41 tests, zero failures. [Trait("Category", "Flaky")] public class multi_tenancy_through_virtual_hosts : IClassFixture { diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/send_by_topics.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/send_by_topics.cs index 63e9fc207..5ac96c955 100644 --- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/send_by_topics.cs +++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/send_by_topics.cs @@ -22,7 +22,6 @@ namespace Wolverine.RabbitMQ.Tests; // races-with-broker-state flake. Skipping in CI via the Flaky filter; revisit // once the topic-binding setup is rewritten with a deterministic readiness gate. // See #2618. -[Trait("Category", "Flaky")] public class send_by_topics : IAsyncLifetime { private IHost theGreenReceiver = null!; @@ -289,7 +288,6 @@ public async Task publish_by_user_message_topic_logic_and_delay() // reliably miss the second receiver in CI when this class runs in the full suite. // Skip via the same Flaky filter the non-durable sibling uses, pending the // deterministic topic-binding readiness gate described in #2618. -[Trait("Category", "Flaky")] public class send_by_topics_durable : IAsyncLifetime { private IHost theGreenReceiver = null!; diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/sending_raw_messages.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/sending_raw_messages.cs index 2324e0788..2533b7b0b 100644 --- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/sending_raw_messages.cs +++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/sending_raw_messages.cs @@ -17,7 +17,6 @@ namespace Wolverine.RabbitMQ.Tests; // to re-declare it without one. Skipping in CI via the Flaky filter; the real // fix is to stop sharing fixed queue names like 'messages1' across tests // (use Guid-suffixed names) or to delete-then-redeclare in setup. See #2618. -[Trait("Category", "Flaky")] public class sending_raw_messages { [Fact] diff --git a/src/Transports/Redis/Wolverine.Redis.Tests/Wolverine.Redis.Tests.csproj b/src/Transports/Redis/Wolverine.Redis.Tests/Wolverine.Redis.Tests.csproj index f74c4e878..6eb3a1ed0 100644 --- a/src/Transports/Redis/Wolverine.Redis.Tests/Wolverine.Redis.Tests.csproj +++ b/src/Transports/Redis/Wolverine.Redis.Tests/Wolverine.Redis.Tests.csproj @@ -1,7 +1,6 @@ - Exe false true diff --git a/src/Transports/SignalR/Wolverine.SignalR.Tests/Wolverine.SignalR.Tests.csproj b/src/Transports/SignalR/Wolverine.SignalR.Tests/Wolverine.SignalR.Tests.csproj index 723a77cd8..b9d0eeb58 100644 --- a/src/Transports/SignalR/Wolverine.SignalR.Tests/Wolverine.SignalR.Tests.csproj +++ b/src/Transports/SignalR/Wolverine.SignalR.Tests/Wolverine.SignalR.Tests.csproj @@ -1,7 +1,6 @@ - Exe Exe false net9.0 diff --git a/src/Wolverine.HealthChecks.Tests/Wolverine.HealthChecks.Tests.csproj b/src/Wolverine.HealthChecks.Tests/Wolverine.HealthChecks.Tests.csproj index 65b2def0c..ff03ad340 100644 --- a/src/Wolverine.HealthChecks.Tests/Wolverine.HealthChecks.Tests.csproj +++ b/src/Wolverine.HealthChecks.Tests/Wolverine.HealthChecks.Tests.csproj @@ -1,7 +1,6 @@ - Exe false net9.0