From 3a6e347c960ba89e75af66cfe9965d8b5d62bdfc Mon Sep 17 00:00:00 2001 From: Anne Erdtsieck Date: Tue, 30 Jun 2026 10:49:49 +0200 Subject: [PATCH 01/18] Managed distribution: fan out per-tenant agents for sharded + tenant-partitioned stores EventStoreAgents.SupportedAgentsAsync now expands each async shard into one agent URI per (database, tenant) when the store reports DistributesAgentsPerTenant, registering the per-tenant ShardName (ForTenant) so ResolveShardNameAsync round-trips it and the daemon starts a per-tenant agent. Databases with no tenants keep the store-global agent. Fixes co-located tenants on a shard sharing one high-water (lagging tenant skipped). See #3280. --- .../Runtime/Agents/EventStoreAgents.cs | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs index 1767cd722..24038f6a7 100644 --- a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs +++ b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs @@ -87,18 +87,43 @@ public async ValueTask> SupportedAgentsAsync(CancellationToke var usage = await _store.TryCreateUsage(cancellation); if (usage == null) return list; + var asyncShards = usage.Subscriptions + .Where(x => x.Lifecycle == ProjectionLifecycle.Async) + .SelectMany(x => x.ShardNames) + .ToArray(); + + // wolverine#3280: under sharded databases + per-tenant event partitioning, multiple tenants are + // co-located in one shard database and each draws its own event sequence, so a single store-global + // agent per (shard, database) cannot track them — fan out one agent per (shard, tenant) instead. + // For any other store (or a database with no tenants yet) keep the one store-global agent. + void addAgentsForShard(DatabaseDescriptor database, DatabaseId id, ShardName shardName) + { + if (_store.DistributesAgentsPerTenant && database.TenantIds.Count > 0) + { + foreach (var tenantId in database.TenantIds.Distinct()) + { + var tenantShard = shardName.ForTenant(tenantId); + _shardNames = _shardNames.AddOrUpdate(tenantShard.RelativeUrl, tenantShard); + list.Add(EventSubscriptionAgentFamily.UriFor(_store.Identity, id, tenantShard)); + } + } + else + { + _shardNames = _shardNames.AddOrUpdate(shardName.RelativeUrl, shardName); + list.Add(EventSubscriptionAgentFamily.UriFor(_store.Identity, id, shardName)); + } + } + // Using this to keep from double dipping var databaseIds = new List(); foreach (var database in usage.Database.Databases) { var id = new DatabaseId(database.ServerName, database.DatabaseName); databaseIds.Add(id); - - foreach (var shardName in usage.Subscriptions.Where(x => x.Lifecycle == ProjectionLifecycle.Async).SelectMany(x => x.ShardNames)) + + foreach (var shardName in asyncShards) { - _shardNames = _shardNames.AddOrUpdate(shardName.RelativeUrl, shardName); - var uri = EventSubscriptionAgentFamily.UriFor(_store.Identity, id, shardName); - list.Add(uri); + addAgentsForShard(database, id, shardName); } } @@ -108,11 +133,9 @@ public async ValueTask> SupportedAgentsAsync(CancellationToke var id = new DatabaseId(database.ServerName, database.DatabaseName); if (!databaseIds.Contains(id)) { - foreach (var shardName in usage.Subscriptions.Where(x => x.Lifecycle == ProjectionLifecycle.Async).SelectMany(x => x.ShardNames)) + foreach (var shardName in asyncShards) { - _shardNames = _shardNames.AddOrUpdate(shardName.RelativeUrl, shardName); - var uri = EventSubscriptionAgentFamily.UriFor(_store.Identity, id, shardName); - list.Add(uri); + addAgentsForShard(database, id, shardName); } } } From 27102239f47b0fec18cba006e7b25738b7db48dd Mon Sep 17 00:00:00 2001 From: Anne Erdtsieck Date: Fri, 3 Jul 2026 19:33:58 +0200 Subject: [PATCH 02/18] Test: EventStoreAgents fans out per-tenant when the store distributes agents per tenant Covers SupportedAgentsAsync emitting one agent URI per tenant when IEventStore.DistributesAgentsPerTenant is true and the database has assigned tenants, and falling back to per-shard when it is not. --- ...nt_store_agents_per_tenant_distribution.cs | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/Testing/CoreTests/Runtime/Agents/event_store_agents_per_tenant_distribution.cs diff --git a/src/Testing/CoreTests/Runtime/Agents/event_store_agents_per_tenant_distribution.cs b/src/Testing/CoreTests/Runtime/Agents/event_store_agents_per_tenant_distribution.cs new file mode 100644 index 000000000..6f5d240bb --- /dev/null +++ b/src/Testing/CoreTests/Runtime/Agents/event_store_agents_per_tenant_distribution.cs @@ -0,0 +1,94 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using JasperFx.Descriptors; +using JasperFx.Events; +using JasperFx.Events.Daemon; +using JasperFx.Events.Descriptors; +using JasperFx.Events.Projections; +using NSubstitute; +using Shouldly; +using Wolverine.Runtime.Agents; +using Xunit; + +namespace CoreTests.Runtime.Agents; + +/// +/// wolverine#3280: under sharded databases + per-tenant event partitioning multiple tenants are +/// co-located in one shard database, each drawing its own event sequence, so a single store-global agent +/// per (shard, database) cannot track them. When the store reports , +/// must fan out one agent per (shard, tenant); for any +/// other store it must keep the single store-global agent. +/// +public class event_store_agents_per_tenant_distribution +{ + private static EventStoreUsage UsageWithOneAsyncShardOnADatabaseWith(params string[] tenantIds) + { + return new EventStoreUsage + { + Database = new DatabaseUsage + { + Databases = + { + new DatabaseDescriptor + { + ServerName = "localhost", + DatabaseName = "claims1", + TenantIds = tenantIds.ToList() + } + } + }, + Subscriptions = + { + new SubscriptionDescriptor(SubscriptionType.SingleStreamProjection) + { + Lifecycle = ProjectionLifecycle.Async, + ShardNames = new[] { ShardName.Compose("provided-cares") } + } + } + }; + } + + private static EventStoreAgents AgentsFor(EventStoreUsage usage, bool distributesPerTenant) + { + var store = Substitute.For(); + store.Identity.Returns(new EventStoreIdentity("main", "marten")); + store.DistributesAgentsPerTenant.Returns(distributesPerTenant); + store.TryCreateUsage(Arg.Any()).Returns(Task.FromResult(usage)); + return new EventStoreAgents(store, Array.Empty>()); + } + + [Fact] + public async Task fans_out_one_agent_per_tenant_when_the_store_distributes_per_tenant() + { + var agents = AgentsFor(UsageWithOneAsyncShardOnADatabaseWith("01059910", "08004464"), distributesPerTenant: true); + + var uris = await agents.SupportedAgentsAsync(CancellationToken.None); + + uris.Count.ShouldBe(2, "one async shard on a two-tenant database must yield one agent per tenant"); + uris.ShouldContain(u => u.AbsolutePath.TrimEnd('/').EndsWith("/01059910", StringComparison.OrdinalIgnoreCase)); + uris.ShouldContain(u => u.AbsolutePath.TrimEnd('/').EndsWith("/08004464", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task keeps_a_single_store_global_agent_when_the_store_does_not_distribute_per_tenant() + { + var agents = AgentsFor(UsageWithOneAsyncShardOnADatabaseWith("01059910", "08004464"), distributesPerTenant: false); + + var uris = await agents.SupportedAgentsAsync(CancellationToken.None); + + uris.Count.ShouldBe(1, "without per-tenant distribution a shard keeps its single store-global agent"); + uris.ShouldNotContain(u => u.AbsolutePath.TrimEnd('/').EndsWith("/01059910", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task keeps_a_single_agent_when_the_database_has_no_tenants_yet() + { + var agents = AgentsFor(UsageWithOneAsyncShardOnADatabaseWith(), distributesPerTenant: true); + + var uris = await agents.SupportedAgentsAsync(CancellationToken.None); + + uris.Count.ShouldBe(1, "a per-tenant store with no tenants provisioned yet keeps one store-global agent"); + } +} From 0a20c9882a7e4b38218eb9d43848f368bed296da Mon Sep 17 00:00:00 2001 From: Anne Erdtsieck Date: Sat, 4 Jul 2026 00:22:13 +0200 Subject: [PATCH 03/18] Database-affine agent assignment (AssignmentGrid.DistributeByGroupAffinity) When a sharded store sets IEventStore.GroupAgentAssignmentsByDatabase, EventSubscriptionAgentFamily assigns event-subscription agents grouped by their shard database (URI database segment) via a new greedy least-loaded DistributeByGroupAffinity, keeping every agent of a database on one node and balancing total agent count across nodes. Each node then opens connection pools only to the databases it owns. Default path (even distribution) unchanged. Unit test covers the grouping + balance. See JasperFx/marten#4806. --- .../Agents/distribute_by_group_affinity.cs | 65 ++++++++++++++++++ .../Agents/AssignmentGrid.Distribution.cs | 67 +++++++++++++++++++ .../Runtime/Agents/EventStoreAgents.cs | 8 +++ .../Agents/EventSubscriptionAgentFamily.cs | 22 +++++- 4 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs diff --git a/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs b/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs new file mode 100644 index 000000000..430a777e6 --- /dev/null +++ b/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs @@ -0,0 +1,65 @@ +using Wolverine.Runtime.Agents; +using Xunit; + +namespace CoreTests.Runtime.Agents; + +// JasperFx/marten#4806: AssignmentGrid.DistributeByGroupAffinity keeps every agent of a group (e.g. one +// shard database) on the same node, and spreads whole groups across nodes — so a node opens connection +// pools only to the databases it owns, instead of every node touching every shard database. +public class distribute_by_group_affinity +{ + // Group key = the database segment of an event-subscriptions agent URI: + // event-subscriptions://{type}/{name}/{databaseId}/{shard...} -> Segments[2]. + private static string DatabaseKey(Uri uri) => uri.Segments[2].Trim('/'); + + private static Uri Agent(string db, string tenant) => + new($"event-subscriptions://marten/main/{db}/Proj:All:{tenant}"); + + [Fact] + public void keeps_a_databases_agents_together_and_spreads_databases_across_nodes() + { + var grid = new AssignmentGrid(); + grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + + // 4 shard databases, 3 tenants each = 12 per-tenant agents. + var agents = new List(); + foreach (var db in new[] { "db1", "db2", "db3", "db4" }) + foreach (var tenant in new[] { "t1", "t2", "t3" }) + agents.Add(Agent(db, tenant)); + + grid.WithAgents(agents.ToArray()); + + grid.DistributeByGroupAffinity("event-subscriptions", DatabaseKey); + + // Every tenant agent of a database lands on exactly one node (the database is never split). + foreach (var db in new[] { "db1", "db2", "db3", "db4" }) + { + var nodes = new[] { "t1", "t2", "t3" } + .Select(t => grid.AgentFor(Agent(db, t)).AssignedNode) + .Distinct() + .ToList(); + + nodes.Count.ShouldBe(1, $"all agents of {db} must be on a single node"); + nodes[0].ShouldNotBeNull(); + } + + // All agents are assigned, and the 4 databases are spread across both nodes (2 each). + grid.AllAgents.ShouldAllBe(a => a.AssignedNode != null); + var perNode = grid.Nodes.Select(n => n.ForScheme("event-subscriptions").Count()).OrderBy(x => x).ToList(); + perNode.ShouldBe(new[] { 6, 6 }); + } + + [Fact] + public void single_node_takes_every_agent() + { + var grid = new AssignmentGrid(); + var node = grid.WithNode(1, Guid.NewGuid()); + + grid.WithAgents(Agent("db1", "t1"), Agent("db1", "t2"), Agent("db2", "t1")); + + grid.DistributeByGroupAffinity("event-subscriptions", DatabaseKey); + + node.ForScheme("event-subscriptions").Count().ShouldBe(3); + } +} diff --git a/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs b/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs index 6489a50cd..f280f18e7 100644 --- a/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs +++ b/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs @@ -83,6 +83,73 @@ public void DistributeEvenly(string scheme) } } + /// + /// Distribute agents of a scheme across nodes with group affinity: every agent whose + /// is equal is placed on the same node, and whole groups are spread + /// evenly across nodes. Intended for sharded event stores where each agent's group is its shard + /// database (JasperFx/marten#4806) — co-locating a database's agents on one node bounds that node + /// to the databases it owns, so connection pools scale with databases rather than nodes × databases. + /// + /// Balanced by a greedy least-loaded fit: each database group (largest first) goes to the node + /// with the fewest agents so far, so nodes end up with roughly equal total agent counts — a + /// proxy for per-tenant work, since a database contributes one agent per resident tenant × projection — + /// rather than merely an equal number of databases. It balances by agent/tenant count, not by raw event + /// volume, so a database that is few-tenants-but-huge can still be heavier than its agent count implies. + /// Deterministic tie-breaks (prefer non-leaders, then node id) keep a steady grid from churning. This + /// prototype does not yet apply blue/green capability matching; use the standard + /// when that is required. + /// + public void DistributeByGroupAffinity(string scheme, Func groupKey) + { + if (_nodes.Count == 0) + { + throw new InvalidOperationException("There are no active nodes"); + } + + var agents = AvailableAgentsForScheme(scheme); + if (agents.Count == 0) + { + return; + } + + var nodes = _nodes.OrderBy(x => x.IsLeader).ThenBy(x => x.AssignedId).ToList(); + + if (nodes.Count == 1) + { + var only = nodes[0]; + foreach (var agent in agents) + { + only.Assign(agent); + } + + return; + } + + var groups = agents + .GroupBy(a => groupKey(a.Uri)) + .OrderByDescending(g => g.Count()) + .ThenBy(g => g.Key, StringComparer.Ordinal) + .ToList(); + + var agentCountByNode = nodes.ToDictionary(n => n, _ => 0); + foreach (var group in groups) + { + // Least-loaded node wins; prefer non-leaders and lower node ids on ties for a stable result. + var node = agentCountByNode + .OrderBy(kv => kv.Value) + .ThenBy(kv => kv.Key.IsLeader) + .ThenBy(kv => kv.Key.AssignedId) + .First().Key; + + foreach (var agent in group) + { + node.Assign(agent); + } + + agentCountByNode[node] += group.Count(); + } + } + public bool AllNodesHaveSameCapabilities(string scheme) { var gold = _nodes[0].OrderedCapabilitiesForScheme(scheme); diff --git a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs index 24038f6a7..cbb5d4104 100644 --- a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs +++ b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs @@ -30,6 +30,14 @@ public EventStoreAgents(IEventStore store, IObserver[] observers) public EventStoreIdentity Identity { get; } + /// + /// When true (opt-in, sharded stores only), this store's per-(shard, tenant) agents should be assigned + /// with database affinity — all agents for a shard database on one node — so a node opens pools only to + /// the databases it owns. Surfaces . See + /// JasperFx/marten#4806. + /// + public bool GroupAgentAssignmentsByDatabase => _store.GroupAgentAssignmentsByDatabase; + public async ValueTask DisposeAsync() { foreach (var entry in _daemons.Enumerate()) diff --git a/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs b/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs index 74b564a60..07d8db859 100644 --- a/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs +++ b/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs @@ -156,10 +156,30 @@ public async ValueTask> SupportedAgentsAsync() public ValueTask EvaluateAssignmentsAsync(AssignmentGrid assignments) { - assignments.DistributeEvenlyWithBlueGreenSemantics(SchemeName); + // JasperFx/marten#4806 (opt-in): when a sharded store asks for database-affine assignment, keep a + // shard database's per-tenant agents together on one node so a node opens connection pools only to + // the databases it owns (pools scale with databases, not nodes × databases, which otherwise + // exhausts a shared server). Otherwise distribute evenly with blue/green semantics as before. + if (_stores.Enumerate().Any(e => e.Value.GroupAgentAssignmentsByDatabase)) + { + assignments.DistributeByGroupAffinity(SchemeName, DatabaseKeyOf); + } + else + { + assignments.DistributeEvenlyWithBlueGreenSemantics(SchemeName); + } + return new ValueTask(); } + // Agent URIs are event-subscriptions://{type}/{name}/{databaseId}/{shard...} (see UriFor); the + // (type, name, databaseId) prefix identifies the shard database an agent belongs to, so grouping on it + // co-locates every tenant/projection agent for one database on the same node. + internal static string DatabaseKeyOf(Uri uri) + => uri.Segments.Length >= 3 + ? $"{uri.Host}/{uri.Segments[1].Trim('/')}/{uri.Segments[2].Trim('/')}" + : uri.AbsoluteUri; + public async ValueTask DisposeAsync() { foreach (var store in _stores.Enumerate()) From ff0c8be148e302263176e1b1cca1ab0dca9d7d68 Mon Sep 17 00:00:00 2001 From: Anne Erdtsieck Date: Sat, 4 Jul 2026 10:42:57 +0200 Subject: [PATCH 04/18] Bounded fan-out for database-affine agent assignment (the mix) DistributeByGroupAffinity gains a maxNodesPerGroup bound: a shard database's agents are placed on its k = min(bound, size, nodes) least-loaded nodes and greedily balanced, so a heavy database parallelizes across up to N nodes while its server connection ceiling stays N x pool. EventSubscriptionAgentFamily passes the largest MaxNodesPerDatabaseForAgents among affine stores. maxNodesPerGroup=1 is the prior strict-affinity behavior. Tests cover fan-out + small-group cases. See JasperFx/marten#4806. --- .../Agents/distribute_by_group_affinity.cs | 36 ++++++++++++ .../Agents/AssignmentGrid.Distribution.cs | 57 ++++++++++++------- .../Runtime/Agents/EventStoreAgents.cs | 6 ++ .../Agents/EventSubscriptionAgentFamily.cs | 13 ++++- 4 files changed, 89 insertions(+), 23 deletions(-) diff --git a/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs b/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs index 430a777e6..14e8d6339 100644 --- a/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs +++ b/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs @@ -62,4 +62,40 @@ public void single_node_takes_every_agent() node.ForScheme("event-subscriptions").Count().ShouldBe(3); } + + [Fact] + public void bounded_fan_out_spreads_a_heavy_database_across_up_to_k_nodes() + { + var grid = new AssignmentGrid(); + grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + grid.WithNode(3, Guid.NewGuid()); + + // One heavy database with 6 tenants; bound = 2 → its agents span exactly 2 nodes (not 1, not 3). + var agents = Enumerable.Range(1, 6).Select(t => Agent("dbHeavy", "t" + t)).ToArray(); + grid.WithAgents(agents); + + grid.DistributeByGroupAffinity("event-subscriptions", DatabaseKey, maxNodesPerGroup: 2); + + var nodesUsed = agents.Select(a => grid.AgentFor(a).AssignedNode).Distinct().ToList(); + nodesUsed.Count.ShouldBe(2, "a heavy database must fan out across up to the bound (2) nodes"); + nodesUsed.ShouldAllBe(n => n != null); + } + + [Fact] + public void a_group_smaller_than_the_bound_uses_fewer_nodes() + { + var grid = new AssignmentGrid(); + grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + grid.WithNode(3, Guid.NewGuid()); + + // A single-tenant database can only ever use one node, even with a higher bound. + grid.WithAgents(Agent("dbLight", "t1")); + + grid.DistributeByGroupAffinity("event-subscriptions", DatabaseKey, maxNodesPerGroup: 3); + + grid.AgentFor(Agent("dbLight", "t1")).AssignedNode.ShouldNotBeNull(); + grid.Nodes.Count(n => n.ForScheme("event-subscriptions").Any()).ShouldBe(1); + } } diff --git a/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs b/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs index f280f18e7..bbeb33580 100644 --- a/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs +++ b/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs @@ -84,22 +84,21 @@ public void DistributeEvenly(string scheme) } /// - /// Distribute agents of a scheme across nodes with group affinity: every agent whose - /// is equal is placed on the same node, and whole groups are spread - /// evenly across nodes. Intended for sharded event stores where each agent's group is its shard - /// database (JasperFx/marten#4806) — co-locating a database's agents on one node bounds that node - /// to the databases it owns, so connection pools scale with databases rather than nodes × databases. + /// Distribute agents of a scheme across nodes with bounded group affinity: agents that share a + /// (e.g. a shard database) are kept together, but a group may fan out across + /// up to nodes. This is the "mix" between strict affinity + /// (maxNodesPerGroup == 1 — fewest connections, but a heavy group is serialized on one node's + /// pool) and even spreading: a higher bound lets a heavy group parallelize across several nodes while + /// still capping how many nodes reach that group's database, so its server-side connection ceiling stays + /// bounded at (bound × per-node pool). Intended for sharded event stores (JasperFx/marten#4806). /// - /// Balanced by a greedy least-loaded fit: each database group (largest first) goes to the node - /// with the fewest agents so far, so nodes end up with roughly equal total agent counts — a - /// proxy for per-tenant work, since a database contributes one agent per resident tenant × projection — - /// rather than merely an equal number of databases. It balances by agent/tenant count, not by raw event - /// volume, so a database that is few-tenants-but-huge can still be heavier than its agent count implies. - /// Deterministic tie-breaks (prefer non-leaders, then node id) keep a steady grid from churning. This - /// prototype does not yet apply blue/green capability matching; use the standard - /// when that is required. + /// Groups are placed largest-first onto their k = min(bound, group size, node count) least-loaded + /// nodes; each group's agents then greedily fill those nodes least-loaded-first, so total agent count + /// stays balanced and the result is deterministic (a steady grid does not churn). A group with fewer + /// agents than the bound naturally uses fewer nodes. This prototype does not apply blue/green capability + /// matching; use when that is required. /// - public void DistributeByGroupAffinity(string scheme, Func groupKey) + public void DistributeByGroupAffinity(string scheme, Func groupKey, int maxNodesPerGroup = 1) { if (_nodes.Count == 0) { @@ -112,6 +111,11 @@ public void DistributeByGroupAffinity(string scheme, Func groupKey) return; } + if (maxNodesPerGroup < 1) + { + maxNodesPerGroup = 1; + } + var nodes = _nodes.OrderBy(x => x.IsLeader).ThenBy(x => x.AssignedId).ToList(); if (nodes.Count == 1) @@ -125,28 +129,39 @@ public void DistributeByGroupAffinity(string scheme, Func groupKey) return; } + var load = nodes.ToDictionary(n => n, _ => 0); + var groups = agents .GroupBy(a => groupKey(a.Uri)) .OrderByDescending(g => g.Count()) .ThenBy(g => g.Key, StringComparer.Ordinal) .ToList(); - var agentCountByNode = nodes.ToDictionary(n => n, _ => 0); foreach (var group in groups) { - // Least-loaded node wins; prefer non-leaders and lower node ids on ties for a stable result. - var node = agentCountByNode + var members = group.ToList(); + var k = Math.Min(Math.Min(maxNodesPerGroup, members.Count), nodes.Count); + + // The k least-loaded nodes host this group; a group smaller than the bound uses fewer nodes. + var chosen = load .OrderBy(kv => kv.Value) .ThenBy(kv => kv.Key.IsLeader) .ThenBy(kv => kv.Key.AssignedId) - .First().Key; + .Take(k) + .Select(kv => kv.Key) + .ToList(); - foreach (var agent in group) + // Greedily fill the group's agents onto the chosen nodes, least-loaded first, to balance work. + foreach (var agent in members) { + var node = chosen + .OrderBy(n => load[n]) + .ThenBy(n => n.IsLeader) + .ThenBy(n => n.AssignedId) + .First(); node.Assign(agent); + load[node]++; } - - agentCountByNode[node] += group.Count(); } } diff --git a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs index cbb5d4104..e98194a3a 100644 --- a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs +++ b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs @@ -38,6 +38,12 @@ public EventStoreAgents(IEventStore store, IObserver[] observers) /// public bool GroupAgentAssignmentsByDatabase => _store.GroupAgentAssignmentsByDatabase; + /// + /// Upper bound on how many nodes a single shard database's agents may fan out across under database-affine + /// assignment (the "mix"; 1 = strict affinity). Surfaces . + /// + public int MaxNodesPerDatabaseForAgents => _store.MaxNodesPerDatabaseForAgents; + public async ValueTask DisposeAsync() { foreach (var entry in _daemons.Enumerate()) diff --git a/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs b/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs index 07d8db859..a0d035684 100644 --- a/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs +++ b/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs @@ -160,9 +160,18 @@ public ValueTask EvaluateAssignmentsAsync(AssignmentGrid assignments) // shard database's per-tenant agents together on one node so a node opens connection pools only to // the databases it owns (pools scale with databases, not nodes × databases, which otherwise // exhausts a shared server). Otherwise distribute evenly with blue/green semantics as before. - if (_stores.Enumerate().Any(e => e.Value.GroupAgentAssignmentsByDatabase)) + var affineStores = _stores.Enumerate() + .Select(e => e.Value) + .Where(s => s.GroupAgentAssignmentsByDatabase) + .ToList(); + + if (affineStores.Count != 0) { - assignments.DistributeByGroupAffinity(SchemeName, DatabaseKeyOf); + // Bounded fan-out ("mix"): a shard database's agents may spread across up to this many nodes. + // One grid pass covers every store's agents (their per-store URI prefix keeps groups distinct), + // so use the largest bound any affine store asks for. + var maxNodesPerGroup = affineStores.Max(s => s.MaxNodesPerDatabaseForAgents); + assignments.DistributeByGroupAffinity(SchemeName, DatabaseKeyOf, maxNodesPerGroup); } else { From ad1efd796ef99210a2c6ccf2f121dd7297408ad6 Mon Sep 17 00:00:00 2001 From: Anne Erdtsieck Date: Sat, 4 Jul 2026 10:56:50 +0200 Subject: [PATCH 05/18] Test EvaluateAssignmentsAsync affine branch, fan-out bound + DatabaseKeyOf Covers the branch selection in EventSubscriptionAgentFamily (affine grouping when a store opts in, even blue/green spread otherwise), that the family plumbs the store's MaxNodesPerDatabaseForAgents bound into the grid, DatabaseKeyOf URI grouping/fallback, and the EventStoreAgents pass-through of both flags. Fast unit tests over an AssignmentGrid with NSubstitute stores. See JasperFx/marten#4806. Claude-Session: https://claude.ai/code/session_01LXFw8u9F71jLhMTFNi9DyX --- ...t_subscription_family_affine_assignment.cs | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 src/Testing/CoreTests/Runtime/Agents/event_subscription_family_affine_assignment.cs diff --git a/src/Testing/CoreTests/Runtime/Agents/event_subscription_family_affine_assignment.cs b/src/Testing/CoreTests/Runtime/Agents/event_subscription_family_affine_assignment.cs new file mode 100644 index 000000000..ad91e7a8d --- /dev/null +++ b/src/Testing/CoreTests/Runtime/Agents/event_subscription_family_affine_assignment.cs @@ -0,0 +1,154 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using JasperFx.Events; +using NSubstitute; +using Shouldly; +using Wolverine.Runtime.Agents; +using Xunit; + +namespace CoreTests.Runtime.Agents; + +/// +/// JasperFx/marten#4806: when a sharded store opts into , +/// must assign that store's per-(shard, tenant) +/// agents with database affinity (a database's agents kept on one node, bounded by +/// ) instead of the default even blue/green spread. These +/// are fast unit tests: agents are registered directly on an and the store flags are +/// stubbed, so no host or database is required. +/// +public class event_subscription_family_affine_assignment +{ + // Agent URI grammar is event-subscriptions://{type}/{name}/{databaseId}/{shard...} (see EventStoreAgents.UriFor). + private static Uri Agent(string db, string tenant) => + new($"event-subscriptions://marten/main/{db}/Proj:All:{tenant}"); + + private static EventSubscriptionAgentFamily FamilyFor(bool affine, int maxNodesPerDatabase = 1) + { + var store = Substitute.For(); + store.Identity.Returns(new EventStoreIdentity("main", "marten")); + store.GroupAgentAssignmentsByDatabase.Returns(affine); + store.MaxNodesPerDatabaseForAgents.Returns(maxNodesPerDatabase); + return new EventSubscriptionAgentFamily(new[] { store }, Array.Empty>()); + } + + [Fact] + public async Task keeps_a_databases_agents_together_when_the_store_opts_into_affinity() + { + var family = FamilyFor(affine: true); + + var grid = new AssignmentGrid(); + grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + // One shard database with two tenants: even distribution would split it across the two nodes. + grid.WithAgents(Agent("dbShared", "t1"), Agent("dbShared", "t2")); + + await family.EvaluateAssignmentsAsync(grid); + + var nodes = new[] { "t1", "t2" } + .Select(t => grid.AgentFor(Agent("dbShared", t)).AssignedNode) + .Distinct() + .ToList(); + + nodes.Count.ShouldBe(1, "an affine store must keep a shard database's agents on a single node"); + nodes[0].ShouldNotBeNull(); + } + + [Fact] + public async Task spreads_a_databases_agents_evenly_when_the_store_does_not_opt_in() + { + var family = FamilyFor(affine: false); + + var grid = new AssignmentGrid(); + grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + grid.WithAgents(Agent("dbShared", "t1"), Agent("dbShared", "t2")); + + await family.EvaluateAssignmentsAsync(grid); + + var nodes = new[] { "t1", "t2" } + .Select(t => grid.AgentFor(Agent("dbShared", t)).AssignedNode) + .Distinct() + .ToList(); + + nodes.Count.ShouldBe(2, "without affinity even distribution splits a two-agent database across both nodes"); + } + + [Fact] + public async Task fans_a_heavy_database_out_across_the_stores_max_nodes_bound() + { + // The family must pass the store's MaxNodesPerDatabaseForAgents through to the grid so a heavy database + // parallelizes across up to that many nodes — not one (strict affinity) and not all three (even spread). + var family = FamilyFor(affine: true, maxNodesPerDatabase: 2); + + var grid = new AssignmentGrid(); + grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + grid.WithNode(3, Guid.NewGuid()); + var agents = Enumerable.Range(1, 6).Select(t => Agent("dbHeavy", "t" + t)).ToArray(); + grid.WithAgents(agents); + + await family.EvaluateAssignmentsAsync(grid); + + var nodesUsed = agents.Select(a => grid.AgentFor(a).AssignedNode).Distinct().ToList(); + nodesUsed.Count.ShouldBe(2, "the family must honor the store's fan-out bound of 2"); + nodesUsed.ShouldAllBe(n => n != null); + } + + [Fact] + public void database_key_groups_a_databases_agents_and_separates_databases() + { + var t1 = EventSubscriptionAgentFamily.DatabaseKeyOf(Agent("claims1", "t1")); + var t2 = EventSubscriptionAgentFamily.DatabaseKeyOf(Agent("claims1", "t2")); + var other = EventSubscriptionAgentFamily.DatabaseKeyOf(Agent("claims2", "t1")); + + t1.ShouldBe("marten/main/claims1"); + t2.ShouldBe(t1, "two tenants on the same shard database must share a group key"); + other.ShouldNotBe(t1, "a different shard database must be a different group"); + } + + [Fact] + public void database_key_falls_back_to_the_whole_uri_when_there_is_no_database_segment() + { + // Defensive: a URI without the {databaseId} segment (fewer than 3 path segments) can't be grouped by + // database, so each such agent is its own group rather than silently collapsing together. + var uri = new Uri("event-subscriptions://marten/main"); + EventSubscriptionAgentFamily.DatabaseKeyOf(uri).ShouldBe(uri.AbsoluteUri); + } +} + +/// +/// surfaces the affine-assignment flags from its underlying +/// unchanged; the agent family reads them off the wrapper. See JasperFx/marten#4806. +/// +public class event_store_agents_affinity_passthrough +{ + private static EventStoreAgents AgentsFor(bool affine, int maxNodes) + { + var store = Substitute.For(); + store.Identity.Returns(new EventStoreIdentity("main", "marten")); + store.GroupAgentAssignmentsByDatabase.Returns(affine); + store.MaxNodesPerDatabaseForAgents.Returns(maxNodes); + return new EventStoreAgents(store, Array.Empty>()); + } + + [Fact] + public void surfaces_group_by_database_and_max_nodes_from_the_store() + { + var agents = AgentsFor(affine: true, maxNodes: 3); + + agents.GroupAgentAssignmentsByDatabase.ShouldBeTrue(); + agents.MaxNodesPerDatabaseForAgents.ShouldBe(3); + } + + [Fact] + public void defaults_flow_through_for_a_non_affine_store() + { + var agents = AgentsFor(affine: false, maxNodes: 1); + + agents.GroupAgentAssignmentsByDatabase.ShouldBeFalse(); + agents.MaxNodesPerDatabaseForAgents.ShouldBe(1); + } +} From b6ac166ae410f8eb8b4b92cab6fd49b547cfc665 Mon Sep 17 00:00:00 2001 From: Anne Erdtsieck Date: Sat, 4 Jul 2026 11:06:34 +0200 Subject: [PATCH 06/18] Docs: database-affine (bounded fan-out) distribution for sharded stores New section under Projection/Subscription Distribution documenting the opt-in database-affine assignment driven by Marten's UseDatabaseAffineAgentAssignment / DatabaseAffineAgentFanout, surfaced via IEventStore.GroupAgentAssignmentsByDatabase and MaxNodesPerDatabaseForAgents. See JasperFx/marten#4806. Claude-Session: https://claude.ai/code/session_01LXFw8u9F71jLhMTFNi9DyX --- docs/guide/durability/marten/distribution.md | 39 ++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/guide/durability/marten/distribution.md b/docs/guide/durability/marten/distribution.md index 6bc1381a5..fe631a4e8 100644 --- a/docs/guide/durability/marten/distribution.md +++ b/docs/guide/durability/marten/distribution.md @@ -59,6 +59,45 @@ Some other facts about this integration: new versions of a projection or subscription on some application nodes in order to do try ["blue/green deployment."](https://en.wikipedia.org/wiki/Blue%E2%80%93green_deployment) * This capability does depend on Wolverine's built-in [leadership election](https://en.wikipedia.org/wiki/Leader_election) -- which fortunately got a _lot_ better in Wolverine 3.0 +## Database-Affine Distribution for Sharded Stores + +By default Wolverine spreads subscription and projection agents *evenly* across the cluster (with blue/green +capability matching). That is the right choice for a store-global event store or database-per-tenant multi-tenancy, +where there is one agent per database anyway. + +It is *not* the right choice for a Marten store that combines [sharded multi-tenancy](https://martendb.io/configuration/multitenancy.html#sharded-multi-tenancy-with-database-pooling) +with [per-tenant event partitioning](https://martendb.io/events/multitenancy.html#per-tenant-event-partitioning). There, +many tenants are co-located in one shard database and each draws its own event sequence, so Wolverine fans agents out +one-per-`(shard, tenant)` rather than one-per-database. With hundreds of tenants scattered across many shard databases, +an even per-agent spread makes *every* node open a connection pool to *nearly every* shard database — so the pool count +grows as `nodes × databases` and quickly exhausts a shared server's `max_connections`. + +To bound that, opt into **database-affine assignment** on the Marten side. Wolverine then keeps every agent for a given +shard database together on a single node, so a node only opens pools to the databases it actually owns and the pool +count scales with the number of shard databases, not `nodes × databases`: + +```cs +services.AddMarten(opts => +{ + // ... sharded tenancy + per-tenant event partitioning ... + opts.Events.UseTenantPartitionedEvents = true; + + // Keep a shard database's per-tenant agents together on one node + opts.Events.UseDatabaseAffineAgentAssignment = true; + + // The "mix": allow a single shard database's agents to fan out across up to N nodes + // for more parallelism (default 1 = strict affinity). Server-side connections for one + // database then stay bounded at roughly N × per-node pool size. + opts.Events.DatabaseAffineAgentFanout = 2; +}) +.IntegrateWithWolverine(); +``` + +Wolverine reads these off the store through `IEventStore.GroupAgentAssignmentsByDatabase` and +`IEventStore.MaxNodesPerDatabaseForAgents`, so the behavior is entirely opt-in and store-driven — a store that does not +set them keeps the default even distribution. The grouping key is the `[event store type]/[event store name]/[database]` +prefix of the agent `Uri` (see below), so all of a shard database's per-tenant agents share one group. + ## Uri Structure The `Uri` structure for event subscriptions or projections is: From 6db8484a0a5d601d9a808ccfa035cfaaa4a651fd Mon Sep 17 00:00:00 2001 From: Anne Erdtsieck Date: Sat, 4 Jul 2026 12:32:41 +0200 Subject: [PATCH 07/18] IntegrateWithWolverine: database-affine agent assignment settings MartenIntegration gains UseDatabaseAffineAgentAssignment + DatabaseAffineAgentFanout, bridged onto the Marten event store (main store only) via the existing MartenOverrides IConfigureMarten hook, so the database-affine distribution can be configured where you opt into Wolverine-managed distribution rather than only on StoreOptions.Events. Doc updated to show the IntegrateWithWolverine surface. See JasperFx/marten#4806. Claude-Session: https://claude.ai/code/session_01LXFw8u9F71jLhMTFNi9DyX --- docs/guide/durability/marten/distribution.md | 29 ++++++++------ .../Wolverine.Marten/MartenIntegration.cs | 38 ++++++++++++++++++- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/docs/guide/durability/marten/distribution.md b/docs/guide/durability/marten/distribution.md index fe631a4e8..c6063e907 100644 --- a/docs/guide/durability/marten/distribution.md +++ b/docs/guide/durability/marten/distribution.md @@ -81,22 +81,27 @@ services.AddMarten(opts => { // ... sharded tenancy + per-tenant event partitioning ... opts.Events.UseTenantPartitionedEvents = true; +}) +.IntegrateWithWolverine(x => +{ + x.UseWolverineManagedEventSubscriptionDistribution = true; - // Keep a shard database's per-tenant agents together on one node - opts.Events.UseDatabaseAffineAgentAssignment = true; + // Keep a shard database's per-tenant agents together on one node. + x.UseDatabaseAffineAgentAssignment = true; - // The "mix": allow a single shard database's agents to fan out across up to N nodes - // for more parallelism (default 1 = strict affinity). Server-side connections for one - // database then stay bounded at roughly N × per-node pool size. - opts.Events.DatabaseAffineAgentFanout = 2; -}) -.IntegrateWithWolverine(); + // The "mix": allow a single shard database's agents to fan out across up to N nodes for more + // parallelism (default 1 = strict affinity). Server-side connections for one database then stay + // bounded at roughly N × per-node pool size. + x.DatabaseAffineAgentFanout = 2; +}); ``` -Wolverine reads these off the store through `IEventStore.GroupAgentAssignmentsByDatabase` and -`IEventStore.MaxNodesPerDatabaseForAgents`, so the behavior is entirely opt-in and store-driven — a store that does not -set them keeps the default even distribution. The grouping key is the `[event store type]/[event store name]/[database]` -prefix of the agent `Uri` (see below), so all of a shard database's per-tenant agents share one group. +These settings on `IntegrateWithWolverine` flow through to `StoreOptions.Events.UseDatabaseAffineAgentAssignment` +and `DatabaseAffineAgentFanout` (you can also set them there directly). Wolverine reads them off the store through +`IEventStore.GroupAgentAssignmentsByDatabase` and `IEventStore.MaxNodesPerDatabaseForAgents`, so the behavior is entirely +opt-in and store-driven — a store that does not set them keeps the default even distribution. The grouping key is the +`[event store type]/[event store name]/[database]` prefix of the agent `Uri` (see below), so all of a shard database's +per-tenant agents share one group. ## Uri Structure diff --git a/src/Persistence/Wolverine.Marten/MartenIntegration.cs b/src/Persistence/Wolverine.Marten/MartenIntegration.cs index abceea335..06831ef66 100644 --- a/src/Persistence/Wolverine.Marten/MartenIntegration.cs +++ b/src/Persistence/Wolverine.Marten/MartenIntegration.cs @@ -40,12 +40,34 @@ public class MartenIntegration : IWolverineExtension, IEventForwarding public bool UseFastEventForwarding { get; set; } /// - /// Use this when using Wolverine to evenly distribute event projection and subscription - /// work of Marten asynchronous projections. This replaces Marten's AddAsyncDaemon(HotCold) + /// Use this when using Wolverine to evenly distribute event projection and subscription + /// work of Marten asynchronous projections. This replaces Marten's AddAsyncDaemon(HotCold) /// option and should not be used in combination with Marten's own load distribution. /// public bool UseWolverineManagedEventSubscriptionDistribution { get; set; } + /// + /// Opt-in (default false): when is on and + /// the Marten store uses sharded databases with per-tenant event partitioning, assign each shard + /// database's per-(shard, tenant) agents with database affinity — all of a database's agents run + /// on one node — instead of spreading them evenly by count. This bounds each node to the shard databases + /// it owns, so connection pools scale with the number of shard databases rather than nodes × databases + /// (which otherwise exhausts a shared server's max_connections). Flows through to + /// StoreOptions.Events.UseDatabaseAffineAgentAssignment; only takes effect for a sharded, + /// per-tenant-partitioned store. See JasperFx/marten#4806. + /// + public bool UseDatabaseAffineAgentAssignment { get; set; } + + /// + /// When is on, the maximum number of nodes a single shard + /// database's agents may fan out across (the "mix" between strict affinity and even spreading). 1 = strict + /// affinity (one node per database — fewest connections). A higher value lets a heavy database parallelize + /// across up to N least-loaded nodes, at the cost of that database being reachable from up to N nodes + /// (server-side connection ceiling per database ≈ N × per-node pool). Flows through to + /// StoreOptions.Events.DatabaseAffineAgentFanout. Default 1. See JasperFx/marten#4806. + /// + public int DatabaseAffineAgentFanout { get; set; } = 1; + public void Configure(WolverineOptions options) { // Duplicate incoming messages @@ -201,6 +223,18 @@ public void Configure(IServiceProvider services, StoreOptions options) nameof(Saga.Version), typeof(int))!; } }); + + // Bridge the Wolverine-managed distribution's database-affine agent-assignment settings — configured + // on IntegrateWithWolverine — onto the Marten event store, because Wolverine's distribution is what + // consumes them (via IEventStore.GroupAgentAssignmentsByDatabase / MaxNodesPerDatabaseForAgents). + // Only the main store (StoreType == null): this is a sharded per-tenant-store concern, not ancillary. + // See JasperFx/marten#4806. + if (StoreType == null && + services.GetService() is { UseDatabaseAffineAgentAssignment: true } integration) + { + options.Events.UseDatabaseAffineAgentAssignment = true; + options.Events.DatabaseAffineAgentFanout = integration.DatabaseAffineAgentFanout; + } } } From 49c0644fe3021623c1bcfd7f94513197b12a1ec0 Mon Sep 17 00:00:00 2001 From: Anne Erdtsieck Date: Sat, 4 Jul 2026 13:26:49 +0200 Subject: [PATCH 08/18] Test: IntegrateWithWolverine affine settings flow to the event store Guards the MartenOverrides bridge (resolving MartenIntegration and applying the settings to StoreOptions.Events) against a DI-resolution regression: asserts UseDatabaseAffine- AgentAssignment / DatabaseAffineAgentFanout set on IntegrateWithWolverine land on Events, and that defaults are unchanged when not configured. See JasperFx/marten#4806. Claude-Session: https://claude.ai/code/session_01LXFw8u9F71jLhMTFNi9DyX --- ...ntegrate_with_wolverine_affine_settings.cs | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/Persistence/MartenTests/Distribution/integrate_with_wolverine_affine_settings.cs diff --git a/src/Persistence/MartenTests/Distribution/integrate_with_wolverine_affine_settings.cs b/src/Persistence/MartenTests/Distribution/integrate_with_wolverine_affine_settings.cs new file mode 100644 index 000000000..43d535155 --- /dev/null +++ b/src/Persistence/MartenTests/Distribution/integrate_with_wolverine_affine_settings.cs @@ -0,0 +1,66 @@ +using IntegrationTests; +using JasperFx.CodeGeneration; +using Marten; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Shouldly; +using Wolverine; +using Wolverine.Marten; +using Xunit; + +namespace MartenTests.Distribution; + +/// +/// JasperFx/marten#4806: the database-affine agent-assignment settings can be configured on +/// IntegrateWithWolverine (next to UseWolverineManagedEventSubscriptionDistribution), and they must flow +/// through to the Marten event store — which is where Wolverine's distribution reads them via +/// IEventStore.GroupAgentAssignmentsByDatabase / MaxNodesPerDatabaseForAgents. This guards the bridge +/// (MartenOverrides resolving the MartenIntegration and applying the settings) against a DI-resolution +/// regression that would silently leave the app on the default even distribution. +/// +public class integrate_with_wolverine_affine_settings +{ + private static async Task StartHostAsync(Action configure, string schema) + { + return await Host.CreateDefaultBuilder() + .UseWolverine(opts => + { + opts.Services.AddMarten(m => + { + m.DisableNpgsqlLogging = true; + m.Connection(Servers.PostgresConnectionString); + m.DatabaseSchemaName = schema; + }) + .IntegrateWithWolverine(configure); + + opts.Discovery.DisableConventionalDiscovery(); + opts.CodeGeneration.TypeLoadMode = TypeLoadMode.Auto; + }).StartAsync(); + } + + [Fact] + public async Task affine_settings_flow_to_the_event_store() + { + using var host = await StartHostAsync(m => + { + m.UseWolverineManagedEventSubscriptionDistribution = true; + m.UseDatabaseAffineAgentAssignment = true; + m.DatabaseAffineAgentFanout = 3; + }, "affine_on"); + + var store = (DocumentStore)host.Services.GetRequiredService(); + store.Options.Events.UseDatabaseAffineAgentAssignment.ShouldBeTrue(); + store.Options.Events.DatabaseAffineAgentFanout.ShouldBe(3); + } + + [Fact] + public async Task defaults_are_unchanged_when_not_configured() + { + using var host = await StartHostAsync( + m => m.UseWolverineManagedEventSubscriptionDistribution = true, "affine_off"); + + var store = (DocumentStore)host.Services.GetRequiredService(); + store.Options.Events.UseDatabaseAffineAgentAssignment.ShouldBeFalse(); + store.Options.Events.DatabaseAffineAgentFanout.ShouldBe(1); + } +} From 26b3f32a16fa607d9e75c53a7da73192b3bc4f03 Mon Sep 17 00:00:00 2001 From: Anne Erdtsieck Date: Sun, 5 Jul 2026 13:18:33 +0200 Subject: [PATCH 09/18] Drop bounded fan-out; DistributeByGroupAffinity is strict group affinity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DistributeByGroupAffinity loses the maxNodesPerGroup bound: a group (shard database) is always placed whole on the least-loaded node, largest-first, deterministic. The family/EventStoreAgents fan-out pass-throughs and the MartenIntegration fanout setting go with it — the mix was speculative and never needed >1; a minimal surface is easier to accept upstream. Tests + docs updated. See JasperFx/marten#4806. Claude-Session: https://claude.ai/code/session_01LXFw8u9F71jLhMTFNi9DyX --- docs/guide/durability/marten/distribution.md | 13 ++--- ...ntegrate_with_wolverine_affine_settings.cs | 19 +++---- .../Wolverine.Marten/MartenIntegration.cs | 19 ++----- .../Agents/distribute_by_group_affinity.cs | 41 +++++++-------- ...t_subscription_family_affine_assignment.cs | 52 ++++--------------- .../Agents/AssignmentGrid.Distribution.cs | 48 ++++++----------- .../Runtime/Agents/EventStoreAgents.cs | 6 --- .../Agents/EventSubscriptionAgentFamily.cs | 15 ++---- 8 files changed, 67 insertions(+), 146 deletions(-) diff --git a/docs/guide/durability/marten/distribution.md b/docs/guide/durability/marten/distribution.md index c6063e907..fd6477a2c 100644 --- a/docs/guide/durability/marten/distribution.md +++ b/docs/guide/durability/marten/distribution.md @@ -88,18 +88,13 @@ services.AddMarten(opts => // Keep a shard database's per-tenant agents together on one node. x.UseDatabaseAffineAgentAssignment = true; - - // The "mix": allow a single shard database's agents to fan out across up to N nodes for more - // parallelism (default 1 = strict affinity). Server-side connections for one database then stay - // bounded at roughly N × per-node pool size. - x.DatabaseAffineAgentFanout = 2; }); ``` -These settings on `IntegrateWithWolverine` flow through to `StoreOptions.Events.UseDatabaseAffineAgentAssignment` -and `DatabaseAffineAgentFanout` (you can also set them there directly). Wolverine reads them off the store through -`IEventStore.GroupAgentAssignmentsByDatabase` and `IEventStore.MaxNodesPerDatabaseForAgents`, so the behavior is entirely -opt-in and store-driven — a store that does not set them keeps the default even distribution. The grouping key is the +This setting on `IntegrateWithWolverine` flows through to `StoreOptions.Events.UseDatabaseAffineAgentAssignment` +(you can also set it there directly). Wolverine reads it off the store through +`IEventStore.GroupAgentAssignmentsByDatabase`, so the behavior is entirely opt-in and store-driven — a store that +does not set it keeps the default even distribution. The grouping key is the `[event store type]/[event store name]/[database]` prefix of the agent `Uri` (see below), so all of a shard database's per-tenant agents share one group. diff --git a/src/Persistence/MartenTests/Distribution/integrate_with_wolverine_affine_settings.cs b/src/Persistence/MartenTests/Distribution/integrate_with_wolverine_affine_settings.cs index 43d535155..0d0763f90 100644 --- a/src/Persistence/MartenTests/Distribution/integrate_with_wolverine_affine_settings.cs +++ b/src/Persistence/MartenTests/Distribution/integrate_with_wolverine_affine_settings.cs @@ -11,12 +11,12 @@ namespace MartenTests.Distribution; /// -/// JasperFx/marten#4806: the database-affine agent-assignment settings can be configured on -/// IntegrateWithWolverine (next to UseWolverineManagedEventSubscriptionDistribution), and they must flow -/// through to the Marten event store — which is where Wolverine's distribution reads them via -/// IEventStore.GroupAgentAssignmentsByDatabase / MaxNodesPerDatabaseForAgents. This guards the bridge -/// (MartenOverrides resolving the MartenIntegration and applying the settings) against a DI-resolution -/// regression that would silently leave the app on the default even distribution. +/// JasperFx/marten#4806: the database-affine agent-assignment setting can be configured on +/// IntegrateWithWolverine (next to UseWolverineManagedEventSubscriptionDistribution), and it must flow +/// through to the Marten event store — which is where Wolverine's distribution reads it via +/// IEventStore.GroupAgentAssignmentsByDatabase. This guards the bridge (MartenOverrides resolving the +/// MartenIntegration and applying the setting) against a DI-resolution regression that would silently +/// leave the app on the default even distribution. /// public class integrate_with_wolverine_affine_settings { @@ -39,28 +39,25 @@ private static async Task StartHostAsync(Action config } [Fact] - public async Task affine_settings_flow_to_the_event_store() + public async Task affine_setting_flows_to_the_event_store() { using var host = await StartHostAsync(m => { m.UseWolverineManagedEventSubscriptionDistribution = true; m.UseDatabaseAffineAgentAssignment = true; - m.DatabaseAffineAgentFanout = 3; }, "affine_on"); var store = (DocumentStore)host.Services.GetRequiredService(); store.Options.Events.UseDatabaseAffineAgentAssignment.ShouldBeTrue(); - store.Options.Events.DatabaseAffineAgentFanout.ShouldBe(3); } [Fact] - public async Task defaults_are_unchanged_when_not_configured() + public async Task default_is_unchanged_when_not_configured() { using var host = await StartHostAsync( m => m.UseWolverineManagedEventSubscriptionDistribution = true, "affine_off"); var store = (DocumentStore)host.Services.GetRequiredService(); store.Options.Events.UseDatabaseAffineAgentAssignment.ShouldBeFalse(); - store.Options.Events.DatabaseAffineAgentFanout.ShouldBe(1); } } diff --git a/src/Persistence/Wolverine.Marten/MartenIntegration.cs b/src/Persistence/Wolverine.Marten/MartenIntegration.cs index 06831ef66..852cbd2fc 100644 --- a/src/Persistence/Wolverine.Marten/MartenIntegration.cs +++ b/src/Persistence/Wolverine.Marten/MartenIntegration.cs @@ -58,16 +58,6 @@ public class MartenIntegration : IWolverineExtension, IEventForwarding /// public bool UseDatabaseAffineAgentAssignment { get; set; } - /// - /// When is on, the maximum number of nodes a single shard - /// database's agents may fan out across (the "mix" between strict affinity and even spreading). 1 = strict - /// affinity (one node per database — fewest connections). A higher value lets a heavy database parallelize - /// across up to N least-loaded nodes, at the cost of that database being reachable from up to N nodes - /// (server-side connection ceiling per database ≈ N × per-node pool). Flows through to - /// StoreOptions.Events.DatabaseAffineAgentFanout. Default 1. See JasperFx/marten#4806. - /// - public int DatabaseAffineAgentFanout { get; set; } = 1; - public void Configure(WolverineOptions options) { // Duplicate incoming messages @@ -224,16 +214,15 @@ public void Configure(IServiceProvider services, StoreOptions options) } }); - // Bridge the Wolverine-managed distribution's database-affine agent-assignment settings — configured + // Bridge the Wolverine-managed distribution's database-affine agent-assignment setting — configured // on IntegrateWithWolverine — onto the Marten event store, because Wolverine's distribution is what - // consumes them (via IEventStore.GroupAgentAssignmentsByDatabase / MaxNodesPerDatabaseForAgents). - // Only the main store (StoreType == null): this is a sharded per-tenant-store concern, not ancillary. + // consumes it (via IEventStore.GroupAgentAssignmentsByDatabase). Only the main store + // (StoreType == null): this is a sharded per-tenant-store concern, not ancillary. // See JasperFx/marten#4806. if (StoreType == null && - services.GetService() is { UseDatabaseAffineAgentAssignment: true } integration) + services.GetService() is { UseDatabaseAffineAgentAssignment: true }) { options.Events.UseDatabaseAffineAgentAssignment = true; - options.Events.DatabaseAffineAgentFanout = integration.DatabaseAffineAgentFanout; } } } diff --git a/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs b/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs index 14e8d6339..335f0b311 100644 --- a/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs +++ b/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs @@ -64,38 +64,33 @@ public void single_node_takes_every_agent() } [Fact] - public void bounded_fan_out_spreads_a_heavy_database_across_up_to_k_nodes() + public void a_heavy_database_stays_on_one_node_and_light_groups_balance_around_it() { var grid = new AssignmentGrid(); grid.WithNode(1, Guid.NewGuid()); grid.WithNode(2, Guid.NewGuid()); - grid.WithNode(3, Guid.NewGuid()); - // One heavy database with 6 tenants; bound = 2 → its agents span exactly 2 nodes (not 1, not 3). - var agents = Enumerable.Range(1, 6).Select(t => Agent("dbHeavy", "t" + t)).ToArray(); - grid.WithAgents(agents); - - grid.DistributeByGroupAffinity("event-subscriptions", DatabaseKey, maxNodesPerGroup: 2); + // One heavy database (4 tenants) + two light ones (1 tenant each). Largest-first placement puts the + // heavy group on one node and both light groups on the other — totals stay as balanced as whole + // groups allow, and a database is never split across nodes. + var agents = new List(); + foreach (var tenant in new[] { "t1", "t2", "t3", "t4" }) + agents.Add(Agent("dbHeavy", tenant)); + agents.Add(Agent("dbLight1", "t1")); + agents.Add(Agent("dbLight2", "t1")); - var nodesUsed = agents.Select(a => grid.AgentFor(a).AssignedNode).Distinct().ToList(); - nodesUsed.Count.ShouldBe(2, "a heavy database must fan out across up to the bound (2) nodes"); - nodesUsed.ShouldAllBe(n => n != null); - } + grid.WithAgents(agents.ToArray()); - [Fact] - public void a_group_smaller_than_the_bound_uses_fewer_nodes() - { - var grid = new AssignmentGrid(); - grid.WithNode(1, Guid.NewGuid()); - grid.WithNode(2, Guid.NewGuid()); - grid.WithNode(3, Guid.NewGuid()); + grid.DistributeByGroupAffinity("event-subscriptions", DatabaseKey); - // A single-tenant database can only ever use one node, even with a higher bound. - grid.WithAgents(Agent("dbLight", "t1")); + grid.AllAgents.ShouldAllBe(a => a.AssignedNode != null); - grid.DistributeByGroupAffinity("event-subscriptions", DatabaseKey, maxNodesPerGroup: 3); + var heavyNode = grid.AgentFor(Agent("dbHeavy", "t1")).AssignedNode; + new[] { "t2", "t3", "t4" } + .Select(t => grid.AgentFor(Agent("dbHeavy", t)).AssignedNode) + .ShouldAllBe(n => n == heavyNode); - grid.AgentFor(Agent("dbLight", "t1")).AssignedNode.ShouldNotBeNull(); - grid.Nodes.Count(n => n.ForScheme("event-subscriptions").Any()).ShouldBe(1); + grid.AgentFor(Agent("dbLight1", "t1")).AssignedNode.ShouldNotBe(heavyNode); + grid.AgentFor(Agent("dbLight2", "t1")).AssignedNode.ShouldNotBe(heavyNode); } } diff --git a/src/Testing/CoreTests/Runtime/Agents/event_subscription_family_affine_assignment.cs b/src/Testing/CoreTests/Runtime/Agents/event_subscription_family_affine_assignment.cs index ad91e7a8d..2b52fbfef 100644 --- a/src/Testing/CoreTests/Runtime/Agents/event_subscription_family_affine_assignment.cs +++ b/src/Testing/CoreTests/Runtime/Agents/event_subscription_family_affine_assignment.cs @@ -14,10 +14,9 @@ namespace CoreTests.Runtime.Agents; /// /// JasperFx/marten#4806: when a sharded store opts into , /// must assign that store's per-(shard, tenant) -/// agents with database affinity (a database's agents kept on one node, bounded by -/// ) instead of the default even blue/green spread. These -/// are fast unit tests: agents are registered directly on an and the store flags are -/// stubbed, so no host or database is required. +/// agents with database affinity (a database's agents kept together on one node) instead of the default even +/// blue/green spread. These are fast unit tests: agents are registered directly on an +/// and the store flag is stubbed, so no host or database is required. /// public class event_subscription_family_affine_assignment { @@ -25,12 +24,11 @@ public class event_subscription_family_affine_assignment private static Uri Agent(string db, string tenant) => new($"event-subscriptions://marten/main/{db}/Proj:All:{tenant}"); - private static EventSubscriptionAgentFamily FamilyFor(bool affine, int maxNodesPerDatabase = 1) + private static EventSubscriptionAgentFamily FamilyFor(bool affine) { var store = Substitute.For(); store.Identity.Returns(new EventStoreIdentity("main", "marten")); store.GroupAgentAssignmentsByDatabase.Returns(affine); - store.MaxNodesPerDatabaseForAgents.Returns(maxNodesPerDatabase); return new EventSubscriptionAgentFamily(new[] { store }, Array.Empty>()); } @@ -76,27 +74,6 @@ public async Task spreads_a_databases_agents_evenly_when_the_store_does_not_opt_ nodes.Count.ShouldBe(2, "without affinity even distribution splits a two-agent database across both nodes"); } - [Fact] - public async Task fans_a_heavy_database_out_across_the_stores_max_nodes_bound() - { - // The family must pass the store's MaxNodesPerDatabaseForAgents through to the grid so a heavy database - // parallelizes across up to that many nodes — not one (strict affinity) and not all three (even spread). - var family = FamilyFor(affine: true, maxNodesPerDatabase: 2); - - var grid = new AssignmentGrid(); - grid.WithNode(1, Guid.NewGuid()); - grid.WithNode(2, Guid.NewGuid()); - grid.WithNode(3, Guid.NewGuid()); - var agents = Enumerable.Range(1, 6).Select(t => Agent("dbHeavy", "t" + t)).ToArray(); - grid.WithAgents(agents); - - await family.EvaluateAssignmentsAsync(grid); - - var nodesUsed = agents.Select(a => grid.AgentFor(a).AssignedNode).Distinct().ToList(); - nodesUsed.Count.ShouldBe(2, "the family must honor the store's fan-out bound of 2"); - nodesUsed.ShouldAllBe(n => n != null); - } - [Fact] public void database_key_groups_a_databases_agents_and_separates_databases() { @@ -120,35 +97,28 @@ public void database_key_falls_back_to_the_whole_uri_when_there_is_no_database_s } /// -/// surfaces the affine-assignment flags from its underlying -/// unchanged; the agent family reads them off the wrapper. See JasperFx/marten#4806. +/// surfaces the affine-assignment flag from its underlying +/// unchanged; the agent family reads it off the wrapper. See JasperFx/marten#4806. /// public class event_store_agents_affinity_passthrough { - private static EventStoreAgents AgentsFor(bool affine, int maxNodes) + private static EventStoreAgents AgentsFor(bool affine) { var store = Substitute.For(); store.Identity.Returns(new EventStoreIdentity("main", "marten")); store.GroupAgentAssignmentsByDatabase.Returns(affine); - store.MaxNodesPerDatabaseForAgents.Returns(maxNodes); return new EventStoreAgents(store, Array.Empty>()); } [Fact] - public void surfaces_group_by_database_and_max_nodes_from_the_store() + public void surfaces_group_by_database_from_the_store() { - var agents = AgentsFor(affine: true, maxNodes: 3); - - agents.GroupAgentAssignmentsByDatabase.ShouldBeTrue(); - agents.MaxNodesPerDatabaseForAgents.ShouldBe(3); + AgentsFor(affine: true).GroupAgentAssignmentsByDatabase.ShouldBeTrue(); } [Fact] - public void defaults_flow_through_for_a_non_affine_store() + public void default_flows_through_for_a_non_affine_store() { - var agents = AgentsFor(affine: false, maxNodes: 1); - - agents.GroupAgentAssignmentsByDatabase.ShouldBeFalse(); - agents.MaxNodesPerDatabaseForAgents.ShouldBe(1); + AgentsFor(affine: false).GroupAgentAssignmentsByDatabase.ShouldBeFalse(); } } diff --git a/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs b/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs index bbeb33580..963847e48 100644 --- a/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs +++ b/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs @@ -84,21 +84,20 @@ public void DistributeEvenly(string scheme) } /// - /// Distribute agents of a scheme across nodes with bounded group affinity: agents that share a - /// (e.g. a shard database) are kept together, but a group may fan out across - /// up to nodes. This is the "mix" between strict affinity - /// (maxNodesPerGroup == 1 — fewest connections, but a heavy group is serialized on one node's - /// pool) and even spreading: a higher bound lets a heavy group parallelize across several nodes while - /// still capping how many nodes reach that group's database, so its server-side connection ceiling stays - /// bounded at (bound × per-node pool). Intended for sharded event stores (JasperFx/marten#4806). + /// Distribute agents of a scheme across nodes with group affinity: all agents that share a + /// (e.g. a shard database) are assigned to the same node. Intended for + /// sharded event stores whose per-(shard, tenant) agents each connect to their shard database: an even + /// per-agent spread makes every node open pools to (nearly) every database (pools grow as + /// nodes×databases and exhaust a shared server's max_connections), while grouping keeps each node + /// connected only to the databases it owns, so pools scale with the number of databases + /// (JasperFx/marten#4806). /// - /// Groups are placed largest-first onto their k = min(bound, group size, node count) least-loaded - /// nodes; each group's agents then greedily fill those nodes least-loaded-first, so total agent count - /// stays balanced and the result is deterministic (a steady grid does not churn). A group with fewer - /// agents than the bound naturally uses fewer nodes. This prototype does not apply blue/green capability - /// matching; use when that is required. + /// Groups are placed largest-first onto the least-loaded node (deterministic tie-breaks), so total + /// agent count stays balanced and a steady grid does not churn. This does not apply blue/green + /// capability matching; use when that is + /// required. /// - public void DistributeByGroupAffinity(string scheme, Func groupKey, int maxNodesPerGroup = 1) + public void DistributeByGroupAffinity(string scheme, Func groupKey) { if (_nodes.Count == 0) { @@ -111,11 +110,6 @@ public void DistributeByGroupAffinity(string scheme, Func groupKey, return; } - if (maxNodesPerGroup < 1) - { - maxNodesPerGroup = 1; - } - var nodes = _nodes.OrderBy(x => x.IsLeader).ThenBy(x => x.AssignedId).ToList(); if (nodes.Count == 1) @@ -140,28 +134,20 @@ public void DistributeByGroupAffinity(string scheme, Func groupKey, foreach (var group in groups) { var members = group.ToList(); - var k = Math.Min(Math.Min(maxNodesPerGroup, members.Count), nodes.Count); - // The k least-loaded nodes host this group; a group smaller than the bound uses fewer nodes. - var chosen = load + // The least-loaded node hosts the whole group (tie-breaks: non-leader first, then node id). + var node = load .OrderBy(kv => kv.Value) .ThenBy(kv => kv.Key.IsLeader) .ThenBy(kv => kv.Key.AssignedId) - .Take(k) - .Select(kv => kv.Key) - .ToList(); + .First().Key; - // Greedily fill the group's agents onto the chosen nodes, least-loaded first, to balance work. foreach (var agent in members) { - var node = chosen - .OrderBy(n => load[n]) - .ThenBy(n => n.IsLeader) - .ThenBy(n => n.AssignedId) - .First(); node.Assign(agent); - load[node]++; } + + load[node] += members.Count; } } diff --git a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs index e98194a3a..cbb5d4104 100644 --- a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs +++ b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs @@ -38,12 +38,6 @@ public EventStoreAgents(IEventStore store, IObserver[] observers) /// public bool GroupAgentAssignmentsByDatabase => _store.GroupAgentAssignmentsByDatabase; - /// - /// Upper bound on how many nodes a single shard database's agents may fan out across under database-affine - /// assignment (the "mix"; 1 = strict affinity). Surfaces . - /// - public int MaxNodesPerDatabaseForAgents => _store.MaxNodesPerDatabaseForAgents; - public async ValueTask DisposeAsync() { foreach (var entry in _daemons.Enumerate()) diff --git a/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs b/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs index a0d035684..dd8279f8d 100644 --- a/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs +++ b/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs @@ -160,18 +160,13 @@ public ValueTask EvaluateAssignmentsAsync(AssignmentGrid assignments) // shard database's per-tenant agents together on one node so a node opens connection pools only to // the databases it owns (pools scale with databases, not nodes × databases, which otherwise // exhausts a shared server). Otherwise distribute evenly with blue/green semantics as before. - var affineStores = _stores.Enumerate() - .Select(e => e.Value) - .Where(s => s.GroupAgentAssignmentsByDatabase) - .ToList(); + var anyAffineStore = _stores.Enumerate() + .Any(e => e.Value.GroupAgentAssignmentsByDatabase); - if (affineStores.Count != 0) + if (anyAffineStore) { - // Bounded fan-out ("mix"): a shard database's agents may spread across up to this many nodes. - // One grid pass covers every store's agents (their per-store URI prefix keeps groups distinct), - // so use the largest bound any affine store asks for. - var maxNodesPerGroup = affineStores.Max(s => s.MaxNodesPerDatabaseForAgents); - assignments.DistributeByGroupAffinity(SchemeName, DatabaseKeyOf, maxNodesPerGroup); + // One grid pass covers every store's agents — their per-store URI prefix keeps groups distinct. + assignments.DistributeByGroupAffinity(SchemeName, DatabaseKeyOf); } else { From 6cb95eac994012b7afe9b4d4d1871cf42706132b Mon Sep 17 00:00:00 2001 From: Anne Erdtsieck Date: Mon, 6 Jul 2026 10:34:52 +0200 Subject: [PATCH 10/18] Address Copilot review: tenant-id normalization + the #3280 integration test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Per-tenant fan-out now dedupes tenant ids case-insensitively (matching TryResolveTenantDatabaseIdAsync) and skips blank ids, so a malformed usage descriptor cannot yield duplicate or invalid per-tenant agents. Unit-tested. - New integration test sharded_tenant_partitioned_distribution: the #3280 regression end-to-end — MultiTenantedWithShardedDatabases + UseTenantPartitionedEvents with two tenants CO-LOCATED on one shard database under managed distribution asserts one agent per tenant (":all/" URIs) and that both tenants' events project against their own high-water marks. Tenants are provisioned before host start; adding a database's FIRST tenants at runtime leaves the empty-phase store-global agent running alongside the new per-tenant agents until assignments reconcile — noted in the test as a known transition edge. Runs green locally against the companion jasperfx#482 + marten#4799 builds; goes green in CI once those merge (stated merge order). Claude-Session: https://claude.ai/code/session_01LXFw8u9F71jLhMTFNi9DyX --- ...sharded_tenant_partitioned_distribution.cs | 190 ++++++++++++++++++ ...nt_store_agents_per_tenant_distribution.cs | 17 ++ .../Runtime/Agents/EventStoreAgents.cs | 7 +- 3 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 src/Persistence/MartenTests/Distribution/sharded_tenant_partitioned_distribution.cs diff --git a/src/Persistence/MartenTests/Distribution/sharded_tenant_partitioned_distribution.cs b/src/Persistence/MartenTests/Distribution/sharded_tenant_partitioned_distribution.cs new file mode 100644 index 000000000..6bfdbf703 --- /dev/null +++ b/src/Persistence/MartenTests/Distribution/sharded_tenant_partitioned_distribution.cs @@ -0,0 +1,190 @@ +using IntegrationTests; +using JasperFx.CodeGeneration; +using JasperFx.Core; +using JasperFx.Events; +using JasperFx.Events.Projections; +using JasperFx.MultiTenancy; +using Marten; +using Marten.Events; +using MartenTests.Distribution.Support; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Npgsql; +using Shouldly; +using Weasel.Postgresql; +using Weasel.Postgresql.Migrations; +using Wolverine; +using Wolverine.Marten; +using Wolverine.Runtime.Agents; +using Wolverine.Tracking; +using Xunit; +using Xunit.Abstractions; + +namespace MartenTests.Distribution; + +// Phase 3 of the per-tenant-partitioned-events matrix — the #3280 regression itself: under +// MultiTenantedWithShardedDatabases + UseTenantPartitionedEvents, multiple tenants are CO-LOCATED in one +// shard database, each drawing its own event sequence. A single store-global agent per (shard, database) +// cannot track them (the lagging tenant's appends fall below the shared high-water mark and are skipped), +// so managed distribution must fan out one agent per (shard, tenant) — asserted here end-to-end: two +// ":all/" agents for the co-located database, and both tenants' events actually projected. +// +// NOTE: this requires the companion JasperFx.Events (jasperfx#482) + Marten (marten#4799) changes — it +// goes green in CI once those merge, per the PR's stated merge order. +public class sharded_tenant_partitioned_distribution(ITestOutputHelper output) : PostgresqlContext, IAsyncLifetime +{ + private const string TenantA = "tenant-a"; + private const string TenantB = "tenant-b"; + + private IHost theHost = null!; + private IDocumentStore theStore = null!; + + // Unique names per run: managed partitions/databases from one run otherwise break the next run's + // resource-setup DDL reconciliation. + private readonly string theSuffix = Guid.NewGuid().ToString("N")[..8]; + private string ShardDatabase => $"w3281_shard_{theSuffix}"; + private string MasterSchema => $"csp_sharded_{theSuffix}"; + + public async Task InitializeAsync() + { + // Provision the physical shard database (the master/pool lives in the default test database + // under its own schema). + await using (var conn = new NpgsqlConnection(Servers.PostgresConnectionString)) + { + await conn.OpenAsync(); + if (!await conn.DatabaseExists(ShardDatabase)) + { + await new DatabaseSpecification().BuildDatabase(conn, ShardDatabase); + } + } + + var shardConnectionString = new NpgsqlConnectionStringBuilder(Servers.PostgresConnectionString) + { + Database = ShardDatabase + }.ConnectionString; + + void ConfigureShardedStore(StoreOptions m) + { + m.DisableNpgsqlLogging = true; + m.MultiTenantedWithShardedDatabases(x => + { + x.ConnectionString = Servers.PostgresConnectionString; + x.SchemaName = MasterSchema; + x.PartitionSchemaName = MasterSchema + "_tenants"; + x.AddDatabase("shard-1", shardConnectionString); + }); + + m.Events.StreamIdentity = StreamIdentity.AsString; + m.Events.TenancyStyle = TenancyStyle.Conjoined; + m.Events.AppendMode = EventAppendMode.Quick; + m.Events.UseTenantPartitionedEvents = true; + m.Events.UseIdentityMapForAggregates = false; + + m.Schema.For().MultiTenanted(); + m.Projections.Snapshot(SnapshotLifecycle.Async); + } + + // Provision the two CO-LOCATED tenants BEFORE the Wolverine host starts, so the first agent + // enumeration already sees them and fans out per tenant. (Adding a database's FIRST tenants while + // the host runs leaves the store-global agent from the empty-database phase running alongside the + // new per-tenant agents until assignments reconcile — a transition edge outside this test's scope.) + await using (var provisioning = DocumentStore.For(ConfigureShardedStore)) + { + // Apply the event schema (parent partitioned tables) to the shard database first, so the + // per-tenant LIST partitions + sequences created by the explicit assignment have a parent. + await provisioning.Storage.ApplyAllConfiguredChangesToDatabaseAsync(); + await provisioning.Advanced.AddTenantToShardAsync(TenantA, "shard-1", default); + await provisioning.Advanced.AddTenantToShardAsync(TenantB, "shard-1", default); + } + + theHost = await Host.CreateDefaultBuilder() + .UseWolverine(opts => + { + opts.Durability.HealthCheckPollingTime = 1.Seconds(); + opts.Durability.CheckAssignmentPeriod = 1.Seconds(); + + opts.Services.AddMarten(ConfigureShardedStore) + .IntegrateWithWolverine(m => + { + m.UseWolverineManagedEventSubscriptionDistribution = true; + // Sharded tenancy has no single "main" database, so Wolverine's message store needs + // an explicit master (mirrors a real deployment pointing this at a references DB). + m.MainDatabaseConnectionString = Servers.PostgresConnectionString; + }); + + opts.Services.AddSingleton(new OutputLoggerProvider(output)); + opts.Discovery.DisableConventionalDiscovery(); + opts.CodeGeneration.TypeLoadMode = TypeLoadMode.Auto; + }).StartAsync(); + + theStore = theHost.Services.GetRequiredService(); + } + + public async Task DisposeAsync() + { + theHost.GetRuntime().Agents.DisableHealthChecks(); + await theHost.StopAsync(); + theHost.Dispose(); + } + + [Fact] + public async Task co_located_tenants_get_their_own_agents_and_both_project() + { + await theHost.WaitUntilAssumesLeadershipAsync(10.Seconds()); + + // One async projection over one shard database with two co-located tenants => one agent PER TENANT. + await theHost.WaitUntilAssignmentsChangeTo(w => + { + w.AgentScheme = EventSubscriptionAgentFamily.SchemeName; + w.ExpectRunningAgents(theHost, 2); + }, 60.Seconds()); + + var uris = await GetAgentUrisAsync(theHost); + uris.Length.ShouldBe(2); + uris.ShouldContain(u => u.TrimEnd('/').EndsWith("/" + TenantA, StringComparison.OrdinalIgnoreCase)); + uris.ShouldContain(u => u.TrimEnd('/').EndsWith("/" + TenantB, StringComparison.OrdinalIgnoreCase)); + + // Both tenants' events are projected — each tenant advances against its OWN high-water mark, so the + // second tenant's appends are not skipped below a shared mark (the #3280 failure mode). + var id = "pc-" + Guid.NewGuid().ToString("N"); + await AppendAsync(TenantA, id, 5); + await AppendAsync(TenantB, id, 11); // same stream id, other tenant partition + + (await WaitForProjectionAsync(TenantA, id, 60.Seconds()))!.Total.ShouldBe(5); + (await WaitForProjectionAsync(TenantB, id, 60.Seconds()))!.Total.ShouldBe(11); + } + + private async Task AppendAsync(string tenant, string id, int amount) + { + await using var session = theStore.LightweightSession(tenant); + session.Events.StartStream(id, new CounterChanged(amount)); + await session.SaveChangesAsync(); + } + + private async Task WaitForProjectionAsync(string tenant, string id, TimeSpan timeout) + { + var deadline = DateTimeOffset.UtcNow + timeout; + while (DateTimeOffset.UtcNow < deadline) + { + await using var session = theStore.QuerySession(tenant); + var doc = await session.LoadAsync(id); + if (doc != null) + { + return doc; + } + + await Task.Delay(250.Milliseconds()); + } + + return null; + } + + private static async Task GetAgentUrisAsync(IHost host) + { + var family = host.Services.GetServices() + .OfType().Single(); + var agents = await family.AllKnownAgentsAsync(); + return [.. agents.Select(x => x.AbsoluteUri)]; + } +} diff --git a/src/Testing/CoreTests/Runtime/Agents/event_store_agents_per_tenant_distribution.cs b/src/Testing/CoreTests/Runtime/Agents/event_store_agents_per_tenant_distribution.cs index 6f5d240bb..351e451d6 100644 --- a/src/Testing/CoreTests/Runtime/Agents/event_store_agents_per_tenant_distribution.cs +++ b/src/Testing/CoreTests/Runtime/Agents/event_store_agents_per_tenant_distribution.cs @@ -91,4 +91,21 @@ public async Task keeps_a_single_agent_when_the_database_has_no_tenants_yet() uris.Count.ShouldBe(1, "a per-tenant store with no tenants provisioned yet keeps one store-global agent"); } + + [Fact] + public async Task dedupes_tenants_case_insensitively_and_skips_blank_ids() + { + // Tenant ids are matched case-insensitively elsewhere (TryResolveTenantDatabaseIdAsync), so the + // fan-out must not produce two agents for the same logical tenant in different casing, nor an + // invalid agent for a blank id that slipped into the usage descriptor. + var agents = AgentsFor( + UsageWithOneAsyncShardOnADatabaseWith("Clinic-A", "clinic-a", "clinic-b", "", " "), + distributesPerTenant: true); + + var uris = await agents.SupportedAgentsAsync(CancellationToken.None); + + uris.Count.ShouldBe(2, "one agent per logical tenant: clinic-a (deduped across casing) and clinic-b"); + uris.ShouldContain(u => u.AbsolutePath.TrimEnd('/').EndsWith("/clinic-a", StringComparison.OrdinalIgnoreCase)); + uris.ShouldContain(u => u.AbsolutePath.TrimEnd('/').EndsWith("/clinic-b", StringComparison.OrdinalIgnoreCase)); + } } diff --git a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs index cbb5d4104..5adacf91e 100644 --- a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs +++ b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs @@ -108,7 +108,12 @@ void addAgentsForShard(DatabaseDescriptor database, DatabaseId id, ShardName sha { if (_store.DistributesAgentsPerTenant && database.TenantIds.Count > 0) { - foreach (var tenantId in database.TenantIds.Distinct()) + // Tenant ids are matched case-insensitively elsewhere in this class (see + // TryResolveTenantDatabaseIdAsync), so dedupe the fan-out the same way, and skip blank ids — + // a malformed usage descriptor must not yield duplicate or invalid per-tenant agent URIs. + foreach (var tenantId in database.TenantIds + .Where(t => !string.IsNullOrWhiteSpace(t)) + .Distinct(StringComparer.OrdinalIgnoreCase)) { var tenantShard = shardName.ForTenant(tenantId); _shardNames = _shardNames.AddOrUpdate(tenantShard.RelativeUrl, tenantShard); From e27ddcd37b588c12401c103c5f1afe62ce19cc83 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 6 Jul 2026 12:29:53 -0500 Subject: [PATCH 11/18] =?UTF-8?q?Remove=20the=20opt-in=20database-affine?= =?UTF-8?q?=20surface=20=E2=80=94=20the=20revised=20design=20derives=20it?= =?UTF-8?q?=20from=20store=20cardinality?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops MartenIntegration.UseDatabaseAffineAgentAssignment, the MartenOverrides bridge onto StoreOptions.Events, the EventStoreAgents.GroupAgentAssignmentsByDatabase passthrough (the revised JasperFx.Events contract no longer carries that member), the opt-in branch in EventSubscriptionAgentFamily.EvaluateAssignmentsAsync, the flag-specific test suites, and the opt-in docs section. Database-affine assignment comes back in the next commits keyed off IEventStore.DatabaseCardinality instead of an explicit flag. Epic: JasperFx/jasperfx#486. Co-Authored-By: Claude Fable 5 --- docs/guide/durability/marten/distribution.md | 39 ------ ...ntegrate_with_wolverine_affine_settings.cs | 63 --------- .../Wolverine.Marten/MartenIntegration.cs | 23 ---- ...t_subscription_family_affine_assignment.cs | 124 ------------------ .../Runtime/Agents/EventStoreAgents.cs | 8 -- .../Agents/EventSubscriptionAgentFamily.cs | 17 +-- 6 files changed, 1 insertion(+), 273 deletions(-) delete mode 100644 src/Persistence/MartenTests/Distribution/integrate_with_wolverine_affine_settings.cs delete mode 100644 src/Testing/CoreTests/Runtime/Agents/event_subscription_family_affine_assignment.cs diff --git a/docs/guide/durability/marten/distribution.md b/docs/guide/durability/marten/distribution.md index fd6477a2c..6bc1381a5 100644 --- a/docs/guide/durability/marten/distribution.md +++ b/docs/guide/durability/marten/distribution.md @@ -59,45 +59,6 @@ Some other facts about this integration: new versions of a projection or subscription on some application nodes in order to do try ["blue/green deployment."](https://en.wikipedia.org/wiki/Blue%E2%80%93green_deployment) * This capability does depend on Wolverine's built-in [leadership election](https://en.wikipedia.org/wiki/Leader_election) -- which fortunately got a _lot_ better in Wolverine 3.0 -## Database-Affine Distribution for Sharded Stores - -By default Wolverine spreads subscription and projection agents *evenly* across the cluster (with blue/green -capability matching). That is the right choice for a store-global event store or database-per-tenant multi-tenancy, -where there is one agent per database anyway. - -It is *not* the right choice for a Marten store that combines [sharded multi-tenancy](https://martendb.io/configuration/multitenancy.html#sharded-multi-tenancy-with-database-pooling) -with [per-tenant event partitioning](https://martendb.io/events/multitenancy.html#per-tenant-event-partitioning). There, -many tenants are co-located in one shard database and each draws its own event sequence, so Wolverine fans agents out -one-per-`(shard, tenant)` rather than one-per-database. With hundreds of tenants scattered across many shard databases, -an even per-agent spread makes *every* node open a connection pool to *nearly every* shard database — so the pool count -grows as `nodes × databases` and quickly exhausts a shared server's `max_connections`. - -To bound that, opt into **database-affine assignment** on the Marten side. Wolverine then keeps every agent for a given -shard database together on a single node, so a node only opens pools to the databases it actually owns and the pool -count scales with the number of shard databases, not `nodes × databases`: - -```cs -services.AddMarten(opts => -{ - // ... sharded tenancy + per-tenant event partitioning ... - opts.Events.UseTenantPartitionedEvents = true; -}) -.IntegrateWithWolverine(x => -{ - x.UseWolverineManagedEventSubscriptionDistribution = true; - - // Keep a shard database's per-tenant agents together on one node. - x.UseDatabaseAffineAgentAssignment = true; -}); -``` - -This setting on `IntegrateWithWolverine` flows through to `StoreOptions.Events.UseDatabaseAffineAgentAssignment` -(you can also set it there directly). Wolverine reads it off the store through -`IEventStore.GroupAgentAssignmentsByDatabase`, so the behavior is entirely opt-in and store-driven — a store that -does not set it keeps the default even distribution. The grouping key is the -`[event store type]/[event store name]/[database]` prefix of the agent `Uri` (see below), so all of a shard database's -per-tenant agents share one group. - ## Uri Structure The `Uri` structure for event subscriptions or projections is: diff --git a/src/Persistence/MartenTests/Distribution/integrate_with_wolverine_affine_settings.cs b/src/Persistence/MartenTests/Distribution/integrate_with_wolverine_affine_settings.cs deleted file mode 100644 index 0d0763f90..000000000 --- a/src/Persistence/MartenTests/Distribution/integrate_with_wolverine_affine_settings.cs +++ /dev/null @@ -1,63 +0,0 @@ -using IntegrationTests; -using JasperFx.CodeGeneration; -using Marten; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Shouldly; -using Wolverine; -using Wolverine.Marten; -using Xunit; - -namespace MartenTests.Distribution; - -/// -/// JasperFx/marten#4806: the database-affine agent-assignment setting can be configured on -/// IntegrateWithWolverine (next to UseWolverineManagedEventSubscriptionDistribution), and it must flow -/// through to the Marten event store — which is where Wolverine's distribution reads it via -/// IEventStore.GroupAgentAssignmentsByDatabase. This guards the bridge (MartenOverrides resolving the -/// MartenIntegration and applying the setting) against a DI-resolution regression that would silently -/// leave the app on the default even distribution. -/// -public class integrate_with_wolverine_affine_settings -{ - private static async Task StartHostAsync(Action configure, string schema) - { - return await Host.CreateDefaultBuilder() - .UseWolverine(opts => - { - opts.Services.AddMarten(m => - { - m.DisableNpgsqlLogging = true; - m.Connection(Servers.PostgresConnectionString); - m.DatabaseSchemaName = schema; - }) - .IntegrateWithWolverine(configure); - - opts.Discovery.DisableConventionalDiscovery(); - opts.CodeGeneration.TypeLoadMode = TypeLoadMode.Auto; - }).StartAsync(); - } - - [Fact] - public async Task affine_setting_flows_to_the_event_store() - { - using var host = await StartHostAsync(m => - { - m.UseWolverineManagedEventSubscriptionDistribution = true; - m.UseDatabaseAffineAgentAssignment = true; - }, "affine_on"); - - var store = (DocumentStore)host.Services.GetRequiredService(); - store.Options.Events.UseDatabaseAffineAgentAssignment.ShouldBeTrue(); - } - - [Fact] - public async Task default_is_unchanged_when_not_configured() - { - using var host = await StartHostAsync( - m => m.UseWolverineManagedEventSubscriptionDistribution = true, "affine_off"); - - var store = (DocumentStore)host.Services.GetRequiredService(); - store.Options.Events.UseDatabaseAffineAgentAssignment.ShouldBeFalse(); - } -} diff --git a/src/Persistence/Wolverine.Marten/MartenIntegration.cs b/src/Persistence/Wolverine.Marten/MartenIntegration.cs index 852cbd2fc..67c6ee106 100644 --- a/src/Persistence/Wolverine.Marten/MartenIntegration.cs +++ b/src/Persistence/Wolverine.Marten/MartenIntegration.cs @@ -46,18 +46,6 @@ public class MartenIntegration : IWolverineExtension, IEventForwarding /// public bool UseWolverineManagedEventSubscriptionDistribution { get; set; } - /// - /// Opt-in (default false): when is on and - /// the Marten store uses sharded databases with per-tenant event partitioning, assign each shard - /// database's per-(shard, tenant) agents with database affinity — all of a database's agents run - /// on one node — instead of spreading them evenly by count. This bounds each node to the shard databases - /// it owns, so connection pools scale with the number of shard databases rather than nodes × databases - /// (which otherwise exhausts a shared server's max_connections). Flows through to - /// StoreOptions.Events.UseDatabaseAffineAgentAssignment; only takes effect for a sharded, - /// per-tenant-partitioned store. See JasperFx/marten#4806. - /// - public bool UseDatabaseAffineAgentAssignment { get; set; } - public void Configure(WolverineOptions options) { // Duplicate incoming messages @@ -213,17 +201,6 @@ public void Configure(IServiceProvider services, StoreOptions options) nameof(Saga.Version), typeof(int))!; } }); - - // Bridge the Wolverine-managed distribution's database-affine agent-assignment setting — configured - // on IntegrateWithWolverine — onto the Marten event store, because Wolverine's distribution is what - // consumes it (via IEventStore.GroupAgentAssignmentsByDatabase). Only the main store - // (StoreType == null): this is a sharded per-tenant-store concern, not ancillary. - // See JasperFx/marten#4806. - if (StoreType == null && - services.GetService() is { UseDatabaseAffineAgentAssignment: true }) - { - options.Events.UseDatabaseAffineAgentAssignment = true; - } } } diff --git a/src/Testing/CoreTests/Runtime/Agents/event_subscription_family_affine_assignment.cs b/src/Testing/CoreTests/Runtime/Agents/event_subscription_family_affine_assignment.cs deleted file mode 100644 index 2b52fbfef..000000000 --- a/src/Testing/CoreTests/Runtime/Agents/event_subscription_family_affine_assignment.cs +++ /dev/null @@ -1,124 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using JasperFx.Events; -using NSubstitute; -using Shouldly; -using Wolverine.Runtime.Agents; -using Xunit; - -namespace CoreTests.Runtime.Agents; - -/// -/// JasperFx/marten#4806: when a sharded store opts into , -/// must assign that store's per-(shard, tenant) -/// agents with database affinity (a database's agents kept together on one node) instead of the default even -/// blue/green spread. These are fast unit tests: agents are registered directly on an -/// and the store flag is stubbed, so no host or database is required. -/// -public class event_subscription_family_affine_assignment -{ - // Agent URI grammar is event-subscriptions://{type}/{name}/{databaseId}/{shard...} (see EventStoreAgents.UriFor). - private static Uri Agent(string db, string tenant) => - new($"event-subscriptions://marten/main/{db}/Proj:All:{tenant}"); - - private static EventSubscriptionAgentFamily FamilyFor(bool affine) - { - var store = Substitute.For(); - store.Identity.Returns(new EventStoreIdentity("main", "marten")); - store.GroupAgentAssignmentsByDatabase.Returns(affine); - return new EventSubscriptionAgentFamily(new[] { store }, Array.Empty>()); - } - - [Fact] - public async Task keeps_a_databases_agents_together_when_the_store_opts_into_affinity() - { - var family = FamilyFor(affine: true); - - var grid = new AssignmentGrid(); - grid.WithNode(1, Guid.NewGuid()); - grid.WithNode(2, Guid.NewGuid()); - // One shard database with two tenants: even distribution would split it across the two nodes. - grid.WithAgents(Agent("dbShared", "t1"), Agent("dbShared", "t2")); - - await family.EvaluateAssignmentsAsync(grid); - - var nodes = new[] { "t1", "t2" } - .Select(t => grid.AgentFor(Agent("dbShared", t)).AssignedNode) - .Distinct() - .ToList(); - - nodes.Count.ShouldBe(1, "an affine store must keep a shard database's agents on a single node"); - nodes[0].ShouldNotBeNull(); - } - - [Fact] - public async Task spreads_a_databases_agents_evenly_when_the_store_does_not_opt_in() - { - var family = FamilyFor(affine: false); - - var grid = new AssignmentGrid(); - grid.WithNode(1, Guid.NewGuid()); - grid.WithNode(2, Guid.NewGuid()); - grid.WithAgents(Agent("dbShared", "t1"), Agent("dbShared", "t2")); - - await family.EvaluateAssignmentsAsync(grid); - - var nodes = new[] { "t1", "t2" } - .Select(t => grid.AgentFor(Agent("dbShared", t)).AssignedNode) - .Distinct() - .ToList(); - - nodes.Count.ShouldBe(2, "without affinity even distribution splits a two-agent database across both nodes"); - } - - [Fact] - public void database_key_groups_a_databases_agents_and_separates_databases() - { - var t1 = EventSubscriptionAgentFamily.DatabaseKeyOf(Agent("claims1", "t1")); - var t2 = EventSubscriptionAgentFamily.DatabaseKeyOf(Agent("claims1", "t2")); - var other = EventSubscriptionAgentFamily.DatabaseKeyOf(Agent("claims2", "t1")); - - t1.ShouldBe("marten/main/claims1"); - t2.ShouldBe(t1, "two tenants on the same shard database must share a group key"); - other.ShouldNotBe(t1, "a different shard database must be a different group"); - } - - [Fact] - public void database_key_falls_back_to_the_whole_uri_when_there_is_no_database_segment() - { - // Defensive: a URI without the {databaseId} segment (fewer than 3 path segments) can't be grouped by - // database, so each such agent is its own group rather than silently collapsing together. - var uri = new Uri("event-subscriptions://marten/main"); - EventSubscriptionAgentFamily.DatabaseKeyOf(uri).ShouldBe(uri.AbsoluteUri); - } -} - -/// -/// surfaces the affine-assignment flag from its underlying -/// unchanged; the agent family reads it off the wrapper. See JasperFx/marten#4806. -/// -public class event_store_agents_affinity_passthrough -{ - private static EventStoreAgents AgentsFor(bool affine) - { - var store = Substitute.For(); - store.Identity.Returns(new EventStoreIdentity("main", "marten")); - store.GroupAgentAssignmentsByDatabase.Returns(affine); - return new EventStoreAgents(store, Array.Empty>()); - } - - [Fact] - public void surfaces_group_by_database_from_the_store() - { - AgentsFor(affine: true).GroupAgentAssignmentsByDatabase.ShouldBeTrue(); - } - - [Fact] - public void default_flows_through_for_a_non_affine_store() - { - AgentsFor(affine: false).GroupAgentAssignmentsByDatabase.ShouldBeFalse(); - } -} diff --git a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs index 5adacf91e..61ac53e9e 100644 --- a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs +++ b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs @@ -30,14 +30,6 @@ public EventStoreAgents(IEventStore store, IObserver[] observers) public EventStoreIdentity Identity { get; } - /// - /// When true (opt-in, sharded stores only), this store's per-(shard, tenant) agents should be assigned - /// with database affinity — all agents for a shard database on one node — so a node opens pools only to - /// the databases it owns. Surfaces . See - /// JasperFx/marten#4806. - /// - public bool GroupAgentAssignmentsByDatabase => _store.GroupAgentAssignmentsByDatabase; - public async ValueTask DisposeAsync() { foreach (var entry in _daemons.Enumerate()) diff --git a/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs b/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs index dd8279f8d..fb11fc4fc 100644 --- a/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs +++ b/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs @@ -156,22 +156,7 @@ public async ValueTask> SupportedAgentsAsync() public ValueTask EvaluateAssignmentsAsync(AssignmentGrid assignments) { - // JasperFx/marten#4806 (opt-in): when a sharded store asks for database-affine assignment, keep a - // shard database's per-tenant agents together on one node so a node opens connection pools only to - // the databases it owns (pools scale with databases, not nodes × databases, which otherwise - // exhausts a shared server). Otherwise distribute evenly with blue/green semantics as before. - var anyAffineStore = _stores.Enumerate() - .Any(e => e.Value.GroupAgentAssignmentsByDatabase); - - if (anyAffineStore) - { - // One grid pass covers every store's agents — their per-store URI prefix keeps groups distinct. - assignments.DistributeByGroupAffinity(SchemeName, DatabaseKeyOf); - } - else - { - assignments.DistributeEvenlyWithBlueGreenSemantics(SchemeName); - } + assignments.DistributeEvenlyWithBlueGreenSemantics(SchemeName); return new ValueTask(); } From eaa93082d8f5bbf7cfd2a042b4281439ac1caadf Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 6 Jul 2026 12:34:23 -0500 Subject: [PATCH 12/18] Database-affine assignment keyed off IEventStore.DatabaseCardinality, per store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EventSubscriptionAgentFamily.EvaluateAssignmentsAsync now partitions the scheme's agents per owning store (StoreKeyOf reproduces the _stores key from the agent URI's type/name prefix): every store whose DatabaseCardinality is StaticMultiple or DynamicMultiple gets its agents distributed with group affinity by DatabaseKeyOf, while all remaining agents keep the even blue/green distribution — no opt-in flag. AssignmentGrid grows Func-filtered overloads of DistributeEvenly, DistributeEvenlyWithBlueGreenSemantics, DistributeByGroupAffinity, AllNodesHaveSameCapabilities, MatchAgentsToCapableNodesFor and AvailableAgentsForScheme; the existing signatures delegate to them with an all-pass filter. Filtered passes count per-node load only over their own agent subset so independent passes of one scheme never detach each other's agents. Docs updated to describe the automatic behavior. Epic: JasperFx/jasperfx#486. Co-Authored-By: Claude Fable 5 --- docs/guide/durability/marten/distribution.md | 23 +++++ .../Agents/AssignmentGrid.Distribution.cs | 88 +++++++++++++++---- .../Runtime/Agents/AssignmentGrid.cs | 22 ++++- .../Agents/DynamicListenerAgentFamily.cs | 2 +- .../Runtime/Agents/EventStoreAgents.cs | 10 +++ .../Agents/EventSubscriptionAgentFamily.cs | 36 +++++++- 6 files changed, 159 insertions(+), 22 deletions(-) diff --git a/docs/guide/durability/marten/distribution.md b/docs/guide/durability/marten/distribution.md index 6bc1381a5..74b801840 100644 --- a/docs/guide/durability/marten/distribution.md +++ b/docs/guide/durability/marten/distribution.md @@ -59,6 +59,29 @@ Some other facts about this integration: new versions of a projection or subscription on some application nodes in order to do try ["blue/green deployment."](https://en.wikipedia.org/wiki/Blue%E2%80%93green_deployment) * This capability does depend on Wolverine's built-in [leadership election](https://en.wikipedia.org/wiki/Leader_election) -- which fortunately got a _lot_ better in Wolverine 3.0 +## Database-Affine Distribution for Multi-Database Stores + +By default Wolverine spreads subscription and projection agents *evenly* across the cluster (with blue/green +capability matching). That is the right choice for a single-database event store. + +It is *not* the right choice for a store backed by many databases. The clearest case is a Marten store that combines +[sharded multi-tenancy](https://martendb.io/configuration/multitenancy.html#sharded-multi-tenancy-with-database-pooling) +with [per-tenant event partitioning](https://martendb.io/events/multitenancy.html#per-tenant-event-partitioning). There, +many tenants are co-located in one shard database and each draws its own event sequence, so Wolverine fans agents out +one-per-`(shard, tenant)` rather than one-per-database. With hundreds of tenants scattered across many shard databases, +an even per-agent spread makes *every* node open a connection pool to *nearly every* shard database — so the pool count +grows as `nodes × databases` and quickly exhausts a shared server's `max_connections`. + +So Wolverine keys the distribution off the store itself, with no configuration needed: when a store reports that it is +backed by multiple databases (its `IEventStore.DatabaseCardinality` is static- or dynamic-multiple — sharded tenancy or +database-per-tenant), that store's agents are assigned with **database affinity** — every agent for a given database is +kept together on a single node, so a node only opens pools to the databases it actually owns and the pool count scales +with the number of databases, not `nodes × databases`. The grouping key is the +`[event store type]/[event store name]/[database]` prefix of the agent `Uri` (see below), so all of a database's +agents share one group. Whole groups are still spread across the cluster largest-first, so total agent counts stay +balanced. A single-database store in the same application keeps the default even distribution — each store is +distributed in its own pass. + ## Uri Structure The `Uri` structure for event subscriptions or projections is: diff --git a/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs b/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs index 963847e48..c6f2cd20a 100644 --- a/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs +++ b/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs @@ -11,6 +11,18 @@ public partial class AssignmentGrid /// /// public void DistributeEvenly(string scheme) + { + DistributeEvenly(scheme, _ => true); + } + + /// + /// Attempts to redistribute the agents of a given agent type that match + /// evenly across the known, executing nodes with minimal disruption. Agents of the scheme outside the + /// filter are left completely untouched, so one scheme can be distributed in several independent + /// passes (e.g. per event store). + /// + /// + public void DistributeEvenly(string scheme, Func filter) { if (_nodes.Count == 0) { @@ -18,7 +30,7 @@ public void DistributeEvenly(string scheme) } // Need to weed out agents that aren't "paused" - var agents = AvailableAgentsForScheme(scheme); + var agents = AvailableAgentsForScheme(scheme, filter); if (agents.Count == 0) { return; @@ -35,6 +47,11 @@ public void DistributeEvenly(string scheme) return; } + // Per-node counts must only consider the agents in this pass — otherwise a filtered pass + // would detach or count agents that belong to a different pass of the same scheme. + var agentSet = agents.ToHashSet(); + int countOn(Node node) => node.Agents.Count(agentSet.Contains); + var spread = (double)agents.Count / _nodes.Count; var minimum = (int)Math.Floor(spread); var maximum = (int)Math.Ceiling(spread); // this is helpful to reduce the number of assignments @@ -42,7 +59,7 @@ public void DistributeEvenly(string scheme) // First, pair down number of running agents if necessary. Might have to steal some later foreach (var node in _nodes) { - var extras = node.ForScheme(scheme).Skip(maximum).ToArray(); + var extras = node.Agents.Where(agentSet.Contains).Skip(maximum).ToArray(); foreach (var agent in extras) { agent.Detach(); @@ -59,7 +76,7 @@ public void DistributeEvenly(string scheme) break; } - var count = node.ForScheme(scheme).Count(); + var count = countOn(node); for (var i = 0; i < minimum - count; i++) { @@ -78,7 +95,7 @@ public void DistributeEvenly(string scheme) { var agent = missing.Dequeue(); - var node = _nodes.FirstOrDefault(x => !x.IsLeader && x.ForScheme(scheme).Count() < maximum) ?? _nodes.FirstOrDefault(x => !x.IsLeader) ?? _nodes.First(); + var node = _nodes.FirstOrDefault(x => !x.IsLeader && countOn(x) < maximum) ?? _nodes.FirstOrDefault(x => !x.IsLeader) ?? _nodes.First(); node.Assign(agent); } } @@ -93,18 +110,26 @@ public void DistributeEvenly(string scheme) /// (JasperFx/marten#4806). /// /// Groups are placed largest-first onto the least-loaded node (deterministic tie-breaks), so total - /// agent count stays balanced and a steady grid does not churn. This does not apply blue/green - /// capability matching; use when that is - /// required. + /// agent count stays balanced and a steady grid does not churn. /// public void DistributeByGroupAffinity(string scheme, Func groupKey) + { + DistributeByGroupAffinity(scheme, groupKey, _ => true); + } + + /// + /// Same as , restricted to the + /// agents of the scheme matching . Agents outside the filter are left + /// completely untouched so one scheme can be distributed in several independent passes. + /// + public void DistributeByGroupAffinity(string scheme, Func groupKey, Func filter) { if (_nodes.Count == 0) { throw new InvalidOperationException("There are no active nodes"); } - var agents = AvailableAgentsForScheme(scheme); + var agents = AvailableAgentsForScheme(scheme, filter); if (agents.Count == 0) { return; @@ -153,11 +178,21 @@ public void DistributeByGroupAffinity(string scheme, Func groupKey) public bool AllNodesHaveSameCapabilities(string scheme) { - var gold = _nodes[0].OrderedCapabilitiesForScheme(scheme); + return AllNodesHaveSameCapabilities(scheme, _ => true); + } + + /// + /// Same as , only comparing the capabilities of the + /// scheme matching — so one store's homogeneous capabilities aren't judged + /// "different" because another store of the same scheme is mid blue/green rollout. + /// + public bool AllNodesHaveSameCapabilities(string scheme, Func filter) + { + var gold = _nodes[0].OrderedCapabilitiesForScheme(scheme).Where(filter); foreach (var node in _nodes.Skip(1)) { - var matching = node.OrderedCapabilitiesForScheme(scheme); + var matching = node.OrderedCapabilitiesForScheme(scheme).Where(filter); if (!gold.SequenceEqual(matching)) { @@ -167,7 +202,7 @@ public bool AllNodesHaveSameCapabilities(string scheme) return true; } - + /// /// Attempts to redistribute agents for a given agent type evenly /// across the known, executing nodes with minimal disruption. This version assumes @@ -176,6 +211,17 @@ public bool AllNodesHaveSameCapabilities(string scheme) /// /// public void DistributeEvenlyWithBlueGreenSemantics(string scheme) + { + DistributeEvenlyWithBlueGreenSemantics(scheme, _ => true); + } + + /// + /// Same as , restricted to the agents of + /// the scheme matching . Agents outside the filter are left completely + /// untouched so one scheme can be distributed in several independent passes. + /// + /// + public void DistributeEvenlyWithBlueGreenSemantics(string scheme, Func filter) { var nodes = _nodes; if (nodes.Count == 0) @@ -183,13 +229,13 @@ public void DistributeEvenlyWithBlueGreenSemantics(string scheme) throw new InvalidOperationException("There are no active nodes"); } - if (AllNodesHaveSameCapabilities(scheme)) + if (AllNodesHaveSameCapabilities(scheme, filter)) { - DistributeEvenly(scheme); + DistributeEvenly(scheme, filter); return; } - var agents = MatchAgentsToCapableNodesFor(scheme); + var agents = MatchAgentsToCapableNodesFor(scheme, filter); if (agents.Count == 0) { @@ -204,6 +250,10 @@ public void DistributeEvenlyWithBlueGreenSemantics(string scheme) return; } + // Per-node counts must only consider the agents in this pass; see DistributeEvenly above. + var agentSet = agents.ToHashSet(); + int countOn(Node node) => node.Agents.Count(agentSet.Contains); + var spread = (double)agents.Count / nodes.Count; var minimum = (int)Math.Floor(spread); var maximum = (int)Math.Ceiling(spread); // this is helpful to reduce the number of assignments @@ -217,20 +267,20 @@ public void DistributeEvenlyWithBlueGreenSemantics(string scheme) agent.Detach(); } } - - // In the missing, we're going to put the agents up top that can be supported in fewer places + + // In the missing, we're going to put the agents up top that can be supported in fewer places var missing = agents.Where(x => x.AssignedNode == null).OrderBy(x => x.CandidateNodes.Count).ToList(); foreach (var agent in missing) { // First try to find a node that has less than the minimum number of nodes var candidate = agent .CandidateNodes - .FirstOrDefault(x => x.ForScheme(scheme).Count() < minimum) + .FirstOrDefault(x => countOn(x) < minimum) // Or fall back to the least loaded down node - ?? agent.CandidateNodes.MinBy(x => x.ForScheme(scheme).Count()); + ?? agent.CandidateNodes.MinBy(countOn); candidate?.Assign(agent); } } -} \ No newline at end of file +} diff --git a/src/Wolverine/Runtime/Agents/AssignmentGrid.cs b/src/Wolverine/Runtime/Agents/AssignmentGrid.cs index ba8bcea09..4125c6f20 100644 --- a/src/Wolverine/Runtime/Agents/AssignmentGrid.cs +++ b/src/Wolverine/Runtime/Agents/AssignmentGrid.cs @@ -59,6 +59,17 @@ public IReadOnlyList AvailableAgentsForScheme(string scheme) return _agents.Values.Where(x => x.Uri.Scheme.EqualsIgnoreCase(scheme) && !x.IsPaused).ToList(); } + /// + /// Same as , restricted to the agents matching + /// . Used when one scheme's agents are distributed in independent passes + /// (e.g. per event store; see ). + /// + public IReadOnlyList AvailableAgentsForScheme(string scheme, Func filter) + { + return _agents.Values + .Where(x => x.Uri.Scheme.EqualsIgnoreCase(scheme) && !x.IsPaused && filter(x.Uri)).ToList(); + } + /// /// Add an executing node to the grid. Useful for testing custom agent assignment /// schemes @@ -116,7 +127,16 @@ public AssignmentGrid WithAgents(params Uri[] uris) /// public IReadOnlyList MatchAgentsToCapableNodesFor(string scheme) { - var agents = AvailableAgentsForScheme(scheme); + return MatchAgentsToCapableNodesFor(scheme, _ => true); + } + + /// + /// Match up the agents for a particular scheme, restricted by , to any + /// nodes that could run that agent. This is meant for blue/green development + /// + public IReadOnlyList MatchAgentsToCapableNodesFor(string scheme, Func filter) + { + var agents = AvailableAgentsForScheme(scheme, filter); foreach (var agent in agents) { agent.CandidateNodes.Clear(); diff --git a/src/Wolverine/Runtime/Agents/DynamicListenerAgentFamily.cs b/src/Wolverine/Runtime/Agents/DynamicListenerAgentFamily.cs index fb5103a05..2bc9905ea 100644 --- a/src/Wolverine/Runtime/Agents/DynamicListenerAgentFamily.cs +++ b/src/Wolverine/Runtime/Agents/DynamicListenerAgentFamily.cs @@ -9,7 +9,7 @@ namespace Wolverine.Runtime.Agents; /// listener URIs and projects each one through /// . The cluster's /// then uses -/// to balance them across the +/// to balance them across the /// running nodes — one node per listener URI, so registering an MQTT topic /// activates exactly one consumer somewhere in the cluster regardless of how /// many nodes are alive. diff --git a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs index 61ac53e9e..3a2c8e4f5 100644 --- a/src/Wolverine/Runtime/Agents/EventStoreAgents.cs +++ b/src/Wolverine/Runtime/Agents/EventStoreAgents.cs @@ -30,6 +30,16 @@ public EventStoreAgents(IEventStore store, IObserver[] observers) public EventStoreIdentity Identity { get; } + /// + /// How many databases back this event store. The distribution in + /// keys off this: a store backed by + /// multiple databases (static or dynamic) gets database-affine assignment — all of a database's agents + /// kept together on one node so connection pools scale with the number of databases instead of + /// nodes × databases — while a single-database store keeps the even blue/green spread. + /// See JasperFx/jasperfx#486. + /// + public DatabaseCardinality DatabaseCardinality => _store.DatabaseCardinality; + public async ValueTask DisposeAsync() { foreach (var entry in _daemons.Enumerate()) diff --git a/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs b/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs index fb11fc4fc..0e80d1503 100644 --- a/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs +++ b/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs @@ -156,7 +156,33 @@ public async ValueTask> SupportedAgentsAsync() public ValueTask EvaluateAssignmentsAsync(AssignmentGrid assignments) { - assignments.DistributeEvenlyWithBlueGreenSemantics(SchemeName); + // JasperFx/jasperfx#486 / JasperFx/marten#4806: each store's agents are distributed in their own + // pass, keyed off the store's database cardinality. A store backed by multiple databases gets + // database-affine assignment — all of a shard database's agents (e.g. its per-tenant agents under + // sharded tenancy + per-tenant event partitioning) kept together on one node, so a node opens + // connection pools only to the databases it owns and pools scale with the number of databases + // rather than nodes × databases (which otherwise exhausts a shared server's max_connections). + // Single-database stores keep the even blue/green spread unchanged. + var multiDatabaseStoreKeys = new HashSet(); + foreach (var entry in _stores.Enumerate()) + { + if (entry.Value.DatabaseCardinality is DatabaseCardinality.StaticMultiple + or DatabaseCardinality.DynamicMultiple) + { + multiDatabaseStoreKeys.Add(entry.Key); + } + } + + foreach (var storeKey in multiDatabaseStoreKeys) + { + assignments.DistributeByGroupAffinity(SchemeName, DatabaseKeyOf, + uri => StoreKeyOf(uri) == storeKey); + } + + // Everything else — single-database stores and any URI that doesn't resolve to a known + // multi-database store — keeps the pre-existing even distribution with blue/green semantics. + assignments.DistributeEvenlyWithBlueGreenSemantics(SchemeName, + uri => !multiDatabaseStoreKeys.Contains(StoreKeyOf(uri))); return new ValueTask(); } @@ -169,6 +195,14 @@ internal static string DatabaseKeyOf(Uri uri) ? $"{uri.Host}/{uri.Segments[1].Trim('/')}/{uri.Segments[2].Trim('/')}" : uri.AbsoluteUri; + // The (type, name) prefix of an agent URI identifies the owning store; this reproduces the _stores + // key exactly the way BuildAgentAsync does. A URI too short to carry a store name can't belong to any + // store, so it gets a key no store key ever matches. + internal static string StoreKeyOf(Uri uri) + => uri.Segments.Length >= 2 + ? StoreKey($"{uri.Segments[1].Trim('/')}:{uri.Host}") + : uri.AbsoluteUri; + public async ValueTask DisposeAsync() { foreach (var store in _stores.Enumerate()) From afe02783a47f0a4abb0f91913c9063433f854c89 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 6 Jul 2026 12:41:25 -0500 Subject: [PATCH 13/18] Blue/green capability matching inside DistributeByGroupAffinity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Group placement now mirrors DistributeEvenlyWithBlueGreenSemantics' capability semantics: homogeneous capabilities keep the capability-blind placement; when node capabilities differ, a group's candidate nodes are those capable of every member (via MatchAgentsToCapableNodesFor), the least-loaded candidate hosts the whole group with the existing tie-breaks, and when no node can host the whole group each member falls back individually to its least-loaded capable node — an agent no node declares a capability for is left exactly as the even path leaves it. Epic: JasperFx/jasperfx#486. Co-Authored-By: Claude Fable 5 --- .../Agents/AssignmentGrid.Distribution.cs | 58 ++++++++++++++++--- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs b/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs index c6f2cd20a..bba52104d 100644 --- a/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs +++ b/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs @@ -121,6 +121,13 @@ public void DistributeByGroupAffinity(string scheme, Func groupKey) /// Same as , restricted to the /// agents of the scheme matching . Agents outside the filter are left /// completely untouched so one scheme can be distributed in several independent passes. + /// + /// Blue/green capability matching mirrors : + /// with homogeneous node capabilities placement is capability-blind; otherwise a group's candidate + /// nodes are those capable of running every agent in the group, the least-loaded candidate hosts the + /// group, and when no node can host the whole group each agent falls back individually to its + /// least-loaded capable node — an agent no node declares a capability for is left exactly as the even + /// path leaves it (running where it is, or unassigned). /// public void DistributeByGroupAffinity(string scheme, Func groupKey, Func filter) { @@ -129,7 +136,14 @@ public void DistributeByGroupAffinity(string scheme, Func groupKey, throw new InvalidOperationException("There are no active nodes"); } - var agents = AvailableAgentsForScheme(scheme, filter); + // Mirror DistributeEvenlyWithBlueGreenSemantics: identical capabilities everywhere means placement + // is capability-blind; otherwise match each agent to the nodes that declare it as a capability. + var sameCapabilities = AllNodesHaveSameCapabilities(scheme, filter); + + var agents = sameCapabilities + ? AvailableAgentsForScheme(scheme, filter) + : MatchAgentsToCapableNodesFor(scheme, filter); + if (agents.Count == 0) { return; @@ -139,6 +153,7 @@ public void DistributeByGroupAffinity(string scheme, Func groupKey, if (nodes.Count == 1) { + // Same single-node behavior as both even paths: the only node takes everything. var only = nodes[0]; foreach (var agent in agents) { @@ -160,12 +175,41 @@ public void DistributeByGroupAffinity(string scheme, Func groupKey, { var members = group.ToList(); - // The least-loaded node hosts the whole group (tie-breaks: non-leader first, then node id). - var node = load - .OrderBy(kv => kv.Value) - .ThenBy(kv => kv.Key.IsLeader) - .ThenBy(kv => kv.Key.AssignedId) - .First().Key; + // Candidate nodes for the whole group: nodes capable of running every member (all nodes when + // capabilities are homogeneous). + var candidates = sameCapabilities + ? nodes + : nodes.Where(n => members.All(m => m.CandidateNodes.Contains(n))).ToList(); + + if (candidates.Count == 0) + { + // No single node can host the whole group — mirror the blue/green even path's per-agent + // behavior: each member goes to its least-loaded capable node, and a member no node + // declares a capability for is simply left alone (candidate == null there too). + foreach (var member in members) + { + var candidate = member.CandidateNodes + .OrderBy(n => load.GetValueOrDefault(n)) + .ThenBy(n => n.IsLeader) + .ThenBy(n => n.AssignedId) + .FirstOrDefault(); + + if (candidate != null) + { + candidate.Assign(member); + load[candidate] += 1; + } + } + + continue; + } + + // The least-loaded candidate hosts the whole group (tie-breaks: non-leader first, then node id). + var node = candidates + .OrderBy(n => load[n]) + .ThenBy(n => n.IsLeader) + .ThenBy(n => n.AssignedId) + .First(); foreach (var agent in members) { From e3872bcfffa9567e9da87e6029688b7e6351f310 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 6 Jul 2026 12:43:34 -0500 Subject: [PATCH 14/18] Retire running agents superseded by a tenant-scoping change of their shard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The assignment grid is seeded from every node's ActiveAgents, so when a database gains its first tenant at runtime the old store-global agent is still present and running in the grid even though SupportedAgentsAsync now enumerates per-tenant agents instead — the distribution would keep it assigned and it would process events concurrently with its replacements (double-processing). EvaluateAssignmentsAsync now first detaches any agent of the scheme that (a) is no longer in the supported set and (b) shares a tenant-neutral key — store/database/projection/shardKey/version, tenant slot stripped per the ShardName.RelativeUrl grammar — with a supported agent, and excludes it from both distribution passes; a detached running agent is exactly what TryBuildAssignmentCommand turns into StopRemoteAgent. The inverse transition (tenant removed -> its per-tenant agent superseded by the remaining scoping) retires the same way. Deliberately narrower than 'not in the supported set': blue/green rollouts legitimately run agents the evaluating node's store doesn't enumerate (new projections/versions matched via node capabilities), and a version bump changes the version slot, not the tenant slot, so those keep running untouched. Epic: JasperFx/jasperfx#486. Co-Authored-By: Claude Fable 5 --- .../Agents/EventSubscriptionAgentFamily.cs | 110 +++++++++++++++++- 1 file changed, 106 insertions(+), 4 deletions(-) diff --git a/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs b/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs index 0e80d1503..40942bc10 100644 --- a/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs +++ b/src/Wolverine/Runtime/Agents/EventSubscriptionAgentFamily.cs @@ -154,8 +154,15 @@ public async ValueTask> SupportedAgentsAsync() return list; } - public ValueTask EvaluateAssignmentsAsync(AssignmentGrid assignments) + public async ValueTask EvaluateAssignmentsAsync(AssignmentGrid assignments) { + // Retire superseded agents FIRST — see RetireSupersededAgents below. The grid is seeded from every + // node's ActiveAgents, so an agent that this family no longer enumerates (e.g. the store-global + // agent for a database that just gained its first tenant, now replaced by per-tenant agents) is + // still present and running in the grid; without retirement the distribution below would happily + // keep it assigned and it would process events concurrently with its replacements. + var retired = await RetireSupersededAgentsAsync(assignments); + // JasperFx/jasperfx#486 / JasperFx/marten#4806: each store's agents are distributed in their own // pass, keyed off the store's database cardinality. A store backed by multiple databases gets // database-affine assignment — all of a shard database's agents (e.g. its per-tenant agents under @@ -176,15 +183,61 @@ public ValueTask EvaluateAssignmentsAsync(AssignmentGrid assignments) foreach (var storeKey in multiDatabaseStoreKeys) { assignments.DistributeByGroupAffinity(SchemeName, DatabaseKeyOf, - uri => StoreKeyOf(uri) == storeKey); + uri => !retired.Contains(uri) && StoreKeyOf(uri) == storeKey); } // Everything else — single-database stores and any URI that doesn't resolve to a known // multi-database store — keeps the pre-existing even distribution with blue/green semantics. assignments.DistributeEvenlyWithBlueGreenSemantics(SchemeName, - uri => !multiDatabaseStoreKeys.Contains(StoreKeyOf(uri))); + uri => !retired.Contains(uri) && !multiDatabaseStoreKeys.Contains(StoreKeyOf(uri))); + } + + /// + /// Stop agents of this scheme that the current enumeration no + /// longer lists AND that have been superseded by a tenant-scoping change of the same shard: the + /// supported set contains an agent for the same store, database, projection, shard key, and version + /// that differs only in the tenant slot. That is exactly the runtime transition where a database gains + /// its first tenant (store-global agent replaced by per-tenant agents) or loses one (per-tenant agent + /// replaced by the store-global agent or the remaining tenants) — the stale agent tracks the same + /// events as its replacements and MUST stop, or both run concurrently and double-process. + /// + /// Deliberately narrower than "not in the supported set": blue/green rollouts legitimately run + /// agents the evaluating node's own store does not enumerate (a green node's new projection or new + /// version, matched purely through node capabilities), and those must keep running. A version bump + /// changes the version slot — not the tenant slot — so it never matches this rule. + /// + private async ValueTask> RetireSupersededAgentsAsync(AssignmentGrid assignments) + { + var retired = new HashSet(); + + var supported = (await SupportedAgentsAsync()).ToHashSet(); + var supportedTenantNeutral = supported + .Select(TenantNeutralKeyOf) + .Where(x => x != null) + .ToHashSet(); + + foreach (var agent in assignments.AgentsForScheme(SchemeName)) + { + if (supported.Contains(agent.Uri)) + { + continue; + } + + var neutral = TenantNeutralKeyOf(agent.Uri); + if (neutral == null || !supportedTenantNeutral.Contains(neutral)) + { + continue; + } + + retired.Add(agent.Uri); + + // Detaching a running agent leaves OriginalNode set with no AssignedNode, which is exactly + // what makes TryBuildAssignmentCommand emit a StopRemoteAgent to wherever it runs. The + // distribution passes exclude retired URIs so nothing re-assigns them. + agent.Detach(); + } - return new ValueTask(); + return retired; } // Agent URIs are event-subscriptions://{type}/{name}/{databaseId}/{shard...} (see UriFor); the @@ -203,6 +256,55 @@ internal static string StoreKeyOf(Uri uri) ? StoreKey($"{uri.Segments[1].Trim('/')}:{uri.Host}") : uri.AbsoluteUri; + /// + /// The agent URI with the tenant slot removed: store type + name + database + projection name + shard + /// key + version. Two agent URIs share a tenant-neutral key exactly when they track the same shard of + /// the same projection in the same database and differ only in tenant scoping (store-global vs + /// per-tenant, or different tenants) — the supersession relation RetireSupersededAgentsAsync keys on. + /// The shard path follows JasperFx's ShardName.RelativeUrl grammar, + /// {name}/{shardKey}[/v{version}][/{tenant}], so with three trailing segments a + /// v{digits} third segment is the version (same heuristic as ShardName.TryParse) and anything + /// else is a tenant. Returns null for a URI that doesn't parse as this scheme's grammar — such an + /// agent is never treated as superseded. + /// + internal static string? TenantNeutralKeyOf(Uri uri) + { + // Segments: "/", {storeName}, {databaseId}, {projectionName}, {shardKey}, [v{n} | tenant], [tenant] + var segments = uri.Segments.Skip(1).Select(x => x.Trim('/')).Where(x => x.Length > 0).ToArray(); + if (segments.Length is < 4 or > 6) + { + return null; + } + + var version = "v1"; + if (segments.Length == 5) + { + if (IsVersionSegment(segments[4])) + { + version = segments[4]; + } + // else segments[4] is a tenant id and the shard is version 1 + } + else if (segments.Length == 6) + { + if (!IsVersionSegment(segments[4])) + { + // Six segments only fit the grammar as name/shardKey/v{n}/tenant — anything else is + // malformed, and a malformed URI must never look like somebody's supersession. + return null; + } + + version = segments[4]; + } + + return $"{uri.Host}/{segments[0]}/{segments[1]}/{segments[2]}/{segments[3]}/{version}" + .ToLowerInvariant(); + } + + private static bool IsVersionSegment(string segment) + => segment.Length >= 2 && (segment[0] == 'v' || segment[0] == 'V') && + segment.Skip(1).All(char.IsAsciiDigit); + public async ValueTask DisposeAsync() { foreach (var store in _stores.Enumerate()) From 45c30d78a86b829673850064646e20dcc87322fd Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 6 Jul 2026 12:55:44 -0500 Subject: [PATCH 15/18] Tests for cardinality-driven affinity, mixed stores, failover, and stale retirement Replaces the flag-based family suite with event_subscription_family_cardinality_assignment (affinity auto-on for Static/DynamicMultiple stores, even spread for Single/None, mixed-store independence, blue/green capabilities honored per pass, node-loss failover regrouping on survivors), adds event_subscription_family_stale_agent_retirement (store-global agent stopped when a database gains its first tenants, per-tenant agent stopped when its tenant is removed, unrelated/other-version agents left running), tenant_neutral_key_of grammar tests, and capability-aware DistributeByGroupAffinity tests (group lands only on a capable node; per-member fallback parks agents no node declares). Epic: JasperFx/jasperfx#486. Co-Authored-By: Claude Fable 5 --- .../Agents/distribute_by_group_affinity.cs | 45 ++ ...scription_family_cardinality_assignment.cs | 421 ++++++++++++++++++ 2 files changed, 466 insertions(+) create mode 100644 src/Testing/CoreTests/Runtime/Agents/event_subscription_family_cardinality_assignment.cs diff --git a/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs b/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs index 335f0b311..eb47c6b24 100644 --- a/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs +++ b/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs @@ -93,4 +93,49 @@ public void a_heavy_database_stays_on_one_node_and_light_groups_balance_around_i grid.AgentFor(Agent("dbLight1", "t1")).AssignedNode.ShouldNotBe(heavyNode); grid.AgentFor(Agent("dbLight2", "t1")).AssignedNode.ShouldNotBe(heavyNode); } + + [Fact] + public void a_group_lands_only_on_a_node_capable_of_running_it() + { + // Blue/green: node capabilities differ, so placement must respect them exactly like + // DistributeEvenlyWithBlueGreenSemantics — the db1 group may only land on the capable node 2, + // even though the load-based choice alone would prefer the emptier node 1. + var db1Agents = new[] { Agent("db1", "t1"), Agent("db1", "t2") }; + + var grid = new AssignmentGrid(); + grid.WithNode(1, Guid.NewGuid()); + var node2 = grid.WithNode(2, Guid.NewGuid()).HasCapabilities(db1Agents); + + grid.WithAgents(db1Agents); + + grid.DistributeByGroupAffinity("event-subscriptions", DatabaseKey); + + foreach (var uri in db1Agents) + { + grid.AgentFor(uri).AssignedNode.ShouldBe(node2, + "the group must land on the only node that declares its agents as capabilities"); + } + } + + [Fact] + public void when_no_node_can_host_the_whole_group_members_fall_back_individually() + { + // Mirrors the blue/green even path: no single node is capable of every member, so each member goes + // to its own least-loaded capable node, and a member NO node declares stays unassigned (parked). + var t1 = Agent("db1", "t1"); + var t2 = Agent("db1", "t2"); + var t3 = Agent("db1", "t3"); + + var grid = new AssignmentGrid(); + var node1 = grid.WithNode(1, Guid.NewGuid()).HasCapabilities(new[] { t1 }); + var node2 = grid.WithNode(2, Guid.NewGuid()).HasCapabilities(new[] { t2 }); + + grid.WithAgents(t1, t2, t3); + + grid.DistributeByGroupAffinity("event-subscriptions", DatabaseKey); + + grid.AgentFor(t1).AssignedNode.ShouldBe(node1); + grid.AgentFor(t2).AssignedNode.ShouldBe(node2); + grid.AgentFor(t3).AssignedNode.ShouldBeNull("an agent no node declares a capability for is parked, exactly like the even path"); + } } diff --git a/src/Testing/CoreTests/Runtime/Agents/event_subscription_family_cardinality_assignment.cs b/src/Testing/CoreTests/Runtime/Agents/event_subscription_family_cardinality_assignment.cs new file mode 100644 index 000000000..115eeee9c --- /dev/null +++ b/src/Testing/CoreTests/Runtime/Agents/event_subscription_family_cardinality_assignment.cs @@ -0,0 +1,421 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using JasperFx.Descriptors; +using JasperFx.Events; +using JasperFx.Events.Descriptors; +using JasperFx.Events.Projections; +using NSubstitute; +using Shouldly; +using Wolverine.Runtime.Agents; +using Xunit; + +namespace CoreTests.Runtime.Agents; + +/// +/// JasperFx/jasperfx#486: derives the +/// distribution style from each store's — no opt-in flag. A +/// store backed by multiple databases (Static/DynamicMultiple) gets database-affine assignment (a +/// database's agents kept together on one node); a single-database store keeps the even blue/green spread; +/// both styles coexist store-by-store within the one scheme. These are fast unit tests: agents are +/// registered directly on an and the store is stubbed, so no host or database +/// is required. +/// +public class event_subscription_family_cardinality_assignment +{ + // Agent URI grammar is event-subscriptions://{type}/{name}/{databaseId}/{shard...} (see EventStoreAgents.UriFor). + private static Uri Agent(string db, string tenant) => + new($"event-subscriptions://marten/main/{db}/proj/all/{tenant}"); + + private static Uri SingleStoreAgent(string projection) => + new($"event-subscriptions://other/single/localhost.app/{projection}/all"); + + private static IEventStore StoreWith(string name, string type, DatabaseCardinality cardinality) + { + var store = Substitute.For(); + store.Identity.Returns(new EventStoreIdentity(name, type)); + store.DatabaseCardinality.Returns(cardinality); + store.TryCreateUsage(Arg.Any()).Returns(Task.FromResult(null)); + return store; + } + + private static EventSubscriptionAgentFamily FamilyFor(params IEventStore[] stores) + => new(stores, Array.Empty>()); + + [Fact] + public async Task keeps_a_databases_agents_together_when_the_store_is_multi_database() + { + var family = FamilyFor(StoreWith("main", "marten", DatabaseCardinality.StaticMultiple)); + + var grid = new AssignmentGrid(); + grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + // One shard database with two tenants: even distribution would split it across the two nodes. + grid.WithAgents(Agent("dbShared", "t1"), Agent("dbShared", "t2")); + + await family.EvaluateAssignmentsAsync(grid); + + var nodes = new[] { "t1", "t2" } + .Select(t => grid.AgentFor(Agent("dbShared", t)).AssignedNode) + .Distinct() + .ToList(); + + nodes.Count.ShouldBe(1, "a multi-database store must keep a shard database's agents on a single node"); + nodes[0].ShouldNotBeNull(); + } + + [Theory] + [InlineData(DatabaseCardinality.Single)] + [InlineData(DatabaseCardinality.None)] + public async Task spreads_agents_evenly_when_the_store_is_not_multi_database(DatabaseCardinality cardinality) + { + var family = FamilyFor(StoreWith("main", "marten", cardinality)); + + var grid = new AssignmentGrid(); + grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + grid.WithAgents(Agent("dbShared", "t1"), Agent("dbShared", "t2")); + + await family.EvaluateAssignmentsAsync(grid); + + var nodes = new[] { "t1", "t2" } + .Select(t => grid.AgentFor(Agent("dbShared", t)).AssignedNode) + .Distinct() + .ToList(); + + nodes.Count.ShouldBe(2, "without a multi-database cardinality even distribution splits a two-agent database across both nodes"); + } + + [Fact] + public async Task dynamic_multiple_is_also_distributed_affine() + { + var family = FamilyFor(StoreWith("main", "marten", DatabaseCardinality.DynamicMultiple)); + + var grid = new AssignmentGrid(); + grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + grid.WithAgents(Agent("dbShared", "t1"), Agent("dbShared", "t2")); + + await family.EvaluateAssignmentsAsync(grid); + + new[] { "t1", "t2" } + .Select(t => grid.AgentFor(Agent("dbShared", t)).AssignedNode) + .Distinct() + .Count() + .ShouldBe(1); + } + + [Fact] + public async Task mixed_stores_are_distributed_independently_affine_and_even() + { + // One multi-database store (affine) and one single-database store (even) in the same application. + var family = FamilyFor( + StoreWith("main", "marten", DatabaseCardinality.StaticMultiple), + StoreWith("single", "other", DatabaseCardinality.Single)); + + var grid = new AssignmentGrid(); + grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + + // Multi-database store: two shard databases, two tenants each. + var affineAgents = new List(); + foreach (var db in new[] { "db1", "db2" }) + foreach (var tenant in new[] { "t1", "t2" }) + affineAgents.Add(Agent(db, tenant)); + + // Single-database store: two projection agents. + var evenAgents = new[] { SingleStoreAgent("proj1"), SingleStoreAgent("proj2") }; + + grid.WithAgents(affineAgents.Concat(evenAgents).ToArray()); + + await family.EvaluateAssignmentsAsync(grid); + + // Affine store: each database whole on one node. + foreach (var db in new[] { "db1", "db2" }) + { + new[] { "t1", "t2" } + .Select(t => grid.AgentFor(Agent(db, t)).AssignedNode) + .Distinct() + .Count() + .ShouldBe(1, $"all agents of {db} must be on a single node"); + } + + // Even store: its two agents split 1/1 across the nodes, unaffected by the affine pass. + var evenNodes = evenAgents.Select(u => grid.AgentFor(u).AssignedNode).ToList(); + evenNodes.ShouldAllBe(n => n != null); + evenNodes.Distinct().Count().ShouldBe(2, "the single-database store keeps the even spread"); + } + + [Fact] + public async Task mixed_stores_honor_blue_green_capabilities_on_the_even_store() + { + var family = FamilyFor( + StoreWith("main", "marten", DatabaseCardinality.StaticMultiple), + StoreWith("single", "other", DatabaseCardinality.Single)); + + var evenAgents = new[] { SingleStoreAgent("proj1"), SingleStoreAgent("proj2") }; + + var grid = new AssignmentGrid(); + // Only node 1 declares the single-database store's agents as capabilities (blue/green rollout); + // neither node declares the multi-database store's agents, so THAT pass sees homogeneous + // (empty) capabilities and stays capability-blind. + grid.WithNode(1, Guid.NewGuid()).HasCapabilities(evenAgents); + grid.WithNode(2, Guid.NewGuid()); + + var affineAgents = new[] { Agent("db1", "t1"), Agent("db1", "t2") }; + grid.WithAgents(affineAgents.Concat(evenAgents).ToArray()); + + await family.EvaluateAssignmentsAsync(grid); + + // The even store's agents may only land on the node capable of running them. + foreach (var uri in evenAgents) + { + grid.AgentFor(uri).AssignedNode!.AssignedId.ShouldBe(1, + "a blue/green agent must land on the node that declares it as a capability"); + } + + // The affine store is untouched by the other store's rollout: db1 whole on one node. + affineAgents.Select(u => grid.AgentFor(u).AssignedNode).Distinct().Count().ShouldBe(1); + } + + [Fact] + public async Task failover_reassigns_whole_groups_to_the_surviving_nodes() + { + var family = FamilyFor(StoreWith("main", "marten", DatabaseCardinality.StaticMultiple)); + + var grid = new AssignmentGrid(); + grid.WithNode(1, Guid.NewGuid()); + var node2 = grid.WithNode(2, Guid.NewGuid()); + + var agents = new List(); + foreach (var db in new[] { "db1", "db2" }) + foreach (var tenant in new[] { "t1", "t2" }) + agents.Add(Agent(db, tenant)); + + grid.WithAgents(agents.ToArray()); + + await family.EvaluateAssignmentsAsync(grid); + grid.AllAgents.ShouldAllBe(a => a.AssignedNode != null); + + // Node 2 dies. Re-evaluating must land every group whole on a survivor. + grid.Remove(node2); + await family.EvaluateAssignmentsAsync(grid); + + foreach (var db in new[] { "db1", "db2" }) + { + var assigned = new[] { "t1", "t2" } + .Select(t => grid.AgentFor(Agent(db, t)).AssignedNode) + .Distinct() + .ToList(); + + assigned.Count.ShouldBe(1, $"{db} must be whole on a surviving node"); + assigned[0]!.AssignedId.ShouldBe(1); + } + } +} + +/// +/// JasperFx/jasperfx#486 (stale-agent retirement): when the supported agent set changes shape because a +/// database's tenant scoping changed — a database gains its first tenant (store-global agent replaced by +/// per-tenant agents) or loses one — an agent still running from the previous shape must be STOPPED by the +/// next assignment evaluation, or it double-processes the same events as its replacements. +/// +public class event_subscription_family_stale_agent_retirement +{ + private static readonly EventStoreIdentity TheIdentity = new("main", "marten"); + private static readonly DatabaseId TheDatabase = new("localhost", "claims1"); + private static readonly ShardName BaseShard = ShardName.Compose("provided-cares"); + + private static EventStoreUsage UsageWith(params string[] tenantIds) + { + return new EventStoreUsage + { + Database = new DatabaseUsage + { + Databases = + { + new DatabaseDescriptor + { + ServerName = TheDatabase.Server, + DatabaseName = TheDatabase.Name, + TenantIds = tenantIds.ToList() + } + } + }, + Subscriptions = + { + new SubscriptionDescriptor(SubscriptionType.SingleStreamProjection) + { + Lifecycle = ProjectionLifecycle.Async, + ShardNames = new[] { BaseShard } + } + } + }; + } + + private static EventSubscriptionAgentFamily FamilyFor(EventStoreUsage usage, bool distributesPerTenant = true) + { + var store = Substitute.For(); + store.Identity.Returns(TheIdentity); + store.DatabaseCardinality.Returns(DatabaseCardinality.StaticMultiple); + store.DistributesAgentsPerTenant.Returns(distributesPerTenant); + store.TryCreateUsage(Arg.Any()).Returns(Task.FromResult(usage)); + return new EventSubscriptionAgentFamily(new[] { store }, Array.Empty>()); + } + + private static Uri UriForTenant(string? tenantId) + => EventSubscriptionAgentFamily.UriFor(TheIdentity, TheDatabase, + tenantId == null ? BaseShard : BaseShard.ForTenant(tenantId)); + + [Fact] + public async Task running_store_global_agent_is_stopped_when_the_database_gains_its_first_tenants() + { + // The store now enumerates per-tenant agents (two tenants were provisioned at runtime) … + var family = FamilyFor(UsageWith("t1", "t2")); + + var staleStoreGlobal = UriForTenant(null); + + var grid = new AssignmentGrid(); + var node1 = grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + + // … but the store-global agent from the empty-database phase is still RUNNING on node 1. + node1.Running(staleStoreGlobal); + grid.WithAgents(UriForTenant("t1"), UriForTenant("t2")); + + await family.EvaluateAssignmentsAsync(grid); + + var stale = grid.AgentFor(staleStoreGlobal); + stale.AssignedNode.ShouldBeNull("the superseded store-global agent must be detached so it is stopped"); + stale.OriginalNode.ShouldBe(node1, "the stop must target the node it was running on"); + + // Its per-tenant replacements are assigned as one database group. + var tenantNodes = new[] { "t1", "t2" } + .Select(t => grid.AgentFor(UriForTenant(t)).AssignedNode) + .ToList(); + tenantNodes.ShouldAllBe(n => n != null); + tenantNodes.Distinct().Count().ShouldBe(1); + } + + [Fact] + public async Task running_per_tenant_agent_is_stopped_when_its_tenant_is_removed() + { + // The inverse transition: tenant t2 was removed, so only t1 is enumerated … + var family = FamilyFor(UsageWith("t1")); + + var staleTenantAgent = UriForTenant("t2"); + + var grid = new AssignmentGrid(); + var node1 = grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + + // … but t2's agent is still running from before the removal. + node1.Running(staleTenantAgent); + grid.WithAgents(UriForTenant("t1")); + + await family.EvaluateAssignmentsAsync(grid); + + grid.AgentFor(staleTenantAgent).AssignedNode.ShouldBeNull("the removed tenant's agent must be stopped"); + grid.AgentFor(UriForTenant("t1")).AssignedNode.ShouldNotBeNull(); + } + + [Fact] + public async Task an_agent_with_no_supported_counterpart_is_left_alone() + { + // A running agent of a DIFFERENT projection (e.g. a green node's new projection the evaluating + // store doesn't enumerate) shares no tenant-neutral key with the supported set, so retirement must + // NOT touch it — that is the blue/green escape hatch. + var family = FamilyFor(UsageWith("t1")); + + var unrelated = EventSubscriptionAgentFamily.UriFor(TheIdentity, TheDatabase, + ShardName.Compose("brand-new-green-projection")); + + var grid = new AssignmentGrid(); + var node1 = grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + + node1.Running(unrelated); + grid.WithAgents(UriForTenant("t1")); + + await family.EvaluateAssignmentsAsync(grid); + + grid.AgentFor(unrelated).AssignedNode.ShouldNotBeNull("an agent without a superseding counterpart keeps running"); + } + + [Fact] + public async Task a_different_version_of_the_same_shard_is_not_retired() + { + // Blue/green version bump: v2 is enumerated, v1 still runs on a blue node. The version slot — + // not the tenant slot — differs, so v1 must keep running side by side during the rollout. + var v2 = ShardName.Compose("provided-cares", version: 2); + var usage = UsageWith("t1"); + usage.Subscriptions[0].ShardNames = new[] { v2 }; + var family = FamilyFor(usage); + + var v1Agent = EventSubscriptionAgentFamily.UriFor(TheIdentity, TheDatabase, BaseShard.ForTenant("t1")); + + var grid = new AssignmentGrid(); + var node1 = grid.WithNode(1, Guid.NewGuid()); + grid.WithNode(2, Guid.NewGuid()); + + node1.Running(v1Agent); + + await family.EvaluateAssignmentsAsync(grid); + + grid.AgentFor(v1Agent).AssignedNode.ShouldNotBeNull("a different projection version is a blue/green rollout, not supersession"); + } +} + +public class tenant_neutral_key_of +{ + private static string? KeyOf(string uri) => EventSubscriptionAgentFamily.TenantNeutralKeyOf(new Uri(uri)); + + [Fact] + public void base_shard_and_its_tenant_scoped_variants_share_a_key() + { + var baseKey = KeyOf("event-subscriptions://marten/main/localhost.db1/proj/all"); + baseKey.ShouldNotBeNull(); + KeyOf("event-subscriptions://marten/main/localhost.db1/proj/all/t1").ShouldBe(baseKey); + KeyOf("event-subscriptions://marten/main/localhost.db1/proj/all/t2").ShouldBe(baseKey); + } + + [Fact] + public void versions_do_not_share_a_key() + { + KeyOf("event-subscriptions://marten/main/localhost.db1/proj/all/v2") + .ShouldNotBe(KeyOf("event-subscriptions://marten/main/localhost.db1/proj/all")); + + // … but a versioned base and its versioned tenant variant do. + KeyOf("event-subscriptions://marten/main/localhost.db1/proj/all/v2/t1") + .ShouldBe(KeyOf("event-subscriptions://marten/main/localhost.db1/proj/all/v2")); + } + + [Fact] + public void different_databases_projections_and_stores_do_not_share_a_key() + { + var key = KeyOf("event-subscriptions://marten/main/localhost.db1/proj/all/t1"); + + KeyOf("event-subscriptions://marten/main/localhost.db2/proj/all/t1").ShouldNotBe(key); + KeyOf("event-subscriptions://marten/main/localhost.db1/other/all/t1").ShouldNotBe(key); + KeyOf("event-subscriptions://marten/other/localhost.db1/proj/all/t1").ShouldNotBe(key); + } + + [Fact] + public void malformed_uris_yield_no_key() + { + // Too short to carry a shard path at all. + KeyOf("event-subscriptions://marten/main/localhost.db1").ShouldBeNull(); + + // Six segments only fit the grammar as name/shardKey/v{n}/tenant. + KeyOf("event-subscriptions://marten/main/localhost.db1/proj/all/nonversion/t1").ShouldBeNull(); + } + + [Fact] + public void a_tenant_that_merely_looks_versionish_is_still_a_tenant_when_a_version_segment_precedes_it() + { + KeyOf("event-subscriptions://marten/main/localhost.db1/proj/all/v2/v3") + .ShouldBe(KeyOf("event-subscriptions://marten/main/localhost.db1/proj/all/v2")); + } +} From 445b10ca85d3d255d4720331ba9d39cc97d58bb3 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 6 Jul 2026 13:13:17 -0500 Subject: [PATCH 16/18] E2E: two shard databases co-locate their per-tenant agents per node, automatically Two shard databases with two co-located tenants each across a two-node cluster: the Marten store's multi-database cardinality alone (no opt-in) makes managed distribution assign each database's per-(shard, tenant) agents together on one node, and spread the two database groups one per node. Requires the Marten side's DistributesAgentsPerTenant override, so like sharded_tenant_partitioned_distribution this goes green once the companion Marten release ships. Epic: JasperFx/jasperfx#486. Co-Authored-By: Claude Fable 5 --- ...sharded_two_databases_affine_colocation.cs | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 src/Persistence/MartenTests/Distribution/sharded_two_databases_affine_colocation.cs diff --git a/src/Persistence/MartenTests/Distribution/sharded_two_databases_affine_colocation.cs b/src/Persistence/MartenTests/Distribution/sharded_two_databases_affine_colocation.cs new file mode 100644 index 000000000..c1460cc74 --- /dev/null +++ b/src/Persistence/MartenTests/Distribution/sharded_two_databases_affine_colocation.cs @@ -0,0 +1,166 @@ +using IntegrationTests; +using JasperFx.CodeGeneration; +using JasperFx.Core; +using JasperFx.Events; +using JasperFx.Events.Projections; +using JasperFx.MultiTenancy; +using Marten; +using Marten.Events; +using MartenTests.Distribution.Support; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Npgsql; +using Shouldly; +using Weasel.Postgresql; +using Weasel.Postgresql.Migrations; +using Wolverine; +using Wolverine.Marten; +using Wolverine.Runtime.Agents; +using Wolverine.Tracking; +using Xunit; +using Xunit.Abstractions; + +namespace MartenTests.Distribution; + +// JasperFx/jasperfx#486, the database-affine half of the revised #3280 design: with TWO shard databases +// (two co-located tenants each) across a TWO-node cluster, the per-(shard, tenant) agents must be assigned +// with database affinity — each shard database's agents land together on a single node — because the +// Marten store reports a multi-database cardinality. No opt-in configuration anywhere. +public class sharded_two_databases_affine_colocation(ITestOutputHelper output) : PostgresqlContext, IAsyncLifetime +{ + private readonly List _hosts = []; + + // Unique names per run: managed partitions/databases from one run otherwise break the next run's + // resource-setup DDL reconciliation. + private readonly string theSuffix = Guid.NewGuid().ToString("N")[..8]; + private string ShardDatabase1 => $"w3281_aff1_{theSuffix}"; + private string ShardDatabase2 => $"w3281_aff2_{theSuffix}"; + private string MasterSchema => $"csp_affine_{theSuffix}"; + + private void ConfigureShardedStore(StoreOptions m) + { + string ConnectionStringFor(string database) => new NpgsqlConnectionStringBuilder(Servers.PostgresConnectionString) + { + Database = database + }.ConnectionString; + + m.DisableNpgsqlLogging = true; + m.MultiTenantedWithShardedDatabases(x => + { + x.ConnectionString = Servers.PostgresConnectionString; + x.SchemaName = MasterSchema; + x.PartitionSchemaName = MasterSchema + "_tenants"; + x.AddDatabase("shard-1", ConnectionStringFor(ShardDatabase1)); + x.AddDatabase("shard-2", ConnectionStringFor(ShardDatabase2)); + }); + + m.Events.StreamIdentity = StreamIdentity.AsString; + m.Events.TenancyStyle = TenancyStyle.Conjoined; + m.Events.AppendMode = EventAppendMode.Quick; + m.Events.UseTenantPartitionedEvents = true; + m.Events.UseIdentityMapForAggregates = false; + + m.Schema.For().MultiTenanted(); + m.Projections.Snapshot(SnapshotLifecycle.Async); + } + + public async Task InitializeAsync() + { + await using (var conn = new NpgsqlConnection(Servers.PostgresConnectionString)) + { + await conn.OpenAsync(); + foreach (var database in new[] { ShardDatabase1, ShardDatabase2 }) + { + if (!await conn.DatabaseExists(database)) + { + await new DatabaseSpecification().BuildDatabase(conn, database); + } + } + } + + // Two co-located tenants PER SHARD DATABASE, provisioned before any host starts so the first + // agent enumeration already fans out per tenant. + await using (var provisioning = DocumentStore.For(ConfigureShardedStore)) + { + await provisioning.Storage.ApplyAllConfiguredChangesToDatabaseAsync(); + await provisioning.Advanced.AddTenantToShardAsync("alpha1", "shard-1", default); + await provisioning.Advanced.AddTenantToShardAsync("alpha2", "shard-1", default); + await provisioning.Advanced.AddTenantToShardAsync("beta1", "shard-2", default); + await provisioning.Advanced.AddTenantToShardAsync("beta2", "shard-2", default); + } + } + + public async Task DisposeAsync() + { + foreach (var host in _hosts) + { + host.GetRuntime().Agents.DisableHealthChecks(); + await host.StopAsync(); + host.Dispose(); + } + } + + private async Task StartHostAsync() + { + var host = await Host.CreateDefaultBuilder() + .UseWolverine(opts => + { + opts.Durability.HealthCheckPollingTime = 1.Seconds(); + opts.Durability.CheckAssignmentPeriod = 1.Seconds(); + + opts.Services.AddMarten(ConfigureShardedStore) + .IntegrateWithWolverine(m => + { + m.UseWolverineManagedEventSubscriptionDistribution = true; + // Sharded tenancy has no single "main" database, so Wolverine's message store needs + // an explicit master. + m.MainDatabaseConnectionString = Servers.PostgresConnectionString; + }); + + opts.Services.AddSingleton(new OutputLoggerProvider(output)); + opts.Discovery.DisableConventionalDiscovery(); + opts.CodeGeneration.TypeLoadMode = TypeLoadMode.Auto; + }).StartAsync(); + + _hosts.Add(host); + return host; + } + + [Fact] + public async Task each_shard_databases_agents_are_co_located_on_one_node() + { + var leader = await StartHostAsync(); + await leader.WaitUntilAssumesLeadershipAsync(10.Seconds()); + + var second = await StartHostAsync(); + + // 1 async projection × 2 tenants per database × 2 databases = 4 per-tenant agents; whole-database + // groups of equal size across two nodes balance 2 + 2. + await leader.WaitUntilAssignmentsChangeTo(w => + { + w.AgentScheme = EventSubscriptionAgentFamily.SchemeName; + w.ExpectRunningAgents(leader, 2); + w.ExpectRunningAgents(second, 2); + }, 60.Seconds()); + + // Every shard database's agents run whole on exactly one node. + var byDatabase = _hosts + .SelectMany(host => host.GetRuntime().Agents.AllRunningAgentUris() + .Where(u => u.Scheme == EventSubscriptionAgentFamily.SchemeName) + .Select(uri => (Node: host, Uri: uri))) + .GroupBy(x => EventSubscriptionAgentFamily.DatabaseKeyOf(x.Uri)) + .ToList(); + + byDatabase.Count.ShouldBe(2, "two shard databases, each one agent group"); + foreach (var databaseGroup in byDatabase) + { + databaseGroup.Select(x => x.Node).Distinct().Count().ShouldBe(1, + $"agents of {databaseGroup.Key} must all run on the same node"); + databaseGroup.Count().ShouldBe(2, "one agent per co-located tenant"); + } + + // And the two databases are actually spread — one per node, not both piled on one. + byDatabase.Select(g => g.Select(x => x.Node).Distinct().Single()).Distinct().Count().ShouldBe(2); + } +} From 748762a422e18cad610144d13e15987d2a4312d5 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 6 Jul 2026 13:48:13 -0500 Subject: [PATCH 17/18] Group placement: minimal disruption + grandfathering for stale capability snapshots Two even-path semantics the group-affinity placement was missing, both exposed by MartenTests' MultiTenantContext suites (spread_databases_out_via_host): 1. Grandfathering: node capability snapshots are persisted once at node startup, so a node that started before its store's tenant databases were provisioned declares no event-subscription capabilities even though it runs those agents. The even paths leave running agents in place regardless of capabilities; a node already running part of a group is now a candidate for that group. 2. Minimal disruption with a ceiling: the even paths only move agents off a node when it exceeds ceil(agents/nodes). Group placement reshuffled every group from scratch each evaluation, so a stale-capability node could be starved permanently once an intermediate evaluation moved its groups away. The node currently running a WHOLE group now keeps it unless that pushes it past the ceiling; only over-ceiling groups move (to the least-loaded candidate, same tie-breaks). The no-candidate fallback likewise leaves already-assigned members in place. Epic: JasperFx/jasperfx#486. Co-Authored-By: Claude Fable 5 --- .../Agents/distribute_by_group_affinity.cs | 59 +++++++++++++++++++ .../Agents/AssignmentGrid.Distribution.cs | 45 ++++++++++++-- 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs b/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs index eb47c6b24..6f5e6b550 100644 --- a/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs +++ b/src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs @@ -138,4 +138,63 @@ public void when_no_node_can_host_the_whole_group_members_fall_back_individually grid.AgentFor(t2).AssignedNode.ShouldBe(node2); grid.AgentFor(t3).AssignedNode.ShouldBeNull("an agent no node declares a capability for is parked, exactly like the even path"); } + + [Fact] + public void a_node_already_running_a_groups_agents_stays_a_candidate_despite_a_stale_capability_snapshot() + { + // Capability snapshots are persisted once at node startup, so a node that started before the + // tenant databases were provisioned declares NO event-subscription capabilities even though it is + // running all the agents (MartenTests' MultiTenantContext starts exactly this way). The even paths + // tolerate that by leaving running agents in place; group placement must grandfather such a node + // as a candidate too, or it is starved and another node ends up with several whole databases. + var databases = new[] { "db1", "db2", "db3" }; + var tenants = new[] { "t1", "t2", "t3" }; + var all = databases.SelectMany(db => tenants.Select(t => Agent(db, t))).ToArray(); + + var grid = new AssignmentGrid(); + var node1 = grid.WithNode(1, Guid.NewGuid()); + node1.Running(all); // was the only node; no capabilities declared + grid.WithNode(2, Guid.NewGuid()).HasCapabilities(all); + grid.WithNode(3, Guid.NewGuid()).HasCapabilities(all); + + grid.DistributeByGroupAffinity("event-subscriptions", DatabaseKey); + + // Every database whole on one node, and the three databases spread across all three nodes -- + // including the stale-capability node. + var hosts = databases.Select(db => + { + var nodes = tenants.Select(t => grid.AgentFor(Agent(db, t)).AssignedNode).Distinct().ToList(); + nodes.Count.ShouldBe(1, $"all agents of {db} must be on a single node"); + return nodes[0]!; + }).ToList(); + + hosts.Distinct().Count().ShouldBe(3, "three equal databases across three nodes must land one per node"); + } + + [Fact] + public void an_incumbent_node_keeps_its_group_up_to_the_ceiling() + { + // Mid-convergence snapshot of the MultiTenantContext scenario: node 1 (stale capability snapshot, + // declares nothing) still runs db1's agent, node 2 runs db2's and db3's, node 3 just joined. The + // even path resolves this to 1/1/1 by moving only the over-ceiling extra; group placement must do + // the same — keep incumbents up to the ceiling, move only db3 — instead of reshuffling from + // scratch, which starves the stale-capability node forever. + var g1 = Agent("db1", "only"); + var g2 = Agent("db2", "only"); + var g3 = Agent("db3", "only"); + + var grid = new AssignmentGrid(); + var node1 = grid.WithNode(1, Guid.NewGuid()); + node1.Running(g1); // no declared capabilities + var node2 = grid.WithNode(2, Guid.NewGuid()).HasCapabilities(new[] { g1, g2, g3 }); + node2.Running(g2, g3); + var node3 = grid.WithNode(3, Guid.NewGuid()).HasCapabilities(new[] { g1, g2, g3 }); + + grid.DistributeByGroupAffinity("event-subscriptions", DatabaseKey); + + grid.AgentFor(g1).AssignedNode.ShouldBe(node1, "the incumbent keeps its group despite the stale capability snapshot"); + grid.AgentFor(g2).AssignedNode.ShouldBe(node2, "an under-ceiling incumbent keeps its group"); + grid.AgentFor(g3).AssignedNode.ShouldBe(node3, "only the over-ceiling group moves, to the empty node"); + } } + diff --git a/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs b/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs index bba52104d..d2f305e23 100644 --- a/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs +++ b/src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs @@ -165,6 +165,11 @@ public void DistributeByGroupAffinity(string scheme, Func groupKey, var load = nodes.ToDictionary(n => n, _ => 0); + // Mirror the even paths' per-node ceiling so groups spread instead of piling up on the node that + // happens to be running them today. A single group larger than the ceiling still occupies one + // node whole — groups are indivisible by design. + var maximum = (int)Math.Ceiling((double)agents.Count / nodes.Count); + var groups = agents .GroupBy(a => groupKey(a.Uri)) .OrderByDescending(g => g.Count()) @@ -176,18 +181,30 @@ public void DistributeByGroupAffinity(string scheme, Func groupKey, var members = group.ToList(); // Candidate nodes for the whole group: nodes capable of running every member (all nodes when - // capabilities are homogeneous). + // capabilities are homogeneous) — plus any node that was already running part of the group + // when the grid was assembled. The grandfathering mirrors the even paths, which leave running + // agents in place regardless of declared capabilities: a node's capability snapshot is + // persisted once at node startup, so a node that started before (say) a tenant database was + // provisioned never declares that database's agents even though it is happily running them. var candidates = sameCapabilities ? nodes - : nodes.Where(n => members.All(m => m.CandidateNodes.Contains(n))).ToList(); + : nodes.Where(n => members.All(m => m.CandidateNodes.Contains(n)) + || members.Any(m => m.OriginalNode == n)).ToList(); if (candidates.Count == 0) { // No single node can host the whole group — mirror the blue/green even path's per-agent - // behavior: each member goes to its least-loaded capable node, and a member no node - // declares a capability for is simply left alone (candidate == null there too). + // behavior: an already-running member stays where it is (minimal disruption), an + // unassigned member goes to its least-loaded capable node, and a member no node declares + // a capability for is simply left alone (candidate == null there too). foreach (var member in members) { + if (member.AssignedNode != null) + { + load[member.AssignedNode] = load.GetValueOrDefault(member.AssignedNode) + 1; + continue; + } + var candidate = member.CandidateNodes .OrderBy(n => load.GetValueOrDefault(n)) .ThenBy(n => n.IsLeader) @@ -204,7 +221,25 @@ public void DistributeByGroupAffinity(string scheme, Func groupKey, continue; } - // The least-loaded candidate hosts the whole group (tie-breaks: non-leader first, then node id). + // Minimal disruption, mirroring DistributeEvenly: the node already running the WHOLE group + // keeps it as long as that doesn't push the node past the ceiling. Without this, every + // evaluation reshuffles groups from scratch and a node whose stale capability snapshot keeps + // it out of the capability candidates can be starved permanently across evaluations. + var incumbent = members[0].AssignedNode; + if (incumbent != null && members.Any(m => m.AssignedNode != incumbent)) + { + incumbent = null; + } + + if (incumbent != null && candidates.Contains(incumbent) && + load[incumbent] + members.Count <= maximum) + { + load[incumbent] += members.Count; + continue; + } + + // Otherwise the least-loaded candidate hosts the whole group (tie-breaks: non-leader first, + // then node id). var node = candidates .OrderBy(n => load[n]) .ThenBy(n => n.IsLeader) From 85af35a6c5d0603353dacabca4983dec1e55c2db Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 6 Jul 2026 14:36:50 -0500 Subject: [PATCH 18/18] Consume JasperFx 2.20.0 + Marten 9.13.0-alpha.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JasperFx 2.20.0 (jasperfx#487): IEventStore.DistributesAgentsPerTenant + hardened per-tenant StartAgentAsync. Marten 9.13.0-alpha.2 (marten#4856): DocumentStore reports DistributesAgentsPerTenant for sharded + tenant-partitioned stores — required by the two end-to-end distribution tests. (alpha.1 was a burned version: nuget.org serves stale 2026-06-27 bits for it, without the override.) Marten 9.13 also moved the storage-operation contract (marten#4820/#4821/ #4861 storage-dialect extraction): ConfigureCommand now takes Weasel.Postgresql.ICommandBuilder + Weasel.Storage.IStorageSession, and NoDataReturnedCall lives in Weasel.Storage — both envelope operations adapted. Co-Authored-By: Claude Fable 5 --- Directory.Packages.props | 6 +++--- .../Persistence/Operations/StoreIncomingEnvelope.cs | 3 ++- .../Persistence/Operations/StoreOutgoingEnvelope.cs | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index bffc19283..f20df00cc 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -35,15 +35,15 @@ - - + + - + diff --git a/src/Persistence/Wolverine.Marten/Persistence/Operations/StoreIncomingEnvelope.cs b/src/Persistence/Wolverine.Marten/Persistence/Operations/StoreIncomingEnvelope.cs index 90b8ec53f..16248c1d5 100644 --- a/src/Persistence/Wolverine.Marten/Persistence/Operations/StoreIncomingEnvelope.cs +++ b/src/Persistence/Wolverine.Marten/Persistence/Operations/StoreIncomingEnvelope.cs @@ -4,6 +4,7 @@ using Marten.Services; using Weasel.Core; using Weasel.Postgresql; +using Weasel.Storage; using Wolverine.RDBMS; using Wolverine.Runtime.Serialization; @@ -21,7 +22,7 @@ public StoreIncomingEnvelope(string incomingTable, Envelope envelope) public Envelope Envelope { get; } - public void ConfigureCommand(ICommandBuilder builder, IMartenSession session) + public void ConfigureCommand(Weasel.Postgresql.ICommandBuilder builder, IStorageSession session) { builder.Append( $"insert into {_incomingTable} ({DatabaseConstants.IncomingFields}) values ("); diff --git a/src/Persistence/Wolverine.Marten/Persistence/Operations/StoreOutgoingEnvelope.cs b/src/Persistence/Wolverine.Marten/Persistence/Operations/StoreOutgoingEnvelope.cs index 37d0d6747..104f24405 100644 --- a/src/Persistence/Wolverine.Marten/Persistence/Operations/StoreOutgoingEnvelope.cs +++ b/src/Persistence/Wolverine.Marten/Persistence/Operations/StoreOutgoingEnvelope.cs @@ -4,6 +4,7 @@ using Marten.Services; using Weasel.Core; using Weasel.Postgresql; +using Weasel.Storage; using Wolverine.RDBMS; using Wolverine.Runtime.Serialization; @@ -23,7 +24,7 @@ public StoreOutgoingEnvelope(string outgoingTable, Envelope envelope, int ownerI public Envelope Envelope { get; } - public void ConfigureCommand(ICommandBuilder builder, IMartenSession session) + public void ConfigureCommand(Weasel.Postgresql.ICommandBuilder builder, IStorageSession session) { builder.Append( $"insert into {_outgoingTable} ({DatabaseConstants.OutgoingFields}) values (");