diff --git a/Directory.Packages.props b/Directory.Packages.props index 7f7ea9b4a6..e8168eb525 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -87,13 +87,13 @@ IEventStore.DistributesAgentsPerTenant into BuildDistributor (ProjectionCoordinator). Also DaemonMode.ExternallyManaged (jasperfx#490, wolverine#3290 — external hosts run projections; the store hosts no coordination and must not warn). --> - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/src/TenantPartitionedEventsTests/Daemon/dynamic_tenant_lifecycle_during_continuous_daemon.cs b/src/TenantPartitionedEventsTests/Daemon/dynamic_tenant_lifecycle_during_continuous_daemon.cs index 6f775e5702..e68f898ecf 100644 --- a/src/TenantPartitionedEventsTests/Daemon/dynamic_tenant_lifecycle_during_continuous_daemon.cs +++ b/src/TenantPartitionedEventsTests/Daemon/dynamic_tenant_lifecycle_during_continuous_daemon.cs @@ -50,10 +50,10 @@ namespace TenantPartitionedEventsTests.Daemon; /// /// This test previously pinned the pre-#491 behavior (no mid-run discovery, lingering agent after /// removal) and was flipped when the fix landed. LeadershipPollingTime is tuned down to 250ms so -/// convergence is fast and deterministic instead of sleeping the default window. NOTE the cadence -/// caveat documented at Phase 1: the per-tenant high-water poll currently rides the GLOBAL -/// high-water cadence, so the mid-run tenant is seeded above the store's global mark to converge -/// deterministically on either side of the discovery-vs-append race. +/// convergence is fast and deterministic instead of sleeping the default window. Phase 1 also +/// pins the jasperfx#492 fix (2.21.1): the mid-run tenant is seeded BELOW the store's global mark, +/// which converges only because the per-tenant high-water poll now runs on the daemon's own timer +/// cadence rather than solely on store-global mark changes. /// /// public partial class dynamic_tenant_lifecycle_during_continuous_daemon @@ -140,30 +140,26 @@ await WaitForProgressionAsync(rows => }.OrderBy(x => x)); // ---- Phase 1: a NEW tenant joins while the coordinator keeps running ---- - // Tenant C is deliberately seeded ABOVE the store's current global high water (a is at - // 5, so C gets 7 events in its own overlapping sequence). KNOWN CADENCE GAP (jasperfx, - // observed against JasperFx.Events 2.21.0): the vectorized per-tenant high-water poll - // rides the GLOBAL high-water cadence — JasperFxAsyncDaemon only calls - // pollTenantHighWaterAsync when the store-global HighWaterMark shard state fires, which - // under overlapping per-tenant sequences only happens when max(seq_id) over the whole - // table moves. If the coordinator's discovery wins the race against this append (it - // polls every 250ms here), C's agent is primed at ceiling 0, and events that stay - // BELOW the global mark would never be routed to it — the agent stalls at 0 - // indefinitely. Seeding C past the global mark forces a global high-water change, which - // triggers the per-tenant poll and converges C on EITHER side of the race. When the - // cadence gap is fixed upstream (per-tenant polls on the timer, not on global-mark - // change), C's height can go back below the global mark. + // Tenant C is deliberately seeded BELOW the store's current global high water (A is at + // 5, C gets only 3 events in its own overlapping sequence). This actively pins the + // jasperfx#492 fix (JasperFx 2.21.1 / jasperfx#493): the vectorized per-tenant + // high-water poll now runs on the daemon's SlowPollingTime timer, not solely on + // store-global mark changes — so even though C's appends never move max(seq_id) over + // the whole table, C's agent still learns its per-tenant ceiling and converges. Against + // 2.21.0 this exact shape stalled C's agent at 0 indefinitely whenever the + // coordinator's discovery won the race against the append (the test previously worked + // around it by seeding C ABOVE the global mark to force a global tick). const string tenantC = "dynlife_c"; await store.Advanced.AddMartenManagedTenantsAsync(CancellationToken.None, tenantC); - var streamC = await AppendStreamAsync(store, tenantC, 6); // 7 events > global HW of 5 + var streamC = await AppendStreamAsync(store, tenantC, 2); // 3 events < global HW of 5 // FLIPPED (was the pre-#491 pin): the running coordinator DOES discover the new tenant // on its own — the next BuildDistributionAsync re-expands from mt_tenant_partitions and // starts tenant C's agent, which catches up to the tenant's own height. No explicit // per-tenant StartAgentAsync (wolverine#3280) workaround, no restart. await WaitForProgressionAsync(rows => - SeqOf(rows, $"{DynLifeProjection.ProjectionName}:All:{tenantC}") >= 7 && - SeqOf(rows, $"HighWaterMark:{tenantC}") >= 7, + SeqOf(rows, $"{DynLifeProjection.ProjectionName}:All:{tenantC}") >= 3 && + SeqOf(rows, $"HighWaterMark:{tenantC}") >= 3, 30.Seconds(), "tenant C converges via the coordinator's own re-enumeration, no explicit agent start"); @@ -175,7 +171,7 @@ await WaitForProgressionAsync(rows => { var doc = await query.LoadAsync(streamC); doc.ShouldNotBeNull(); - doc.Count.ShouldBe(6); + doc.Count.ShouldBe(2); } // The original tenants were untouched by the dynamic add. @@ -216,7 +212,7 @@ await WaitForConditionAsync( // Survivors are unaffected by the removal. SeqOf(finalRows, $"{DynLifeProjection.ProjectionName}:All:{tenantA}").ShouldBe(5); - SeqOf(finalRows, $"{DynLifeProjection.ProjectionName}:All:{tenantC}").ShouldBe(7); + SeqOf(finalRows, $"{DynLifeProjection.ProjectionName}:All:{tenantC}").ShouldBe(3); } finally { diff --git a/src/TenantPartitionedEventsTests/Sharded/dynamic_tenant_lifecycle_on_shard_during_daemon.cs b/src/TenantPartitionedEventsTests/Sharded/dynamic_tenant_lifecycle_on_shard_during_daemon.cs index 2334b44326..d1b514d610 100644 --- a/src/TenantPartitionedEventsTests/Sharded/dynamic_tenant_lifecycle_on_shard_during_daemon.cs +++ b/src/TenantPartitionedEventsTests/Sharded/dynamic_tenant_lifecycle_on_shard_during_daemon.cs @@ -169,20 +169,18 @@ await WaitForProgressionAsync(shardConnStr, rows => $"AddTenantToShardAsync must provision mt_events_sequence_{tenant2} on shard {shard}"); // ... so appending for the new tenant works right away (Quick append would raise MT002 - // against an unregistered tenant partition). Tenant 2 is deliberately seeded ABOVE the - // shard's current global high water (tenant 1 is at 3, so tenant 2 gets 4 events in its - // own overlapping sequence): the vectorized per-tenant high-water poll rides the GLOBAL - // high-water cadence (fires only when max(seq_id) over the shard's whole events table - // moves), so if the coordinator's discovery wins the race against this append and - // primes tenant 2's agent at ceiling 0, events staying below the shard's global mark - // would never be routed to it. See the single-database flavor - // (dynamic_tenant_lifecycle_during_continuous_daemon) for the full cadence-gap notes. + // against an unregistered tenant partition). Tenant 2 is deliberately seeded BELOW the + // shard's current global high water (tenant 1 is at 3, tenant 2 gets only 2 events in + // its own overlapping sequence) — pinning the jasperfx#492 fix (2.21.1): the per-tenant + // high-water poll now runs on the daemon's own timer cadence, so tenant 2 converges + // even though its appends never move max(seq_id) over the shard's whole events table. + // See the single-database flavor (dynamic_tenant_lifecycle_during_continuous_daemon) + // for the full history of the cadence gap this used to work around. var stream2 = Guid.NewGuid(); await using (var session = _store.LightweightSession(tenant2)) { session.Events.StartStream(stream2, - new ShardedDaemonEvent("t2-1"), new ShardedDaemonEvent("t2-2"), - new ShardedDaemonEvent("t2-3"), new ShardedDaemonEvent("t2-4")); + new ShardedDaemonEvent("t2-1"), new ShardedDaemonEvent("t2-2")); await session.SaveChangesAsync(); } @@ -191,8 +189,8 @@ await WaitForProgressionAsync(shardConnStr, rows => // the shard database's set re-expands from its own tenants.mt_tenant_partitions // registry and tenant 2's agent starts and catches up. No StartAgentAsync, no restart. var rows = await WaitForProgressionAsync(shardConnStr, r => - SeqOf(r, $"{ShardedDaemonProjection.ProjectionName}:All:{tenant2}") >= 4 && - SeqOf(r, $"HighWaterMark:{tenant2}") >= 4, + SeqOf(r, $"{ShardedDaemonProjection.ProjectionName}:All:{tenant2}") >= 2 && + SeqOf(r, $"HighWaterMark:{tenant2}") >= 2, 30.Seconds(), "new tenant's per-tenant progression + high-water rows appear on shard A via the " + "coordinator's own re-enumeration"); @@ -217,7 +215,7 @@ await WaitForProgressionAsync(shardConnStr, rows => { await using var query = _store.QuerySession(tenant2); doc2 = await query.LoadAsync(stream2); - if (doc2 is { EventCount: 4 }) + if (doc2 is { EventCount: 2 }) { break; } @@ -226,7 +224,7 @@ await WaitForProgressionAsync(shardConnStr, rows => } doc2.ShouldNotBeNull("tenant 2's projection doc must materialize on shard A"); - doc2!.EventCount.ShouldBe(4); + doc2!.EventCount.ShouldBe(2); // And the original tenant is untouched by the dynamic add — still at its own height. SeqOf(rows, $"{ShardedDaemonProjection.ProjectionName}:All:{tenant1}").ShouldBe(3);