From bc707a9cfd146552a2b569997fb335b631b0b84e Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Tue, 2 Jun 2026 19:34:25 -0500 Subject: [PATCH] Fix #4606: 42P16 inherited-FK on eager-apply + sharded per-tenant events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #4598. The eager `ApplyAllConfiguredChangesToDatabaseAsync()` startup pattern, combined with `MultiTenantedWithShardedDatabases` and `UseTenantPartitionedEvents`, made runtime-provisioned sharded tenants fail their first event append. The user-facing symptom was `23514: no partition of relation "mt_events" found for row`, with the actual root cause hidden behind Weasel's silently-logged `42P16: cannot drop inherited constraint "mt_events_tenant_id_stream_id_fkey"` in the partition-attach path. ## Root cause When `UseTenantPartitionedEvents` is on, BOTH `mt_streams` and `mt_events` are list-partitioned by `tenant_id`. Marten declared an explicit FK on the parent `mt_events` referencing `mt_streams(tenant_id, id)`. Eager schema apply + the first per-tenant partition (`mt_streams_alpha`) triggers Postgres to auto-propagate a partition-targeting FK onto `mt_events`: fkey_mt_events_stream_id_tenant_id → mt_streams (Marten-declared, local) mt_events_tenant_id_stream_id_fkey → mt_streams_alpha (PG-propagated, inherited) The next partition attach for `mt_events_alpha` runs Weasel's `additivelyMigrateTablesForNewPartitions`, which sees the auto-propagated FK as "extra" (not in Marten's declared set) and emits `ALTER TABLE mt_events DROP CONSTRAINT IF EXISTS mt_events_tenant_id_stream_id_fkey` — Postgres refuses with 42P16 because the constraint is inherited (`coninhcount = 1`). The error is logged via `NullLogger` and the partition attach is dropped on the floor; the next append fails with 23514 because the partition that was supposed to exist isn't there. ## Fix Skip the explicit `mt_events → mt_streams` FK declaration in `EventsTable.cs` when `UseTenantPartitionedEvents` is on. Without the parent FK, Postgres never auto-propagates the inherited variant, so Weasel's partition-attach has nothing to incorrectly drop and the attach succeeds. The non-partitioned and `UseArchivedStreamPartitioning` shapes keep their FK unchanged — only the per-tenant-partitioning combination hits this. ## Trade-off Database-level referential integrity between `mt_events` and `mt_streams` is lost in the `UseTenantPartitionedEvents` configuration. Marten's append path always inserts/updates the stream row before persisting events (application-level integrity preserved), so the only at-risk surface is external tooling that writes to `mt_events` directly without ensuring the stream exists in `mt_streams` — already outside Marten's contract. The alternative would be a Weasel-side fix to skip inherited FKs in the table-delta introspection. That's the architecturally cleaner fix but requires a coordinated Weasel release; this change unblocks the eager-apply pattern on the current Marten release line without that dependency. ## Tests `sharded_eager_apply_per_tenant_events_tests.eager_apply_then_AddTenantToShardAsync_then_append_succeeds` — the headline regression. Reproduces the exact eager-apply + runtime-provision + first-append shape from the #4606 report; would throw `23514` on master and now succeeds. Regression sweep on net9.0: - All sharded test suites (`sharded_tenancy_tests` + `sharded_tenancy_per_tenant_events_tests` + the new eager-apply test): 21/21 PASS. - DefaultTenancy `use_tenant_partitioned_events_*`: 32/33 PASS — the single fail is the same pre-existing `delete_projection_progress_with_tenant_id_drops_only_that_tenants_rows` partition-setup race that's unrelated to this work (also fails on master). - Conjoined + ForeignKey event-sourcing tests: 75/75 PASS. Closes #4606. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/Marten/Events/Schema/EventsTable.cs | 49 +++++-- ...ded_eager_apply_per_tenant_events_tests.cs | 131 ++++++++++++++++++ 2 files changed, 170 insertions(+), 10 deletions(-) create mode 100644 src/MultiTenancyTests/sharded_eager_apply_per_tenant_events_tests.cs diff --git a/src/Marten/Events/Schema/EventsTable.cs b/src/Marten/Events/Schema/EventsTable.cs index 52598492c1..41be1795c1 100644 --- a/src/Marten/Events/Schema/EventsTable.cs +++ b/src/Marten/Events/Schema/EventsTable.cs @@ -69,14 +69,40 @@ public EventsTable(EventGraph events): base(new PostgresqlObjectName(events.Data if (events.TenancyStyle == TenancyStyle.Conjoined) { + // #4606: under UseTenantPartitionedEvents, mt_events is itself a partitioned + // table (by tenant_id) and so is mt_streams. Declaring a parent-level FK from + // mt_events → mt_streams makes Postgres auto-propagate a partition-targeting + // FK (mt_events → mt_streams_) into mt_events as an INHERITED constraint + // the moment the first mt_streams_ partition is attached. The next + // mt_events_ partition-attach then trips + // `42P16: cannot drop inherited constraint` because Weasel's + // additivelyMigrateTablesForNewPartitions treats the auto-propagated FK as an + // "extra" (not in Marten's declared FK set) and tries to drop it, which + // Postgres refuses on an inherited constraint. The downstream symptom is + // 23514 on the first event append because the partition attach was silently + // swallowed by the migration's catch block. + // + // Skipping the explicit FK trades database-level referential integrity from + // mt_events to mt_streams for the per-tenant-partitioning combination + // working at all. Marten's append path always inserts/updates the stream row + // before persisting events, so application-level integrity is preserved; + // external tooling that writes mt_events directly without ensuring the + // stream exists in mt_streams is the only at-risk path, and that's already + // outside Marten's contract. The non-partitioned and archived-partitioning + // shapes keep their FK because they don't trigger the auto-propagation. + var skipMtEventsFkForTenantPartitioning = events.UseTenantPartitionedEvents; + if (events.UseArchivedStreamPartitioning) { - ForeignKeys.Add(new ForeignKey("fkey_mt_events_stream_id_tenant_id_is_archived") + if (!skipMtEventsFkForTenantPartitioning) { - ColumnNames = new[] { TenantIdColumn.Name, "stream_id", "is_archived" }, - LinkedNames = new[] { TenantIdColumn.Name, "id", "is_archived" }, - LinkedTable = new PostgresqlObjectName(events.DatabaseSchemaName, "mt_streams") - }); + ForeignKeys.Add(new ForeignKey("fkey_mt_events_stream_id_tenant_id_is_archived") + { + ColumnNames = new[] { TenantIdColumn.Name, "stream_id", "is_archived" }, + LinkedNames = new[] { TenantIdColumn.Name, "id", "is_archived" }, + LinkedTable = new PostgresqlObjectName(events.DatabaseSchemaName, "mt_streams") + }); + } Indexes.Add(new IndexDefinition("pk_mt_events_stream_and_version") { @@ -85,12 +111,15 @@ public EventsTable(EventGraph events): base(new PostgresqlObjectName(events.Data } else { - ForeignKeys.Add(new ForeignKey("fkey_mt_events_stream_id_tenant_id") + if (!skipMtEventsFkForTenantPartitioning) { - ColumnNames = new[] { TenantIdColumn.Name, "stream_id" }, - LinkedNames = new[] { TenantIdColumn.Name, "id" }, - LinkedTable = new PostgresqlObjectName(events.DatabaseSchemaName, "mt_streams") - }); + ForeignKeys.Add(new ForeignKey("fkey_mt_events_stream_id_tenant_id") + { + ColumnNames = new[] { TenantIdColumn.Name, "stream_id" }, + LinkedNames = new[] { TenantIdColumn.Name, "id" }, + LinkedTable = new PostgresqlObjectName(events.DatabaseSchemaName, "mt_streams") + }); + } Indexes.Add(new IndexDefinition("pk_mt_events_stream_and_version") { diff --git a/src/MultiTenancyTests/sharded_eager_apply_per_tenant_events_tests.cs b/src/MultiTenancyTests/sharded_eager_apply_per_tenant_events_tests.cs new file mode 100644 index 0000000000..1224c5566b --- /dev/null +++ b/src/MultiTenancyTests/sharded_eager_apply_per_tenant_events_tests.cs @@ -0,0 +1,131 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using JasperFx; +using JasperFx.Events; +using JasperFx.MultiTenancy; +using Marten; +using Marten.Events; +using Marten.Storage; +using Marten.Testing.Documents; +using Marten.Testing.Harness; +using Npgsql; +using Shouldly; +using Weasel.Core; +using Weasel.Postgresql; +using Xunit; + +namespace MultiTenancyTests; + +/// +/// #4606 regression — eager schema apply on a sharded + per-tenant-events store, +/// followed by runtime tenant provisioning, must succeed (no 42P16 on the partition +/// attach's FK drop). +/// +/// +/// The shape this exercises is the production startup pattern: +/// ApplyAllConfiguredChangesToDatabaseAsync() at boot → tenant arrives later +/// → AddTenantToShardAsync(tenantId) → first append. +/// In 9.4.x the FK on the parent mt_events table is inherited the moment +/// the first partition is attached, and Weasel's partition-attach drop-and-recreate +/// of that FK trips Postgres' 42P16 (cannot drop inherited constraint). #4598's +/// tests deliberately exercised the lazy-apply flow to dodge this; #4606 is the +/// eager-apply variant. +/// +/// +[Collection("sharded-tenancy")] +public class sharded_eager_apply_per_tenant_events_tests : IAsyncLifetime +{ + private readonly ShardedTenancyFixture _fixture; + private IDocumentStore _store = null!; + + public sharded_eager_apply_per_tenant_events_tests(ShardedTenancyFixture fixture) + { + _fixture = fixture; + } + + public async Task InitializeAsync() + { + await using var conn = new NpgsqlConnection(ConnectionSource.ConnectionString); + await conn.OpenAsync(); + try { await conn.DropSchemaAsync("sharded"); } catch { } + + foreach (var connStr in _fixture.ConnectionStrings.Values) + { + await using var tenantConn = new NpgsqlConnection(connStr); + await tenantConn.OpenAsync(); + try { await tenantConn.DropSchemaAsync("tenants"); } catch { } + await ShardedTenancyFixture.cleanMartenObjectsInPublicSchema(tenantConn); + } + } + + public Task DisposeAsync() + { + _store?.Dispose(); + return Task.CompletedTask; + } + + private IDocumentStore CreateStore() + { + _store = DocumentStore.For(opts => + { + opts.MultiTenantedWithShardedDatabases(x => + { + x.ConnectionString = ConnectionSource.ConnectionString; + x.SchemaName = "sharded"; + x.PartitionSchemaName = "tenants"; + foreach (var (dbName, connStr) in _fixture.ConnectionStrings) + { + x.AddDatabase(dbName, connStr); + } + x.UseSmallestDatabaseAssignment(); + }); + + opts.AutoCreateSchemaObjects = AutoCreate.All; + opts.Events.TenancyStyle = TenancyStyle.Conjoined; + opts.Events.AppendMode = EventAppendMode.QuickWithServerTimestamps; + opts.Events.UseTenantPartitionedEvents = true; + opts.Events.AddEventType(); + }); + + return _store; + } + + [Fact] + public async Task eager_apply_then_AddTenantToShardAsync_then_append_succeeds() + { + CreateStore(); + + // The eager-apply pattern: create the parent partitioned schema on every + // shard at startup time, BEFORE any tenant is provisioned. This is the + // startup-migrate deployment shape that #4606 needs to support. + var databases = await _store.Options.Tenancy.BuildDatabases(); + foreach (var db in databases.OfType()) + { + await db.ApplyAllConfiguredChangesToDatabaseAsync(); + } + + // Runtime tenant provisioning hits the partition-attach path against an + // already-created parent mt_events. Master fails here with + // 42P16 cannot drop inherited constraint "mt_events_tenant_id_stream_id_fkey". + var dbId = await _store.Advanced.AddTenantToShardAsync("alpha", CancellationToken.None); + dbId.ShouldNotBeNull(); + + // The actual user-facing failure surface: first append for the new tenant. + // On master this throws either the same 42P16 (the attach was silently + // swallowed and the partition is missing) or a downstream 23514 (no + // partition of relation mt_events found for row). + await using var session = _store.LightweightSession("alpha"); + session.Events.StartStream(Guid.NewGuid(), new ShardedTestEvent { Value = "eager-apply-then-tenant" }); + await session.SaveChangesAsync(); + + // Sanity: the partition lives in the assigned shard. + await using var conn = new NpgsqlConnection(_fixture.ConnectionStrings[dbId]); + await conn.OpenAsync(); + var tables = await conn.ExistingTablesAsync(); + tables.Any(t => t.Name == "mt_events_alpha").ShouldBeTrue( + $"mt_events_alpha partition must exist after eager-apply + runtime tenant. Tables: {string.Join(", ", tables.Select(t => t.QualifiedName))}"); + } +}