From a1eff41a19a976ab115eb70bead1306609bde896 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Thu, 6 Aug 2026 05:43:51 -0500 Subject: [PATCH] fix(mysql): give each tenant store its own database as schema (GH-3860) A MySQL schema IS a database. PostgreSQL and SQL Server nest a schema inside the database named by the connection string, so a single configured envelope storage schema name is harmless there -- isolation comes from the connection string. MySQL has no such nesting, so giving every tenant store the same SchemaName discarded the tenant's own database entirely: MySqlTenantedMessageStore.buildTenantStoreForConnectionString/DataSource SchemaName = _persistence.EnvelopeStorageSchemaName // same for all Every tenant therefore shared one physical set of tables -- incoming, outgoing, dead letters, nodes and the saga tables -- with no isolation at all. Measured against the existing static_multi_tenancy suite, which passes: all 8 tables lived in the one configured schema and tenant_db1/2/3 were completely empty. Nothing errored; a host configured this way looked correct. A tenant's database is now its schema. The main store is deliberately left alone: it already has a database of its own, and relocating its tables would be a far larger change than the isolation bug requires. MIGRATION: an existing multi-tenanted MySQL deployment has all of its tenant envelope rows in the shared schema. After upgrading, each tenant reads from its own database instead, so any in-flight envelopes still sitting in the old shared tables need to be drained or copied across before the upgrade. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JKfy5EzLX1i149gjUb3Tfg --- ...tenant_stores_are_isolated_per_database.cs | 223 ++++++++++++++++++ .../MySqlTenantedMessageStore.cs | 20 +- 2 files changed, 241 insertions(+), 2 deletions(-) create mode 100644 src/Persistence/MySql/MySqlTests/MultiTenancy/tenant_stores_are_isolated_per_database.cs diff --git a/src/Persistence/MySql/MySqlTests/MultiTenancy/tenant_stores_are_isolated_per_database.cs b/src/Persistence/MySql/MySqlTests/MultiTenancy/tenant_stores_are_isolated_per_database.cs new file mode 100644 index 000000000..0ce1cf102 --- /dev/null +++ b/src/Persistence/MySql/MySqlTests/MultiTenancy/tenant_stores_are_isolated_per_database.cs @@ -0,0 +1,223 @@ +using IntegrationTests; +using JasperFx.Core; +using JasperFx.Resources; +using Microsoft.Extensions.DependencyInjection; +using MySqlConnector; +using Shouldly; +using Wolverine; +using Wolverine.ComplianceTests; +using Wolverine.MySql; +using Wolverine.Persistence.Durability; +using Wolverine.RDBMS; +using Wolverine.Runtime; +using Wolverine.Tracking; +using Xunit; + +namespace MySqlTests.MultiTenancy; + +/// +/// GH-3860. A MySQL schema IS a database, so giving every tenant store the one configured envelope +/// storage schema name discarded the tenant's own database and collapsed all of them onto a single +/// physical set of tables: wolverine_incoming_envelopes, wolverine_outgoing_envelopes, +/// the node and dead letter tables, and the saga tables, all shared across tenants with no isolation +/// whatsoever. PostgreSQL is immune because its schema nests *inside* the tenant database. +/// +/// The pre-existing static_multi_tenancy suite could not catch this: it only asserts that the tenancy +/// source resolves the expected connection strings, never that anything is stored separately. +/// +[Collection("mysql")] +public class tenant_stores_are_isolated_per_database : MySqlMultiTenancyContext +{ + private const string SchemaName = "tenant_store_isolation"; + + protected override void configureWolverine(WolverineOptions opts) + { + opts.PersistMessagesWithMySql(Servers.MySqlConnectionString, SchemaName) + .RegisterStaticTenants(tenants => + { + tenants.Register("red", tenant1ConnectionString); + tenants.Register("blue", tenant2ConnectionString); + tenants.Register("green", tenant3ConnectionString); + }); + + opts.Services.AddResourceSetupOnStartup(); + } + + /// + /// The tenant databases are shared with every other suite in this collection and survive between runs, + /// so start from a known-empty inbox. Without this the row counts below accumulate and the test only + /// passes the first time it is ever run against a given MySQL instance. + /// + protected override async Task onStartup() + { + foreach (var connectionString in + new[] { tenant1ConnectionString, tenant2ConnectionString, tenant3ConnectionString }) + { + await using var conn = new MySqlConnection(connectionString); + await conn.OpenAsync(); + try + { + await using var cmd = conn.CreateCommand(); + cmd.CommandText = $"delete from {DatabaseConstants.IncomingTable}"; + await cmd.ExecuteNonQueryAsync(); + } + catch (MySqlException) + { + // Nothing provisioned in this database yet, nothing to clean + } + finally + { + await conn.CloseAsync(); + } + } + } + + private async Task storeForAsync(string tenantId) + { + var stores = (MultiTenantedMessageStore)theHost.GetRuntime().Storage; + return await stores.GetDatabaseAsync(tenantId); + } + + /// + /// The structural claim: every tenant database physically owns the envelope tables. Before the fix + /// the tenant databases were completely empty. + /// + [Theory] + [InlineData("red", "tenant_db1")] + [InlineData("blue", "tenant_db2")] + [InlineData("green", "tenant_db3")] + public async Task each_tenant_database_owns_its_own_envelope_tables(string tenantId, string expectedDatabase) + { + // Materialize and migrate the tenant store + await storeForAsync(tenantId); + + var connectionString = connectionStringFor(expectedDatabase); + + foreach (var table in new[] { DatabaseConstants.IncomingTable, DatabaseConstants.OutgoingTable }) + { + (await tableExistsAsync(connectionString, expectedDatabase, table)) + .ShouldBeTrue($"Expected {expectedDatabase}.{table} to exist"); + } + } + + /// + /// And the tenant store really is pointed at that database rather than the shared configured schema. + /// + [Fact] + public async Task tenant_store_schema_is_the_tenants_own_database() + { + var red = (IMessageDatabase)await storeForAsync("red"); + var blue = (IMessageDatabase)await storeForAsync("blue"); + + red.SchemaName.ShouldBe("tenant_db1"); + blue.SchemaName.ShouldBe("tenant_db2"); + + red.SchemaName.ShouldNotBe(blue.SchemaName); + red.SchemaName.ShouldNotBe(SchemaName); + } + + /// + /// The behavioural claim: a tenant's envelope is visible in that tenant's database and nowhere else. + /// This is the one that actually matters -- shared tables meant cross-tenant reads. + /// + [Fact] + public async Task an_envelope_stored_for_one_tenant_is_invisible_to_the_others() + { + var red = await storeForAsync("red"); + + // Materialize the other tenants too, so their tables exist and a "no change" reading below is a + // real measurement rather than a missing table. + await storeForAsync("blue"); + await storeForAsync("green"); + + // These databases are shared with every other suite in the collection, so measure the DELTA this + // store causes rather than absolute counts. + var before = await countsByDatabaseAsync(); + + var envelope = ObjectMother.Envelope(); + envelope.Status = EnvelopeStatus.Incoming; + envelope.OwnerId = 0; + + await red.Inbox.StoreIncomingAsync(envelope); + + var after = await countsByDatabaseAsync(); + + (after["tenant_db1"] - before["tenant_db1"]).ShouldBe(1); + + foreach (var database in new[] { "tenant_db2", "tenant_db3" }) + { + (after[database] - before[database]) + .ShouldBe(0, $"{database} must not receive another tenant's inbox row"); + } + + (after[SchemaName] - before[SchemaName]) + .ShouldBe(0, "The shared configured schema must not be collecting tenant rows"); + } + + private async Task> countsByDatabaseAsync() + { + return new Dictionary + { + ["tenant_db1"] = await incomingCountAsync(tenant1ConnectionString), + ["tenant_db2"] = await incomingCountAsync(tenant2ConnectionString), + ["tenant_db3"] = await incomingCountAsync(tenant3ConnectionString), + [SchemaName] = await incomingCountAsync(Servers.MySqlConnectionString, SchemaName) + }; + } + + private string connectionStringFor(string database) + { + return database switch + { + "tenant_db1" => tenant1ConnectionString, + "tenant_db2" => tenant2ConnectionString, + "tenant_db3" => tenant3ConnectionString, + _ => throw new ArgumentOutOfRangeException(nameof(database)) + }; + } + + private static async Task tableExistsAsync(string connectionString, string database, string tableName) + { + await using var conn = new MySqlConnection(connectionString); + await conn.OpenAsync(); + try + { + await using var cmd = conn.CreateCommand(); + cmd.CommandText = + "select count(*) from information_schema.tables where table_schema = @schema and table_name = @table"; + cmd.Parameters.AddWithValue("schema", database); + cmd.Parameters.AddWithValue("table", tableName); + + return Convert.ToInt64(await cmd.ExecuteScalarAsync()) > 0; + } + finally + { + await conn.CloseAsync(); + } + } + + private static async Task incomingCountAsync(string connectionString, string? schema = null) + { + await using var conn = new MySqlConnection(connectionString); + await conn.OpenAsync(); + try + { + var table = schema.IsEmpty() + ? DatabaseConstants.IncomingTable + : $"{schema}.{DatabaseConstants.IncomingTable}"; + + await using var cmd = conn.CreateCommand(); + cmd.CommandText = $"select count(*) from {table}"; + return Convert.ToInt64(await cmd.ExecuteScalarAsync()); + } + catch (MySqlException) + { + // The table not existing at all is the strongest possible form of "no rows here" + return 0; + } + finally + { + await conn.CloseAsync(); + } + } +} diff --git a/src/Persistence/MySql/Wolverine.MySql/MySqlTenantedMessageStore.cs b/src/Persistence/MySql/Wolverine.MySql/MySqlTenantedMessageStore.cs index d635231a1..627fea2ff 100644 --- a/src/Persistence/MySql/Wolverine.MySql/MySqlTenantedMessageStore.cs +++ b/src/Persistence/MySql/Wolverine.MySql/MySqlTenantedMessageStore.cs @@ -60,6 +60,22 @@ public async ValueTask FindAsync(string tenantId) return store; } + /// + /// GH-3860. A MySQL schema IS a database, so the single configured envelope storage schema name that + /// works for PostgreSQL and SQL Server — where a schema is a namespace *inside* the database named by + /// the connection string — discards the tenant's own database entirely and collapses every tenant onto + /// one physical set of tables. A tenant's database is therefore its schema. Falls back to the configured + /// name only when the connection string names no database at all, which cannot address a tenant anyway. + /// + private string schemaNameFor(string? connectionString) + { + if (connectionString.IsEmpty()) return _persistence.EnvelopeStorageSchemaName; + + var database = new MySqlConnectionStringBuilder(connectionString).Database; + + return database.IsEmpty() ? _persistence.EnvelopeStorageSchemaName : database; + } + private MySqlMessageStore buildTenantStoreForConnectionString(string connectionString) { var dataSource = MySqlDataSourceFactory.Create(connectionString); @@ -70,7 +86,7 @@ private MySqlMessageStore buildTenantStoreForConnectionString(string connectionS ConnectionString = connectionString, Role = MessageStoreRole.Tenant, ScheduledJobLockId = _persistence.ScheduledJobLockId, - SchemaName = _persistence.EnvelopeStorageSchemaName + SchemaName = schemaNameFor(connectionString) }; var store = new MySqlMessageStore(settings, _runtime.Options.Durability, dataSource, @@ -87,7 +103,7 @@ private MySqlMessageStore buildTenantStoreForDataSource(MySqlDataSource source) DataSource = source, Role = MessageStoreRole.Tenant, ScheduledJobLockId = _persistence.ScheduledJobLockId, - SchemaName = _persistence.EnvelopeStorageSchemaName + SchemaName = schemaNameFor(source.ConnectionString) }; var store = new MySqlMessageStore(settings, _runtime.Options.Durability, source,