From c092552512d43771d40308ee66b7edb5118a2813 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Tue, 10 Feb 2026 18:55:12 -0600 Subject: [PATCH 1/2] Add Wolverine.Oracle persistence provider with full test coverage Implements Oracle database support for Wolverine including message storage (inbox/outbox/dead letters), node agent persistence, saga support, a polling-based messaging transport, advisory locking, and multi-tenancy. All 53 tests pass against Oracle 23c Free. Key Oracle-specific adaptations: - Guid stored as RAW(16) with byte[] conversion on read/write - Named parameter binding (BindByName=true) required for all commands - MERGE INTO...USING DUAL for upserts instead of INSERT ON CONFLICT - SELECT FOR UPDATE SKIP LOCKED for queue polling - Row-level locking via wolverine_locks table for advisory locks - ORA-00054 retry logic for table teardown during active listeners Co-Authored-By: Claude Opus 4.6 --- build/build.cs | 1 + docker-compose.yml | 13 +- docker/oracle/init-wolverine-permissions.sql | 46 ++ .../OracleTests/Agents/node_persistence.cs | 47 ++ .../Oracle/OracleTests/GlobalUsings.cs | 1 + .../Oracle/OracleTests/NoParallelization.cs | 1 + .../Oracle/OracleTests/OracleContext.cs | 4 + .../OracleTests/OracleMessageStoreTests.cs | 101 ++++ .../Oracle/OracleTests/OracleTests.csproj | 34 ++ .../Sagas/saga_storage_operations.cs | 179 ++++++ .../Transport/basic_functionality.cs | 209 +++++++ .../Wolverine.Oracle/AssemblyAttributes.cs | 3 + .../Wolverine.Oracle/OracleAdvisoryLock.cs | 133 +++++ .../OracleBackedPersistence.cs | 291 ++++++++++ .../OracleConfigurationExtensions.cs | 89 +++ .../Wolverine.Oracle/OracleDataSource.cs | 37 ++ .../OracleMessageStore.Admin.cs | 259 +++++++++ .../OracleMessageStore.DeadLetters.cs | 233 ++++++++ .../OracleMessageStore.Incoming.cs | 345 ++++++++++++ .../OracleMessageStore.Outgoing.cs | 144 +++++ .../OracleMessageStore.Scheduled.cs | 79 +++ .../Wolverine.Oracle/OracleMessageStore.cs | 522 ++++++++++++++++++ .../Wolverine.Oracle/OracleNodePersistence.cs | 396 +++++++++++++ .../OracleTenantedMessageStore.cs | 123 +++++ .../Sagas/OracleSagaSchema.cs | 177 ++++++ .../Schema/DeadLettersTable.cs | 40 ++ .../Schema/IncomingEnvelopeTable.cs | 38 ++ .../Wolverine.Oracle/Schema/LockTable.cs | 15 + .../Schema/OutgoingEnvelopeTable.cs | 27 + .../Transport/IOracleQueueSender.cs | 8 + .../Transport/MultiTenantedQueueListener.cs | 112 ++++ .../Transport/MultiTenantedQueueSender.cs | 93 ++++ .../Transport/OracleListenerConfiguration.cs | 39 ++ .../Transport/OraclePersistenceExpression.cs | 52 ++ .../Wolverine.Oracle/Transport/OracleQueue.cs | 300 ++++++++++ .../Transport/OracleQueueListener.cs | 439 +++++++++++++++ .../Transport/OracleQueueSender.cs | 272 +++++++++ .../OracleSubscriberConfiguration.cs | 10 + .../Transport/OracleTransport.cs | 115 ++++ .../Wolverine.Oracle/Transport/QueueTable.cs | 18 + .../Transport/ScheduledMessageTable.cs | 24 + .../Util/OracleCommandExtensions.cs | 76 +++ .../Util/OracleEnvelopeReader.cs | 110 ++++ .../Wolverine.Oracle/Wolverine.Oracle.csproj | 25 + .../Wolverine.RDBMS/AssemblyAttributes.cs | 2 + src/Servers.cs | 3 + wolverine.sln | 30 + 47 files changed, 5314 insertions(+), 1 deletion(-) create mode 100644 docker/oracle/init-wolverine-permissions.sql create mode 100644 src/Persistence/Oracle/OracleTests/Agents/node_persistence.cs create mode 100644 src/Persistence/Oracle/OracleTests/GlobalUsings.cs create mode 100644 src/Persistence/Oracle/OracleTests/NoParallelization.cs create mode 100644 src/Persistence/Oracle/OracleTests/OracleContext.cs create mode 100644 src/Persistence/Oracle/OracleTests/OracleMessageStoreTests.cs create mode 100644 src/Persistence/Oracle/OracleTests/OracleTests.csproj create mode 100644 src/Persistence/Oracle/OracleTests/Sagas/saga_storage_operations.cs create mode 100644 src/Persistence/Oracle/OracleTests/Transport/basic_functionality.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/AssemblyAttributes.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/OracleAdvisoryLock.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/OracleBackedPersistence.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/OracleConfigurationExtensions.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/OracleDataSource.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Admin.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.DeadLetters.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Incoming.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Outgoing.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Scheduled.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/OracleNodePersistence.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/OracleTenantedMessageStore.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Sagas/OracleSagaSchema.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Schema/DeadLettersTable.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Schema/IncomingEnvelopeTable.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Schema/LockTable.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Schema/OutgoingEnvelopeTable.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Transport/IOracleQueueSender.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Transport/MultiTenantedQueueListener.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Transport/MultiTenantedQueueSender.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleListenerConfiguration.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Transport/OraclePersistenceExpression.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueue.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueueListener.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueueSender.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleSubscriberConfiguration.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleTransport.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Transport/QueueTable.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Transport/ScheduledMessageTable.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Util/OracleCommandExtensions.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Util/OracleEnvelopeReader.cs create mode 100644 src/Persistence/Oracle/Wolverine.Oracle/Wolverine.Oracle.csproj diff --git a/build/build.cs b/build/build.cs index 6e4f728a0..ad2a62435 100644 --- a/build/build.cs +++ b/build/build.cs @@ -334,6 +334,7 @@ class Build : NukeBuild Solution.Persistence.Wolverine_RavenDb, Solution.Persistence.Wolverine_SqlServer, Solution.Persistence.Wolverine_MySql, + Solution.Persistence.Wolverine_Oracle, Solution.Persistence.Wolverine_Sqlite, Solution.Extensions.Wolverine_FluentValidation, Solution.Extensions.Wolverine_MemoryPack, diff --git a/docker-compose.yml b/docker-compose.yml index 564db7f0f..548f10bfc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -107,4 +107,15 @@ services: - "3306:3306" environment: - "MYSQL_ROOT_PASSWORD=P@55w0rd" - - "MYSQL_DATABASE=wolverine" \ No newline at end of file + - "MYSQL_DATABASE=wolverine" + + oracle: + image: "gvenzl/oracle-free:23-slim" + ports: + - "1521:1521" + environment: + - "ORACLE_PASSWORD=P@55w0rd" + - "APP_USER=wolverine" + - "APP_USER_PASSWORD=wolverine" + volumes: + - ./docker/oracle:/container-entrypoint-initdb.d \ No newline at end of file diff --git a/docker/oracle/init-wolverine-permissions.sql b/docker/oracle/init-wolverine-permissions.sql new file mode 100644 index 000000000..3cc6b837e --- /dev/null +++ b/docker/oracle/init-wolverine-permissions.sql @@ -0,0 +1,46 @@ +-- Grant the wolverine user permissions to create and drop schemas (users in Oracle) +-- This script runs after the APP_USER is created by the container + +ALTER SESSION SET CONTAINER = FREEPDB1; + +-- Grant privileges to create and drop users (schemas) +GRANT CREATE USER TO wolverine; +GRANT DROP USER TO wolverine; +GRANT ALTER USER TO wolverine; + +-- Grant ability to create sessions and manage objects in other schemas +GRANT CREATE SESSION TO wolverine WITH ADMIN OPTION; +GRANT CREATE TABLE TO wolverine WITH ADMIN OPTION; +GRANT CREATE SEQUENCE TO wolverine WITH ADMIN OPTION; +GRANT CREATE PROCEDURE TO wolverine WITH ADMIN OPTION; +GRANT CREATE VIEW TO wolverine WITH ADMIN OPTION; + +-- Grant ability to create/drop/alter objects in ANY schema (needed for cross-schema testing) +GRANT CREATE ANY TABLE TO wolverine; +GRANT DROP ANY TABLE TO wolverine; +GRANT ALTER ANY TABLE TO wolverine; +GRANT SELECT ANY TABLE TO wolverine; +GRANT INSERT ANY TABLE TO wolverine; +GRANT UPDATE ANY TABLE TO wolverine; +GRANT DELETE ANY TABLE TO wolverine; +GRANT CREATE ANY INDEX TO wolverine; +GRANT DROP ANY INDEX TO wolverine; +GRANT CREATE ANY SEQUENCE TO wolverine; +GRANT DROP ANY SEQUENCE TO wolverine; +GRANT CREATE ANY PROCEDURE TO wolverine; +GRANT DROP ANY PROCEDURE TO wolverine; +GRANT EXECUTE ANY PROCEDURE TO wolverine; + +-- Grant unlimited tablespace so wolverine can allocate space to schemas it creates +GRANT UNLIMITED TABLESPACE TO wolverine WITH ADMIN OPTION; + +-- Grant ability to select from system views for schema introspection +GRANT SELECT ON sys.all_tables TO wolverine; +GRANT SELECT ON sys.all_tab_columns TO wolverine; +GRANT SELECT ON sys.all_constraints TO wolverine; +GRANT SELECT ON sys.all_cons_columns TO wolverine; +GRANT SELECT ON sys.all_indexes TO wolverine; +GRANT SELECT ON sys.all_ind_columns TO wolverine; +GRANT SELECT ON sys.all_sequences TO wolverine; +GRANT SELECT ON sys.all_users TO wolverine; +GRANT SELECT ON sys.all_objects TO wolverine; diff --git a/src/Persistence/Oracle/OracleTests/Agents/node_persistence.cs b/src/Persistence/Oracle/OracleTests/Agents/node_persistence.cs new file mode 100644 index 000000000..f7fb36346 --- /dev/null +++ b/src/Persistence/Oracle/OracleTests/Agents/node_persistence.cs @@ -0,0 +1,47 @@ +using IntegrationTests; +using Microsoft.Extensions.Logging.Abstractions; +using Oracle.ManagedDataAccess.Client; +using Weasel.Oracle; +using Wolverine; +using Wolverine.ComplianceTests; +using Wolverine.Oracle; +using Wolverine.Persistence.Durability; +using Wolverine.RDBMS; +using Wolverine.RDBMS.Sagas; + +namespace OracleTests.Agents; + +[Collection("oracle")] +public class node_persistence : NodePersistenceCompliance +{ + protected override async Task buildCleanMessageStore() + { + // Clean up Oracle schema objects + await using var conn = new OracleConnection(Servers.OracleConnectionString); + await conn.OpenAsync(); + try + { + // Drop and recreate the schema by clearing tables + // Oracle doesn't have DROP SCHEMA CASCADE like PostgreSQL + } + finally + { + await conn.CloseAsync(); + } + + var dataSource = new OracleDataSource(Servers.OracleConnectionString); + var settings = new DatabaseSettings + { + ConnectionString = Servers.OracleConnectionString, + SchemaName = "WOLVERINE", + Role = MessageStoreRole.Main + }; + + var database = new OracleMessageStore(settings, new DurabilitySettings(), dataSource, + NullLogger.Instance, Array.Empty()); + + await database.Admin.RebuildAsync(); + + return database; + } +} diff --git a/src/Persistence/Oracle/OracleTests/GlobalUsings.cs b/src/Persistence/Oracle/OracleTests/GlobalUsings.cs new file mode 100644 index 000000000..c802f4480 --- /dev/null +++ b/src/Persistence/Oracle/OracleTests/GlobalUsings.cs @@ -0,0 +1 @@ +global using Xunit; diff --git a/src/Persistence/Oracle/OracleTests/NoParallelization.cs b/src/Persistence/Oracle/OracleTests/NoParallelization.cs new file mode 100644 index 000000000..e5cc5d402 --- /dev/null +++ b/src/Persistence/Oracle/OracleTests/NoParallelization.cs @@ -0,0 +1 @@ +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/src/Persistence/Oracle/OracleTests/OracleContext.cs b/src/Persistence/Oracle/OracleTests/OracleContext.cs new file mode 100644 index 000000000..46789c623 --- /dev/null +++ b/src/Persistence/Oracle/OracleTests/OracleContext.cs @@ -0,0 +1,4 @@ +namespace OracleTests; + +[Collection("oracle")] +public abstract class OracleContext; diff --git a/src/Persistence/Oracle/OracleTests/OracleMessageStoreTests.cs b/src/Persistence/Oracle/OracleTests/OracleMessageStoreTests.cs new file mode 100644 index 000000000..c77652acf --- /dev/null +++ b/src/Persistence/Oracle/OracleTests/OracleMessageStoreTests.cs @@ -0,0 +1,101 @@ +using IntegrationTests; +using JasperFx.Core; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Wolverine; +using Wolverine.ComplianceTests; +using Wolverine.Oracle; +using Wolverine.Persistence.Durability; +using Wolverine.RDBMS; +using Wolverine.Transports.Tcp; + +namespace OracleTests; + +[Collection("oracle")] +public class OracleMessageStoreTests : MessageStoreCompliance +{ + public override async Task BuildCleanHost() + { + var dataSource = new OracleDataSource(Servers.OracleConnectionString); + var settings = new DatabaseSettings + { + SchemaName = "WOLVERINE", + CommandQueuesEnabled = true, + Role = MessageStoreRole.Main + }; + var durabilitySettings = new DurabilitySettings(); + var store = new OracleMessageStore(settings, durabilitySettings, dataSource, + NullLogger.Instance); + + await store.Admin.MigrateAsync(); + + var host = await Host.CreateDefaultBuilder() + .UseWolverine(opts => + { + opts.PersistMessagesWithOracle(Servers.OracleConnectionString, "WOLVERINE"); + opts.ListenAtPort(2345).UseDurableInbox(); + opts.Durability.Mode = DurabilityMode.Solo; + }).StartAsync(); + + var hostStore = host.Get(); + await hostStore.Admin.ClearAllAsync(); + + return host; + } + + [Fact] + public async Task can_persist_and_delete_outgoing_envelope() + { + var envelope = ObjectMother.Envelope(); + + await thePersistence.Outbox.StoreOutgoingAsync(envelope, 1); + + var counts = await thePersistence.Admin.FetchCountsAsync(); + counts.Outgoing.ShouldBeGreaterThanOrEqualTo(1); + + await thePersistence.Outbox.DeleteOutgoingAsync([envelope]); + + var counts2 = await thePersistence.Admin.FetchCountsAsync(); + counts2.Outgoing.ShouldBe(counts.Outgoing - 1); + } + + [Fact] + public async Task can_move_envelope_to_dead_letter_queue() + { + var envelope = ObjectMother.Envelope(); + await thePersistence.Inbox.StoreIncomingAsync(envelope); + + var exception = new InvalidOperationException("Test error"); + await thePersistence.Inbox.MoveToDeadLetterStorageAsync(envelope, exception); + + var counts = await thePersistence.Admin.FetchCountsAsync(); + counts.DeadLetter.ShouldBeGreaterThanOrEqualTo(1); + } + + [Fact] + public async Task can_mark_envelope_as_handled() + { + var envelope = ObjectMother.Envelope(); + await thePersistence.Inbox.StoreIncomingAsync(envelope); + + await thePersistence.Inbox.MarkIncomingEnvelopeAsHandledAsync(envelope); + + var counts = await thePersistence.Admin.FetchCountsAsync(); + counts.Handled.ShouldBeGreaterThanOrEqualTo(1); + } + + [Fact] + public async Task can_schedule_envelope() + { + var envelope = ObjectMother.Envelope(); + envelope.ScheduleDelay = 1.Hours(); + envelope.Status = EnvelopeStatus.Scheduled; + + await thePersistence.Inbox.StoreIncomingAsync(envelope); + + var counts = await thePersistence.Admin.FetchCountsAsync(); + counts.Scheduled.ShouldBeGreaterThanOrEqualTo(1); + } +} diff --git a/src/Persistence/Oracle/OracleTests/OracleTests.csproj b/src/Persistence/Oracle/OracleTests/OracleTests.csproj new file mode 100644 index 000000000..92a81c7cc --- /dev/null +++ b/src/Persistence/Oracle/OracleTests/OracleTests.csproj @@ -0,0 +1,34 @@ + + + + false + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + Servers.cs + + + + diff --git a/src/Persistence/Oracle/OracleTests/Sagas/saga_storage_operations.cs b/src/Persistence/Oracle/OracleTests/Sagas/saga_storage_operations.cs new file mode 100644 index 000000000..2c17c3c92 --- /dev/null +++ b/src/Persistence/Oracle/OracleTests/Sagas/saga_storage_operations.cs @@ -0,0 +1,179 @@ +using IntegrationTests; +using Oracle.ManagedDataAccess.Client; +using Shouldly; +using Weasel.Oracle; +using Wolverine; +using Wolverine.Oracle.Sagas; +using Wolverine.RDBMS; +using Wolverine.RDBMS.Sagas; + +namespace OracleTests.Sagas; + +[Collection("oracle")] +public class saga_storage_operations +{ + private readonly OracleSagaSchema theSchema; + + public saga_storage_operations() + { + var settings = new DatabaseSettings + { + ConnectionString = Servers.OracleConnectionString, + SchemaName = "WOLVERINE", + }; + + var definition = new SagaTableDefinition(typeof(OracleLightweightSaga), null); + theSchema = new OracleSagaSchema(definition, settings); + } + + [Fact] + public async Task load_with_no_document_happily_returns_null() + { + await using var conn = new OracleConnection(Servers.OracleConnectionString); + await conn.OpenAsync(); + + using var tx = (OracleTransaction)await conn.BeginTransactionAsync(); + + var saga = await theSchema.LoadAsync(Guid.NewGuid(), tx, CancellationToken.None); + saga.ShouldBeNull(); + } + + [Fact] + public async Task get_an_argument_out_of_range_exception_for_missing_id() + { + await using var conn = new OracleConnection(Servers.OracleConnectionString); + await conn.OpenAsync(); + var db = (OracleTransaction)await conn.BeginTransactionAsync(); + + var saga = new OracleLightweightSaga + { + Id = Guid.Empty, + Name = "Xavier Worthy", + }; + + await Should.ThrowAsync(async () => + { + await theSchema.InsertAsync(saga, db, CancellationToken.None); + }); + } + + [Fact] + public async Task insert_then_load() + { + await using var conn = new OracleConnection(Servers.OracleConnectionString); + await conn.OpenAsync(); + var db = (OracleTransaction)await conn.BeginTransactionAsync(); + + var saga = new OracleLightweightSaga + { + Id = Guid.NewGuid(), + Name = "Xavier Worthy", + }; + + await theSchema.InsertAsync(saga, db, CancellationToken.None); + await db.CommitAsync(); + + var db2 = (OracleTransaction)await conn.BeginTransactionAsync(); + var saga2 = await theSchema.LoadAsync(saga.Id, db2, CancellationToken.None); + + saga2.ShouldNotBeNull(); + saga2.Name.ShouldBe("Xavier Worthy"); + } + + [Fact] + public async Task insert_update_then_load() + { + await using var conn = new OracleConnection(Servers.OracleConnectionString); + await conn.OpenAsync(); + var db = (OracleTransaction)await conn.BeginTransactionAsync(); + + var saga = new OracleLightweightSaga + { + Id = Guid.NewGuid(), + Name = "Xavier Worthy", + }; + + await theSchema.InsertAsync(saga, db, CancellationToken.None); + + saga.Name = "Hollywood Brown"; + await theSchema.UpdateAsync(saga, db, CancellationToken.None); + await db.CommitAsync(); + + var db2 = (OracleTransaction)await conn.BeginTransactionAsync(); + var saga2 = await theSchema.LoadAsync(saga.Id, db2, CancellationToken.None); + + saga2.ShouldNotBeNull(); + saga2.Name.ShouldBe("Hollywood Brown"); + } + + [Fact] + public async Task insert_then_delete() + { + await using var conn = new OracleConnection(Servers.OracleConnectionString); + await conn.OpenAsync(); + var db = (OracleTransaction)await conn.BeginTransactionAsync(); + + var saga = new OracleLightweightSaga + { + Id = Guid.NewGuid(), + Name = "Xavier Worthy", + }; + + await theSchema.InsertAsync(saga, db, CancellationToken.None); + + await theSchema.DeleteAsync(saga, db, CancellationToken.None); + await db.CommitAsync(); + + var db2 = (OracleTransaction)await conn.BeginTransactionAsync(); + var saga2 = await theSchema.LoadAsync(saga.Id, db2, CancellationToken.None); + saga2.ShouldBeNull(); + } + + [Fact] + public async Task concurrency_exception_when_version_does_not_match() + { + await theSchema.EnsureStorageExistsAsync(CancellationToken.None); + + await using var conn = new OracleConnection(Servers.OracleConnectionString); + await conn.OpenAsync(); + + // Clean up the table + var cleanCmd = conn.CreateCommand( + $"DELETE FROM WOLVERINE.{nameof(OracleLightweightSaga).ToUpperInvariant()}_SAGA"); + await cleanCmd.ExecuteNonQueryAsync(); + + var db = (OracleTransaction)await conn.BeginTransactionAsync(); + + var saga = new OracleLightweightSaga + { + Id = Guid.NewGuid(), + Name = "Xavier Worthy", + }; + + await theSchema.InsertAsync(saga, db, CancellationToken.None); + await db.CommitAsync(); + + db = (OracleTransaction)await conn.BeginTransactionAsync(); + + saga.Name = "Rashee Rice"; + await theSchema.UpdateAsync(saga, db, CancellationToken.None); + await db.CommitAsync(); + + db = (OracleTransaction)await conn.BeginTransactionAsync(); + + // I'm rewinding the version to make it throw + saga.Version = 1; + + await Should.ThrowAsync(async () => + { + await theSchema.UpdateAsync(saga, db, CancellationToken.None); + await db.CommitAsync(); + }); + } +} + +public class OracleLightweightSaga : Saga +{ + public Guid Id { get; set; } + public string Name { get; set; } = string.Empty; +} diff --git a/src/Persistence/Oracle/OracleTests/Transport/basic_functionality.cs b/src/Persistence/Oracle/OracleTests/Transport/basic_functionality.cs new file mode 100644 index 000000000..669066694 --- /dev/null +++ b/src/Persistence/Oracle/OracleTests/Transport/basic_functionality.cs @@ -0,0 +1,209 @@ +using IntegrationTests; +using JasperFx.Core; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Oracle.ManagedDataAccess.Client; +using NSubstitute; +using Shouldly; +using Weasel.Oracle; +using Wolverine; +using Wolverine.ComplianceTests; +using Wolverine.Oracle; +using Wolverine.Oracle.Transport; +using Wolverine.Persistence.Durability; +using Wolverine.Runtime; +using Wolverine.Runtime.WorkerQueues; +using Wolverine.Tracking; +using Xunit.Abstractions; + +namespace OracleTests.Transport; + +[Collection("oracle")] +public class basic_functionality : IAsyncLifetime +{ + private readonly ITestOutputHelper _output; + + public basic_functionality(ITestOutputHelper output) + { + _output = output; + } + + private IHost theHost = null!; + private OracleTransport theTransport = null!; + private OracleQueue theQueue = null!; + private IMessageStore theMessageStore = null!; + private WolverineRuntime theRuntime = null!; + + public async Task InitializeAsync() + { + // Clean the schema + var dataSource = new OracleDataSource(Servers.OracleConnectionString); + await using var conn = await dataSource.OpenConnectionAsync(); + try + { + // Drop queue tables if they exist + try + { + var cmd = conn.CreateCommand("DROP TABLE WOLVERINE.WOLVERINE_QUEUE_ONE CASCADE CONSTRAINTS"); + await cmd.ExecuteNonQueryAsync(); + } + catch (OracleException) { /* table doesn't exist */ } + + try + { + var cmd = conn.CreateCommand("DROP TABLE WOLVERINE.WOLVERINE_QUEUE_ONE_SCHEDULED CASCADE CONSTRAINTS"); + await cmd.ExecuteNonQueryAsync(); + } + catch (OracleException) { /* table doesn't exist */ } + } + finally + { + await conn.CloseAsync(); + } + + theHost = await Host.CreateDefaultBuilder() + .UseWolverine(opts => + { + opts.PersistMessagesWithOracle(Servers.OracleConnectionString, "WOLVERINE") + .EnableMessageTransport(); + opts.ListenToOracleQueue("one"); + opts.Durability.Mode = DurabilityMode.Solo; + }).StartAsync(); + + theTransport = theHost.GetRuntime().Options.Transports.GetOrCreate(); + theQueue = theTransport.Queues["one"]; + + theMessageStore = theHost.GetRuntime().Storage; + + theRuntime = theHost.GetRuntime(); + } + + public async Task DisposeAsync() + { + await theHost.StopAsync(); + theHost.Dispose(); + } + + [Fact] + public async Task connect_smoke_test() + { + await theTransport.ConnectAsync(theHost.GetRuntime()); + } + + [Fact] + public async Task purge_queue_smoke_test() + { + await theQueue.PurgeAsync(NullLogger.Instance); + } + + [Fact] + public async Task check_queue_smoke_test() + { + (await theQueue.CheckAsync()).ShouldBeTrue(); + } + + [Fact] + public async Task teardown_and_setup() + { + await theQueue.TeardownAsync(NullLogger.Instance); + await theQueue.SetupAsync(NullLogger.Instance); + } + + [Fact] + public async Task send_not_scheduled_smoke_test() + { + await theQueue.PurgeAsync(NullLogger.Instance); + + var envelope = ObjectMother.Envelope(); + envelope.DeliverBy = DateTimeOffset.UtcNow.AddHours(1); + await theQueue.SendAsync(envelope); + + (await theQueue.CountAsync()).ShouldBe(1); + (await theQueue.ScheduledCountAsync()).ShouldBe(0); + } + + [Fact] + public async Task send_not_scheduled_is_idempotent_smoke_test() + { + await theQueue.PurgeAsync(NullLogger.Instance); + + var envelope = ObjectMother.Envelope(); + envelope.DeliverBy = DateTimeOffset.UtcNow.AddHours(1); + await theQueue.SendAsync(envelope); + await theQueue.SendAsync(envelope); + await theQueue.SendAsync(envelope); + + (await theQueue.CountAsync()).ShouldBe(1); + (await theQueue.ScheduledCountAsync()).ShouldBe(0); + } + + [Fact] + public async Task send_scheduled_smoke_test() + { + await theQueue.PurgeAsync(NullLogger.Instance); + + var envelope = ObjectMother.Envelope(); + envelope.ScheduleDelay = 1.Hours(); + envelope.IsScheduledForLater(DateTimeOffset.UtcNow).ShouldBeTrue(); + envelope.DeliverBy = DateTimeOffset.UtcNow.AddHours(1); + await theQueue.SendAsync(envelope); + + (await theQueue.CountAsync()).ShouldBe(0); + (await theQueue.ScheduledCountAsync()).ShouldBe(1); + } + + [Fact] + public async Task does_fine_with_double_schedule() + { + await theQueue.PurgeAsync(NullLogger.Instance); + + var envelope = ObjectMother.Envelope(); + envelope.ScheduleDelay = 1.Hours(); + envelope.IsScheduledForLater(DateTimeOffset.UtcNow).ShouldBeTrue(); + envelope.DeliverBy = DateTimeOffset.UtcNow.AddHours(1); + await theQueue.SendAsync(envelope); + + // Does not blow up + await theQueue.SendAsync(envelope); + await theQueue.SendAsync(envelope); + + (await theQueue.CountAsync()).ShouldBe(0); + (await theQueue.ScheduledCountAsync()).ShouldBe(1); + } + + [Fact] + public async Task move_from_outgoing_to_queue_async() + { + await theQueue.PurgeAsync(NullLogger.Instance); + (await theQueue.CountAsync()).ShouldBe(0); + + var envelope = ObjectMother.Envelope(); + await theMessageStore.Outbox.StoreOutgoingAsync(envelope, 0); + + await new OracleQueueSender(theQueue).MoveFromOutgoingToQueueAsync(envelope, CancellationToken.None); + + (await theQueue.CountAsync()).ShouldBe(1); + + var stats = await theMessageStore.Admin.FetchCountsAsync(); + stats.Outgoing.ShouldBe(0); + } + + [Fact] + public async Task move_from_outgoing_to_scheduled_async() + { + await theQueue.PurgeAsync(NullLogger.Instance); + (await theQueue.CountAsync()).ShouldBe(0); + + var envelope = ObjectMother.Envelope(); + envelope.ScheduleDelay = 1.Hours(); + envelope.IsScheduledForLater(DateTimeOffset.UtcNow).ShouldBeTrue(); + await theMessageStore.Outbox.StoreOutgoingAsync(envelope, 0); + + await new OracleQueueSender(theQueue).MoveFromOutgoingToScheduledAsync(envelope, CancellationToken.None); + + (await theQueue.ScheduledCountAsync()).ShouldBe(1); + + var stats = await theMessageStore.Admin.FetchCountsAsync(); + stats.Outgoing.ShouldBe(0); + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/AssemblyAttributes.cs b/src/Persistence/Oracle/Wolverine.Oracle/AssemblyAttributes.cs new file mode 100644 index 000000000..4a0e5faa3 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/AssemblyAttributes.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("OracleTests")] diff --git a/src/Persistence/Oracle/Wolverine.Oracle/OracleAdvisoryLock.cs b/src/Persistence/Oracle/Wolverine.Oracle/OracleAdvisoryLock.cs new file mode 100644 index 000000000..aeffac073 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/OracleAdvisoryLock.cs @@ -0,0 +1,133 @@ +using System.Data; +using JasperFx.Core; +using Microsoft.Extensions.Logging; +using Oracle.ManagedDataAccess.Client; +using Weasel.Core; +using Wolverine.Oracle.Schema; + +namespace Wolverine.Oracle; + +/// +/// Oracle implementation of advisory locks using row-level locks (FOR UPDATE NOWAIT). +/// Uses the wolverine_locks table to hold lock rows. +/// +internal class OracleAdvisoryLock : IAdvisoryLock +{ + private readonly string _schemaName; + private readonly List _locks = new(); + private readonly ILogger _logger; + private readonly OracleDataSource _source; + private readonly Dictionary _heldLocks = new(); + + public OracleAdvisoryLock(OracleDataSource source, ILogger logger, string schemaName) + { + _source = source; + _logger = logger; + _schemaName = schemaName; + } + + public bool HasLock(int lockId) + { + return _locks.Contains(lockId); + } + + public async Task TryAttainLockAsync(int lockId, CancellationToken token) + { + try + { + var conn = await _source.OpenConnectionAsync(token); + + // Ensure lock row exists + var ensureCmd = conn.CreateCommand( + $"MERGE INTO {_schemaName}.{LockTable.TableName} t " + + "USING DUAL ON (t.lock_id = :lockId) " + + "WHEN NOT MATCHED THEN INSERT (lock_id) VALUES (:lockId)"); + ensureCmd.With("lockId", lockId); + + try + { + await ensureCmd.ExecuteNonQueryAsync(token); + } + catch (OracleException) + { + // Race condition - another process may have inserted it + } + + // Start a transaction to hold the row lock + var tx = (OracleTransaction)await conn.BeginTransactionAsync(token); + + var lockCmd = conn.CreateCommand( + $"SELECT lock_id FROM {_schemaName}.{LockTable.TableName} WHERE lock_id = :lockId FOR UPDATE NOWAIT"); + lockCmd.Transaction = tx; + lockCmd.With("lockId", lockId); + + try + { + await lockCmd.ExecuteScalarAsync(token); + _locks.Add(lockId); + _heldLocks[lockId] = (conn, tx); + return true; + } + catch (OracleException ex) when (ex.Number == 54) // ORA-00054: resource busy + { + await tx.RollbackAsync(token); + await conn.CloseAsync(); + await conn.DisposeAsync(); + return false; + } + } + catch (Exception e) + { + _logger.LogError(e, "Error trying to attain advisory lock {LockId}", lockId); + return false; + } + } + + public async Task ReleaseLockAsync(int lockId) + { + if (!_locks.Contains(lockId)) return; + + _locks.Remove(lockId); + + if (_heldLocks.TryGetValue(lockId, out var held)) + { + _heldLocks.Remove(lockId); + try + { + var cancellation = new CancellationTokenSource(); + cancellation.CancelAfter(1.Seconds()); + + await held.tx.RollbackAsync(cancellation.Token); + await held.conn.CloseAsync(); + await held.conn.DisposeAsync(); + } + catch (Exception e) + { + _logger.LogError(e, "Error releasing advisory lock {LockId}", lockId); + } + } + } + + public async ValueTask DisposeAsync() + { + foreach (var lockId in _locks.ToArray()) + { + if (_heldLocks.TryGetValue(lockId, out var held)) + { + try + { + await held.tx.RollbackAsync(CancellationToken.None); + await held.conn.CloseAsync(); + await held.conn.DisposeAsync(); + } + catch (Exception e) + { + _logger.LogError(e, "Error disposing advisory lock {LockId}", lockId); + } + } + } + + _locks.Clear(); + _heldLocks.Clear(); + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/OracleBackedPersistence.cs b/src/Persistence/Oracle/Wolverine.Oracle/OracleBackedPersistence.cs new file mode 100644 index 000000000..93f78eef3 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/OracleBackedPersistence.cs @@ -0,0 +1,291 @@ +using System.Data.Common; +using JasperFx; +using JasperFx.Core; +using JasperFx.MultiTenancy; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Oracle.ManagedDataAccess.Client; +using Weasel.Core; +using Weasel.Core.Migrations; +using Weasel.Oracle; +using Wolverine.ErrorHandling; +using Wolverine.Oracle.Transport; +using Wolverine.Persistence.Durability; +using Wolverine.Persistence.Sagas; +using Wolverine.RDBMS; +using Wolverine.RDBMS.MultiTenancy; +using Wolverine.RDBMS.Sagas; +using Wolverine.Runtime; + +namespace Wolverine.Oracle; + +public interface IOracleBackedPersistence +{ + /// + /// Tell Wolverine that the persistence service (EF Core DbContext? Something else?) of the given + /// type should be enrolled in envelope storage with this Oracle database + /// + IOracleBackedPersistence Enroll(Type type); + + /// + /// Tell Wolverine that the persistence service (EF Core DbContext? Something else?) of the given + /// type should be enrolled in envelope storage with this Oracle database + /// + IOracleBackedPersistence Enroll(); + + /// + /// By default, Wolverine takes the AutoCreate settings from JasperFxOptions, but + /// you can override the application default for just the Oracle backed + /// envelope storage tables + /// + IOracleBackedPersistence OverrideAutoCreateResources(AutoCreate autoCreate); + + /// + /// Override the database schema name for the envelope storage tables (the transactional inbox/outbox). + /// Default is "WOLVERINE" + /// + IOracleBackedPersistence SchemaName(string schemaName); + + /// + /// Override the database advisory lock number that Wolverine uses to grant temporary, exclusive + /// access to execute scheduled messages for this application. + /// + IOracleBackedPersistence OverrideScheduledJobLockId(int lockId); + + /// + /// Should Wolverine provision Oracle command queues for this Wolverine application? The default is true, + /// but these queues are unnecessary if using an external broker for Wolverine command queues + /// + IOracleBackedPersistence EnableCommandQueues(bool enabled); + + /// + /// Opt into using static per-tenant database multi-tenancy. With this option, Wolverine is assuming that + /// there the number of tenant databases is static and does not change at runtime + /// + IOracleBackedPersistence RegisterStaticTenants(Action configure); + + /// + /// Opt into multi-tenancy with separate databases using your own strategy for finding the right connection string + /// for a given tenant id + /// + IOracleBackedPersistence RegisterTenants(ITenantedSource tenantSource); + + /// + /// Opt into multi-tenancy with separate databases using a master table lookup of tenant id to connection string + /// that is controlled by Wolverine. This supports dynamic addition of new tenant databases at runtime without any + /// downtime + /// + IOracleBackedPersistence UseMasterTableTenancy(Action configure); + + /// + /// Enable the Oracle messaging transport for this Wolverine application + /// + IOracleBackedPersistence EnableMessageTransport(Action? configure = null); +} + +/// +/// Activates the Oracle backed message persistence +/// +internal class OracleBackedPersistence : IOracleBackedPersistence, IWolverineExtension +{ + private readonly WolverineOptions _options; + + public OracleBackedPersistence(DurabilitySettings settings, WolverineOptions options) + { + _options = options; + EnvelopeStorageSchemaName = settings.MessageStorageSchemaName ?? "WOLVERINE"; + } + + internal bool AlreadyIncluded { get; set; } + + public string? ConnectionString { get; set; } + + public string EnvelopeStorageSchemaName { get; set; } + + public AutoCreate AutoCreate { get; set; } = JasperFx.AutoCreate.CreateOrUpdate; + + public bool CommandQueuesEnabled { get; set; } = true; + + private int _scheduledJobLockId; + + public int ScheduledJobLockId + { + get + { + if (_scheduledJobLockId > 0) return _scheduledJobLockId; + + return $"{EnvelopeStorageSchemaName}:scheduled-jobs".GetDeterministicHashCode(); + } + set => _scheduledJobLockId = value; + } + + public void Configure(WolverineOptions options) + { + if (ConnectionString.IsEmpty()) + { + throw new InvalidOperationException( + "The Oracle backed persistence needs to have a connection string defined for the main envelope database"); + } + + // Handle duplicate key errors (ORA-00001) + options.OnException(oracle => + oracle.Number == 1) + .Discard(); + + options.Services.AddSingleton(buildMainDatabaseSettings()); + options.CodeGeneration.AddPersistenceStrategy(); + options.CodeGeneration.Sources.Add(new DatabaseBackedPersistenceMarker()); + options.CodeGeneration.Sources.Add(new SagaStorageVariableSource()); + + options.Services.AddSingleton(s => BuildMessageStore(s.GetRequiredService())); + + options.Services.AddSingleton(); + + options.Services.AddSingleton(); + } + + public IMessageStore BuildMessageStore(IWolverineRuntime runtime) + { + var settings = buildMainDatabaseSettings(); + + var sagaTables = runtime.Services.GetServices().ToArray(); + + var dataSource = new OracleDataSource(ConnectionString!); + var logger = runtime.LoggerFactory.CreateLogger(); + + if (UseMasterTableTenancy) + { + var defaultStore = new OracleMessageStore(settings, runtime.DurabilitySettings, dataSource, + logger, sagaTables); + + ConnectionStringTenancy = new MasterTenantSource(defaultStore, runtime.Options); + + return new MultiTenantedMessageStore(defaultStore, runtime, + new OracleTenantedMessageStore(runtime, this, sagaTables)); + } + + if (ConnectionStringTenancy != null) + { + var defaultStore = new OracleMessageStore(settings, runtime.DurabilitySettings, dataSource, + logger, sagaTables); + + return new MultiTenantedMessageStore(defaultStore, runtime, + new OracleTenantedMessageStore(runtime, this, sagaTables)); + } + + settings.Role = Role; + + return new OracleMessageStore(settings, runtime.DurabilitySettings, dataSource, + logger, sagaTables); + } + + public MessageStoreRole Role { get; set; } = MessageStoreRole.Main; + + private DatabaseSettings buildMainDatabaseSettings() + { + var settings = new DatabaseSettings + { + CommandQueuesEnabled = CommandQueuesEnabled, + Role = MessageStoreRole.Main, + ConnectionString = ConnectionString, + ScheduledJobLockId = ScheduledJobLockId, + SchemaName = EnvelopeStorageSchemaName, + AddTenantLookupTable = UseMasterTableTenancy, + TenantConnections = TenantConnections + }; + return settings; + } + + public IOracleBackedPersistence Enroll(Type type) + { + _options.Services.AddSingleton(s => + new AncillaryMessageStore(type, BuildMessageStore(s.GetRequiredService()))); + + return this; + } + + public IOracleBackedPersistence Enroll() + { + return Enroll(typeof(T)); + } + + IOracleBackedPersistence IOracleBackedPersistence.OverrideAutoCreateResources(AutoCreate autoCreate) + { + AutoCreate = autoCreate; + return this; + } + + IOracleBackedPersistence IOracleBackedPersistence.SchemaName(string schemaName) + { + if (schemaName.IsEmpty()) + throw new ArgumentNullException(nameof(schemaName), "Schema Name cannot be empty or null"); + + EnvelopeStorageSchemaName = schemaName.ToUpperInvariant(); + return this; + } + + IOracleBackedPersistence IOracleBackedPersistence.OverrideScheduledJobLockId(int lockId) + { + _scheduledJobLockId = lockId; + return this; + } + + IOracleBackedPersistence IOracleBackedPersistence.EnableCommandQueues(bool enabled) + { + CommandQueuesEnabled = enabled; + return this; + } + + IOracleBackedPersistence IOracleBackedPersistence.RegisterStaticTenants(Action configure) + { + var source = new StaticConnectionStringSource(); + configure(source); + ConnectionStringTenancy = source; + + return this; + } + + public ITenantedSource? ConnectionStringTenancy { get; set; } + + public bool UseMasterTableTenancy { get; set; } + + IOracleBackedPersistence IOracleBackedPersistence.RegisterTenants(ITenantedSource tenantSource) + { + ConnectionStringTenancy = tenantSource; + return this; + } + + IOracleBackedPersistence IOracleBackedPersistence.UseMasterTableTenancy( + Action configure) + { + UseMasterTableTenancy = true; + var source = new StaticConnectionStringSource(); + configure(source); + + TenantConnections = source; + return this; + } + + public StaticConnectionStringSource? TenantConnections { get; set; } + + private List> _transportConfigurations = new(); + + public IOracleBackedPersistence EnableMessageTransport(Action? configure = null) + { + if (configure != null) + { + if (AlreadyIncluded) + { + var transport = _options.Transports.GetOrCreate(); + + var expression = new OraclePersistenceExpression(transport, _options); + configure(expression); + } + else + { + _transportConfigurations.Add(configure); + } + } + return this; + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/OracleConfigurationExtensions.cs b/src/Persistence/Oracle/Wolverine.Oracle/OracleConfigurationExtensions.cs new file mode 100644 index 000000000..f76006470 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/OracleConfigurationExtensions.cs @@ -0,0 +1,89 @@ +using JasperFx.Core; +using JasperFx.Core.Reflection; +using Wolverine.Configuration; +using Wolverine.Oracle.Transport; +using Wolverine.Persistence.Durability; + +namespace Wolverine.Oracle; + +public static class OracleConfigurationExtensions +{ + /// + /// Register Oracle backed message persistence to a known connection string + /// + /// + /// + /// Optional schema name for the Wolverine envelope storage + /// Default is Main. Use this to mark some stores as Ancillary to disambiguate the main storage for Wolverine + public static IOracleBackedPersistence PersistMessagesWithOracle(this WolverineOptions options, + string connectionString, + string? schemaName = null, MessageStoreRole role = MessageStoreRole.Main) + { + var persistence = new OracleBackedPersistence(options.Durability, options) + { + ConnectionString = connectionString, + AlreadyIncluded = true, + Role = role + }; + + if (schemaName.IsNotEmpty()) + { + persistence.EnvelopeStorageSchemaName = schemaName.ToUpperInvariant(); + } + + persistence.Configure(options); + + return persistence; + } + + /// + /// Quick access to the Oracle Transport within this application. + /// This is for advanced usage + /// + internal static OracleTransport OracleTransport(this WolverineOptions endpoints) + { + var transports = endpoints.As().Transports; + + try + { + return transports.GetOrCreate(); + } + catch (Exception) + { + throw new InvalidOperationException("The Oracle transport is not registered in this system"); + } + } + + /// + /// Listen for incoming messages at the designated Oracle queue by name + /// + public static OracleListenerConfiguration ListenToOracleQueue(this WolverineOptions endpoints, string queueName) + { + var transport = endpoints.OracleTransport(); + var corrected = transport.MaybeCorrectName(queueName); + var queue = transport.Queues[corrected]; + queue.EndpointName = queueName; + queue.IsListener = true; + + return new OracleListenerConfiguration(queue); + } + + /// + /// Publish matching messages straight to an Oracle queue using the queue name + /// + public static OracleSubscriberConfiguration ToOracleQueue(this IPublishToExpression publishing, + string queueName) + { + var transports = publishing.As().Parent.Transports; + var transport = transports.GetOrCreate(); + + var corrected = transport.MaybeCorrectName(queueName); + var queue = transport.Queues[corrected]; + queue.EndpointName = queueName; + + // This is necessary unfortunately to hook up the subscription rules + publishing.To(queue.Uri); + + return new OracleSubscriberConfiguration(queue); + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/OracleDataSource.cs b/src/Persistence/Oracle/Wolverine.Oracle/OracleDataSource.cs new file mode 100644 index 000000000..3840b02a1 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/OracleDataSource.cs @@ -0,0 +1,37 @@ +using System.Data.Common; +using Oracle.ManagedDataAccess.Client; + +namespace Wolverine.Oracle; + +/// +/// Thin DbDataSource wrapper for Oracle since Oracle.ManagedDataAccess +/// does not provide a native DbDataSource implementation. +/// +internal class OracleDataSource : DbDataSource +{ + private readonly string _connectionString; + + public OracleDataSource(string connectionString) + { + _connectionString = connectionString ?? throw new ArgumentNullException(nameof(connectionString)); + } + + public override string ConnectionString => _connectionString; + + protected override DbConnection CreateDbConnection() + { + return new OracleConnection(_connectionString); + } + + public new OracleConnection CreateConnection() + { + return new OracleConnection(_connectionString); + } + + public new async Task OpenConnectionAsync(CancellationToken cancellationToken = default) + { + var conn = new OracleConnection(_connectionString); + await conn.OpenAsync(cancellationToken); + return conn; + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Admin.cs b/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Admin.cs new file mode 100644 index 000000000..cc087742a --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Admin.cs @@ -0,0 +1,259 @@ +using JasperFx; +using JasperFx.Core; +using Oracle.ManagedDataAccess.Client; +using Weasel.Core; +using Weasel.Core.Migrations; +using Weasel.Oracle; +using Wolverine.Logging; +using Wolverine.Oracle.Util; +using Wolverine.Persistence.Durability; +using Wolverine.RDBMS; + +namespace Wolverine.Oracle; + +internal partial class OracleMessageStore +{ + // IMessageStoreAdmin + public async Task ClearAllAsync() + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + + var tables = new List + { + DatabaseConstants.IncomingTable, + DatabaseConstants.OutgoingTable, + DatabaseConstants.DeadLetterTable + }; + + if (_settings.Role == MessageStoreRole.Main) + { + tables.Add(DatabaseConstants.AgentRestrictionsTableName); + tables.Add(DatabaseConstants.NodeRecordTableName); + } + + foreach (var tableName in tables) + { + try + { + var cmd = conn.CreateCommand($"DELETE FROM {SchemaName}.{tableName}"); + await cmd.ExecuteNonQueryAsync(_cancellation); + } + catch (OracleException e) when (e.Number == 942) + { + // Table doesn't exist yet, ignore + } + } + + await conn.CloseAsync(); + } + + public async Task RebuildAsync() + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + + // Drop all tables first (CASCADE CONSTRAINTS handles FK dependencies) + var objects = AllObjects().ToArray(); + foreach (var obj in objects.Reverse()) + { + try + { + var dropCmd = conn.CreateCommand( + $"DROP TABLE {obj.Identifier.QualifiedName} CASCADE CONSTRAINTS"); + await dropCmd.ExecuteNonQueryAsync(_cancellation); + } + catch (OracleException e) when (e.Number == 942) // ORA-00942: table or view does not exist + { + // OK - table doesn't exist yet + } + } + + // Also drop the lock table + try + { + var dropLocks = conn.CreateCommand( + $"DROP TABLE {SchemaName}.{Schema.LockTable.TableName} CASCADE CONSTRAINTS"); + await dropLocks.ExecuteNonQueryAsync(_cancellation); + } + catch (OracleException e) when (e.Number == 942) + { + // OK + } + + // Now recreate all tables + foreach (var obj in objects) + { + var migration = await SchemaMigration.DetermineAsync(conn, _cancellation, obj); + if (migration.Difference != SchemaPatchDifference.None) + { + try + { + await new OracleMigrator().ApplyAllAsync(conn, migration, AutoCreate.CreateOrUpdate, ct: _cancellation); + } + catch (OracleException e) when (e.Number is 955 or 2275 or 1408) + { + // ORA-00955: name already used by existing object + // ORA-02275: referential constraint already exists + // ORA-01408: such column list already indexed + } + } + } + + await conn.CloseAsync(); + } + + public async Task FetchCountsAsync() + { + var counts = new PersistedCounts(); + + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + + // Incoming counts by status + var statusCmd = conn.CreateCommand( + $"SELECT status, COUNT(*) FROM {SchemaName}.{DatabaseConstants.IncomingTable} GROUP BY status"); + await using (var reader = await statusCmd.ExecuteReaderAsync(_cancellation)) + { + while (await reader.ReadAsync(_cancellation)) + { + var status = Enum.Parse(await reader.GetFieldValueAsync(0, _cancellation)); + var count = await reader.GetFieldValueAsync(1, _cancellation); + + if (status == EnvelopeStatus.Incoming) counts.Incoming = (int)count; + else if (status == EnvelopeStatus.Handled) counts.Handled = (int)count; + else if (status == EnvelopeStatus.Scheduled) counts.Scheduled = (int)count; + } + } + + // Outgoing count + var outCmd = conn.CreateCommand( + $"SELECT COUNT(*) FROM {SchemaName}.{DatabaseConstants.OutgoingTable}"); + var outCount = await outCmd.ExecuteScalarAsync(_cancellation); + counts.Outgoing = Convert.ToInt32(outCount); + + // Dead letter count + var deadCmd = conn.CreateCommand( + $"SELECT COUNT(*) FROM {SchemaName}.{DatabaseConstants.DeadLetterTable}"); + var deadCount = await deadCmd.ExecuteScalarAsync(_cancellation); + counts.DeadLetter = Convert.ToInt32(deadCount); + + await conn.CloseAsync(); + return counts; + } + + public async Task MigrateAsync() + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + + var objects = AllObjects().ToArray(); + foreach (var obj in objects) + { + var migration = await SchemaMigration.DetermineAsync(conn, _cancellation, obj); + if (migration.Difference != SchemaPatchDifference.None) + { + try + { + await new OracleMigrator().ApplyAllAsync(conn, migration, AutoCreate.CreateOrUpdate, ct: _cancellation); + } + catch (OracleException e) when (e.Number is 955 or 2275 or 1408) + { + // ORA-00955: name already used by existing object + // ORA-02275: referential constraint already exists + // ORA-01408: such column list already indexed + } + } + } + + await conn.CloseAsync(); + } + + public async Task> AllIncomingAsync() + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand( + $"SELECT {DatabaseConstants.IncomingFields} FROM {SchemaName}.{DatabaseConstants.IncomingTable}"); + + var list = await cmd.FetchListAsync( + r => OracleEnvelopeReader.ReadIncomingAsync(r), _cancellation); + await conn.CloseAsync(); + return list; + } + + public async Task> AllOutgoingAsync() + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand( + $"SELECT {DatabaseConstants.OutgoingFields} FROM {SchemaName}.{DatabaseConstants.OutgoingTable}"); + + var list = await cmd.FetchListAsync( + r => OracleEnvelopeReader.ReadOutgoingAsync(r), _cancellation); + await conn.CloseAsync(); + return list; + } + + public async Task ReleaseAllOwnershipAsync() + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + + var inCmd = conn.CreateCommand( + $"UPDATE {SchemaName}.{DatabaseConstants.IncomingTable} SET {DatabaseConstants.OwnerId} = 0"); + await inCmd.ExecuteNonQueryAsync(_cancellation); + + var outCmd = conn.CreateCommand( + $"UPDATE {SchemaName}.{DatabaseConstants.OutgoingTable} SET {DatabaseConstants.OwnerId} = 0"); + await outCmd.ExecuteNonQueryAsync(_cancellation); + + await conn.CloseAsync(); + } + + public async Task ReleaseAllOwnershipAsync(int ownerId) + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + + var inCmd = conn.CreateCommand( + $"UPDATE {SchemaName}.{DatabaseConstants.IncomingTable} SET {DatabaseConstants.OwnerId} = 0 WHERE {DatabaseConstants.OwnerId} = :ownerId"); + inCmd.With("ownerId", ownerId); + await inCmd.ExecuteNonQueryAsync(_cancellation); + + var outCmd = conn.CreateCommand( + $"UPDATE {SchemaName}.{DatabaseConstants.OutgoingTable} SET {DatabaseConstants.OwnerId} = 0 WHERE {DatabaseConstants.OwnerId} = :ownerId"); + outCmd.With("ownerId", ownerId); + await outCmd.ExecuteNonQueryAsync(_cancellation); + + await conn.CloseAsync(); + } + + public async Task CheckConnectivityAsync(CancellationToken token) + { + await using var conn = await _dataSource.OpenConnectionAsync(token); + var cmd = conn.CreateCommand("SELECT 1 FROM DUAL"); + await cmd.ExecuteScalarAsync(token); + await conn.CloseAsync(); + } + + public async Task DeleteAllHandledAsync() + { + await using var conn = await _dataSource.OpenConnectionAsync(CancellationToken.None); + + var deleted = 1; + + var sql = $@"DELETE FROM {SchemaName}.{DatabaseConstants.IncomingTable} WHERE id IN ( + SELECT id FROM {SchemaName}.{DatabaseConstants.IncomingTable} + WHERE status = '{EnvelopeStatus.Handled}' + ORDER BY id + FETCH FIRST 10000 ROWS ONLY + FOR UPDATE SKIP LOCKED)"; + + try + { + while (deleted > 0) + { + var cmd = conn.CreateCommand(sql); + deleted = await cmd.ExecuteNonQueryAsync(); + await Task.Delay(10.Milliseconds()); + } + } + finally + { + await conn.CloseAsync(); + } + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.DeadLetters.cs b/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.DeadLetters.cs new file mode 100644 index 000000000..536c6d484 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.DeadLetters.cs @@ -0,0 +1,233 @@ +using JasperFx.Core; +using Oracle.ManagedDataAccess.Client; +using Weasel.Oracle; +using Wolverine.Oracle.Util; +using Wolverine.Persistence.Durability; +using Wolverine.Persistence.Durability.DeadLetterManagement; +using Wolverine.RDBMS; + +namespace Wolverine.Oracle; + +internal partial class OracleMessageStore +{ + // IDeadLetters + + public async Task DeadLetterEnvelopeByIdAsync(Guid id, string? tenantId = null) + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand( + $"SELECT {DatabaseConstants.DeadLetterFields} FROM {SchemaName}.{DatabaseConstants.DeadLetterTable} WHERE id = :id"); + cmd.With("id", id); + + await using var reader = await cmd.ExecuteReaderAsync(_cancellation); + + if (!await reader.ReadAsync(_cancellation)) + { + await reader.CloseAsync(); + await conn.CloseAsync(); + return null; + } + + var deadLetterEnvelope = await OracleEnvelopeReader.ReadDeadLetterAsync(reader, _cancellation); + await reader.CloseAsync(); + await conn.CloseAsync(); + + return deadLetterEnvelope; + } + + public async Task> SummarizeAllAsync(string serviceName, TimeRange range, + CancellationToken token) + { + var builder = ToOracleCommandBuilder(); + builder.Append( + $"SELECT {DatabaseConstants.ReceivedAt}, {DatabaseConstants.MessageType}, {DatabaseConstants.ExceptionType}, COUNT(*) as total"); + builder.Append($" FROM {SchemaName}.{DatabaseConstants.DeadLetterTable}"); + builder.Append(" WHERE 1 = 1"); + + if (range.From.HasValue) + { + builder.Append($" AND {DatabaseConstants.SentAt} >= "); + builder.AppendParameter(range.From.Value.ToUniversalTime()); + } + + if (range.To.HasValue) + { + builder.Append($" AND {DatabaseConstants.SentAt} <= "); + builder.AppendParameter(range.To.Value.ToUniversalTime()); + } + + builder.Append( + $" GROUP BY {DatabaseConstants.ReceivedAt}, {DatabaseConstants.MessageType}, {DatabaseConstants.ExceptionType}"); + + var cmd = builder.Compile(); + + var envelopes = new List(); + + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + try + { + cmd.Connection = conn; + await using var reader = await cmd.ExecuteReaderAsync(_cancellation); + + while (await reader.ReadAsync(token)) + { + var uri = new Uri(await reader.GetFieldValueAsync(0, token)); + var messageType = await reader.GetFieldValueAsync(1, token); + var exceptionType = await reader.GetFieldValueAsync(2, token); + var count = Convert.ToInt32(await reader.GetFieldValueAsync(3, token)); + + envelopes.Add(new DeadLetterQueueCount(serviceName, uri, messageType, exceptionType, Uri, count)); + } + } + finally + { + await conn.CloseAsync(); + } + + return envelopes; + } + + public async Task QueryAsync(DeadLetterEnvelopeQuery query, CancellationToken token) + { + var builder = ToOracleCommandBuilder(); + + builder.Append( + $"SELECT {DatabaseConstants.DeadLetterFields}, COUNT(*) OVER() as total_rows FROM {SchemaName}.{DatabaseConstants.DeadLetterTable} WHERE 1 = 1"); + + writeDeadLetterWhereClause(query, builder); + + builder.Append($" ORDER BY {DatabaseConstants.ExecutionTime}"); + + if (query.PageNumber <= 0) query.PageNumber = 1; + + if (query.PageSize > 0) + { + var offset = query.PageNumber <= 1 ? 0 : (query.PageNumber - 1) * query.PageSize; + builder.Append($" OFFSET {offset} ROWS FETCH NEXT {query.PageSize} ROWS ONLY"); + } + + await using var conn = CreateConnection(); + await conn.OpenAsync(token); + + var cmd = builder.Compile(); + cmd.Connection = conn; + + await using var reader = await cmd.ExecuteReaderAsync(token); + + var results = new DeadLetterEnvelopeResults { PageNumber = query.PageNumber }; + if (await reader.ReadAsync(token)) + { + var env = await OracleEnvelopeReader.ReadDeadLetterAsync(reader, token); + results.Envelopes.Add(env); + results.TotalCount = Convert.ToInt32(await reader.GetFieldValueAsync(10, token)); + } + + while (await reader.ReadAsync(token)) + { + var env = await OracleEnvelopeReader.ReadDeadLetterAsync(reader, token); + results.Envelopes.Add(env); + } + + await reader.CloseAsync(); + await conn.CloseAsync(); + + return results; + } + + public async Task DiscardAsync(DeadLetterEnvelopeQuery query, CancellationToken token) + { + var builder = ToOracleCommandBuilder(); + + builder.Append($"DELETE FROM {SchemaName}.{DatabaseConstants.DeadLetterTable} WHERE 1 = 1"); + + writeDeadLetterWhereClause(query, builder); + + var cmd = builder.Compile(); + + await using var conn = await _dataSource.OpenConnectionAsync(token); + try + { + cmd.Connection = conn; + await cmd.ExecuteNonQueryAsync(token); + } + finally + { + await conn.CloseAsync(); + } + } + + public async Task ReplayAsync(DeadLetterEnvelopeQuery query, CancellationToken token) + { + var builder = ToOracleCommandBuilder(); + + builder.Append( + $"UPDATE {SchemaName}.{DatabaseConstants.DeadLetterTable} SET {DatabaseConstants.Replayable} = "); + builder.AppendParameter(1); // Oracle uses NUMBER(1) for bool + builder.Append(" WHERE 1 = 1"); + writeDeadLetterWhereClause(query, builder); + + var cmd = builder.Compile(); + + await using var conn = await _dataSource.OpenConnectionAsync(token); + try + { + cmd.Connection = conn; + await cmd.ExecuteNonQueryAsync(token); + } + finally + { + await conn.CloseAsync(); + } + } + + private void writeDeadLetterWhereClause(DeadLetterEnvelopeQuery query, Weasel.Oracle.CommandBuilder builder) + { + if (query.Range.From.HasValue) + { + builder.Append($" AND {DatabaseConstants.SentAt} >= "); + builder.AppendParameter(query.Range.From.Value.ToUniversalTime()); + } + + if (query.Range.To.HasValue) + { + builder.Append($" AND {DatabaseConstants.SentAt} <= "); + builder.AppendParameter(query.Range.To.Value.ToUniversalTime()); + } + + if (query.ExceptionType.IsNotEmpty()) + { + builder.Append($" AND {DatabaseConstants.ExceptionType} = "); + builder.AppendParameter(query.ExceptionType); + } + + if (query.ExceptionMessage.IsNotEmpty()) + { + builder.Append($" AND {DatabaseConstants.ExceptionMessage} LIKE "); + builder.AppendParameter(query.ExceptionMessage); + } + + if (query.MessageType.IsNotEmpty()) + { + builder.Append($" AND {DatabaseConstants.MessageType} = "); + builder.AppendParameter(query.MessageType); + } + + if (query.ReceivedAt.IsNotEmpty()) + { + builder.Append($" AND {DatabaseConstants.ReceivedAt} = "); + builder.AppendParameter(query.ReceivedAt); + } + + if (query.MessageIds is { Length: > 0 }) + { + builder.Append(" AND id IN ("); + for (var i = 0; i < query.MessageIds.Length; i++) + { + if (i > 0) builder.Append(", "); + builder.AppendParameter(query.MessageIds[i]); + } + + builder.Append(")"); + } + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Incoming.cs b/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Incoming.cs new file mode 100644 index 000000000..77dea0f4f --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Incoming.cs @@ -0,0 +1,345 @@ +using System.Data.Common; +using JasperFx.Core.Reflection; +using Oracle.ManagedDataAccess.Client; +using Weasel.Oracle; +using Wolverine.Oracle.Util; +using Wolverine.Persistence.Durability; +using Wolverine.RDBMS; +using Wolverine.Runtime.Serialization; +using Wolverine.Transports; + +namespace Wolverine.Oracle; + +internal partial class OracleMessageStore +{ + public async Task StoreIncomingAsync(Envelope envelope) + { + var data = envelope.Status == EnvelopeStatus.Handled + ? Array.Empty() + : EnvelopeSerializer.Serialize(envelope); + + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand( + $"INSERT INTO {SchemaName}.{DatabaseConstants.IncomingTable} ({DatabaseConstants.IncomingFields}) " + + "VALUES (:body, :id, :status, :ownerId, :executionTime, :attempts, :messageType, :receivedAt, :keepUntil)"); + + cmd.Parameters.Add(new OracleParameter("body", OracleDbType.Blob) { Value = data }); + cmd.With("id", envelope.Id); + cmd.With("status", envelope.Status.ToString()); + cmd.With("ownerId", envelope.OwnerId); + cmd.Parameters.Add(new OracleParameter("executionTime", OracleDbType.TimeStampTZ) { Value = (object?)envelope.ScheduledTime ?? DBNull.Value }); + cmd.With("attempts", envelope.Attempts); + cmd.With("messageType", envelope.MessageType!); + cmd.With("receivedAt", envelope.Destination?.ToString() ?? string.Empty); + cmd.Parameters.Add(new OracleParameter("keepUntil", OracleDbType.TimeStampTZ) { Value = (object?)envelope.KeepUntil ?? DBNull.Value }); + + try + { + await cmd.ExecuteNonQueryAsync(_cancellation); + } + catch (OracleException e) when (e.Number == 1) // ORA-00001: unique constraint violated + { + throw new DuplicateIncomingEnvelopeException(envelope); + } + finally + { + await conn.CloseAsync(); + } + } + + public async Task StoreIncomingAsync(IReadOnlyList envelopes) + { + if (envelopes.Count == 0) return; + + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var tx = (OracleTransaction)await conn.BeginTransactionAsync(_cancellation); + + foreach (var envelope in envelopes) + { + var data = envelope.Status == EnvelopeStatus.Handled + ? Array.Empty() + : EnvelopeSerializer.Serialize(envelope); + + var cmd = conn.CreateCommand( + $"INSERT INTO {SchemaName}.{DatabaseConstants.IncomingTable} ({DatabaseConstants.IncomingFields}) " + + "VALUES (:body, :id, :status, :ownerId, :executionTime, :attempts, :messageType, :receivedAt, :keepUntil)"); + cmd.Transaction = tx; + + cmd.Parameters.Add(new OracleParameter("body", OracleDbType.Blob) { Value = data }); + cmd.With("id", envelope.Id); + cmd.With("status", envelope.Status.ToString()); + cmd.With("ownerId", envelope.OwnerId); + cmd.Parameters.Add(new OracleParameter("executionTime", OracleDbType.TimeStampTZ) { Value = (object?)envelope.ScheduledTime ?? DBNull.Value }); + cmd.With("attempts", envelope.Attempts); + cmd.With("messageType", envelope.MessageType!); + cmd.With("receivedAt", envelope.Destination?.ToString() ?? string.Empty); + cmd.Parameters.Add(new OracleParameter("keepUntil", OracleDbType.TimeStampTZ) { Value = (object?)envelope.KeepUntil ?? DBNull.Value }); + + try + { + await cmd.ExecuteNonQueryAsync(_cancellation); + } + catch (OracleException e) when (e.Number == 1) + { + // Idempotent + } + } + + await tx.CommitAsync(_cancellation); + await conn.CloseAsync(); + } + + public async Task ExistsAsync(Envelope envelope, CancellationToken cancellation) + { + if (HasDisposed) return false; + + await using var conn = await _dataSource.OpenConnectionAsync(cancellation); + OracleCommand cmd; + + if (_durability.MessageIdentity == MessageIdentity.IdOnly) + { + cmd = conn.CreateCommand( + $"SELECT COUNT(id) FROM {SchemaName}.{DatabaseConstants.IncomingTable} WHERE id = :id"); + cmd.With("id", envelope.Id); + } + else + { + cmd = conn.CreateCommand( + $"SELECT COUNT(id) FROM {SchemaName}.{DatabaseConstants.IncomingTable} WHERE id = :id AND {DatabaseConstants.ReceivedAt} = :destination"); + cmd.With("id", envelope.Id); + cmd.With("destination", envelope.Destination!.ToString()); + } + + var count = await cmd.ExecuteScalarAsync(cancellation); + await conn.CloseAsync(); + + return Convert.ToInt64(count) > 0; + } + + public async Task RescheduleExistingEnvelopeForRetryAsync(Envelope envelope) + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand( + $"UPDATE {SchemaName}.{DatabaseConstants.IncomingTable} SET " + + $"{DatabaseConstants.ExecutionTime} = :time, {DatabaseConstants.Attempts} = :attempts " + + "WHERE id = :id"); + cmd.With("id", envelope.Id); + cmd.Parameters.Add(new OracleParameter("time", OracleDbType.TimeStampTZ) { Value = envelope.ScheduledTime }); + cmd.With("attempts", envelope.Attempts); + await cmd.ExecuteNonQueryAsync(_cancellation); + await conn.CloseAsync(); + } + + public async Task ScheduleExecutionAsync(Envelope envelope) + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand( + $"UPDATE {SchemaName}.{DatabaseConstants.IncomingTable} SET " + + $"{DatabaseConstants.ExecutionTime} = :time, {DatabaseConstants.Status} = '{EnvelopeStatus.Scheduled}', " + + $"{DatabaseConstants.Attempts} = :attempts, {DatabaseConstants.OwnerId} = {TransportConstants.AnyNode} " + + $"WHERE id = :id AND {DatabaseConstants.ReceivedAt} = :uri"); + cmd.With("id", envelope.Id); + cmd.Parameters.Add(new OracleParameter("time", OracleDbType.TimeStampTZ) { Value = envelope.ScheduledTime!.Value }); + cmd.With("attempts", envelope.Attempts); + cmd.With("uri", envelope.Destination?.ToString() ?? string.Empty); + await cmd.ExecuteNonQueryAsync(_cancellation); + await conn.CloseAsync(); + } + + public async Task MoveToDeadLetterStorageAsync(Envelope envelope, Exception? exception) + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var tx = (OracleTransaction)await conn.BeginTransactionAsync(_cancellation); + + // Delete from incoming + var deleteCmd = conn.CreateCommand( + $"DELETE FROM {SchemaName}.{DatabaseConstants.IncomingTable} WHERE id = :id"); + deleteCmd.Transaction = tx; + deleteCmd.With("id", envelope.Id); + await deleteCmd.ExecuteNonQueryAsync(_cancellation); + + // Insert into dead letters + byte[] data; + try + { + data = EnvelopeSerializer.Serialize(envelope); + } + catch + { + data = Array.Empty(); + } + + var deadLetterFields = DatabaseConstants.DeadLetterFields; + var deadLetterValues = + ":id, :executionTime, :body, :messageType, :receivedAt, :source, :exceptionType, :exceptionMessage, :sentAt, :replayable"; + + if (_durability.DeadLetterQueueExpirationEnabled) + { + deadLetterFields += ", " + DatabaseConstants.Expires; + deadLetterValues += ", :expires"; + } + + var insertCmd = conn.CreateCommand( + $"INSERT INTO {SchemaName}.{DatabaseConstants.DeadLetterTable} ({deadLetterFields}) " + + $"VALUES ({deadLetterValues})"); + insertCmd.Transaction = tx; + + insertCmd.With("id", envelope.Id); + insertCmd.Parameters.Add(new OracleParameter("executionTime", OracleDbType.TimeStampTZ) { Value = (object?)envelope.ScheduledTime ?? DBNull.Value }); + insertCmd.Parameters.Add(new OracleParameter("body", OracleDbType.Blob) { Value = data }); + insertCmd.With("messageType", envelope.MessageType ?? string.Empty); + insertCmd.With("receivedAt", envelope.Destination?.ToString() ?? string.Empty); + insertCmd.With("source", envelope.Source ?? string.Empty); + insertCmd.With("exceptionType", exception?.GetType().FullNameInCode() ?? string.Empty); + insertCmd.With("exceptionMessage", exception?.Message ?? string.Empty); + insertCmd.Parameters.Add(new OracleParameter("sentAt", OracleDbType.TimeStampTZ) { Value = envelope.SentAt.ToUniversalTime() }); + insertCmd.With("replayable", 0); // Oracle stores bool as NUMBER(1) + + if (_durability.DeadLetterQueueExpirationEnabled) + { + var expiration = envelope.DeliverBy ?? DateTimeOffset.UtcNow.Add(_durability.DeadLetterQueueExpiration); + insertCmd.Parameters.Add(new OracleParameter("expires", OracleDbType.TimeStampTZ) { Value = expiration }); + } + + await insertCmd.ExecuteNonQueryAsync(_cancellation); + + await tx.CommitAsync(_cancellation); + await conn.CloseAsync(); + } + + public async Task IncrementIncomingEnvelopeAttemptsAsync(Envelope envelope) + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand( + $"UPDATE {SchemaName}.{DatabaseConstants.IncomingTable} SET {DatabaseConstants.Attempts} = :attempts " + + $"WHERE id = :id AND {DatabaseConstants.ReceivedAt} = :uri"); + cmd.With("attempts", envelope.Attempts); + cmd.With("id", envelope.Id); + cmd.With("uri", envelope.Destination?.ToString() ?? string.Empty); + await cmd.ExecuteNonQueryAsync(_cancellation); + await conn.CloseAsync(); + } + + public async Task MarkIncomingEnvelopeAsHandledAsync(Envelope envelope) + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand( + $"UPDATE {SchemaName}.{DatabaseConstants.IncomingTable} SET " + + $"{DatabaseConstants.Status} = '{EnvelopeStatus.Handled}', {DatabaseConstants.KeepUntil} = :keepUntil " + + $"WHERE id = :id AND {DatabaseConstants.ReceivedAt} = :uri"); + cmd.Parameters.Add(new OracleParameter("keepUntil", OracleDbType.TimeStampTZ) { Value = (object?)envelope.KeepUntil ?? DBNull.Value }); + cmd.With("id", envelope.Id); + cmd.With("uri", envelope.Destination?.ToString() ?? string.Empty); + await cmd.ExecuteNonQueryAsync(_cancellation); + await conn.CloseAsync(); + } + + public async Task MarkIncomingEnvelopeAsHandledAsync(IReadOnlyList envelopes) + { + if (envelopes.Count == 0) return; + + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var tx = (OracleTransaction)await conn.BeginTransactionAsync(_cancellation); + + foreach (var envelope in envelopes) + { + var cmd = conn.CreateCommand( + $"UPDATE {SchemaName}.{DatabaseConstants.IncomingTable} SET " + + $"{DatabaseConstants.Status} = '{EnvelopeStatus.Handled}', {DatabaseConstants.KeepUntil} = :keepUntil " + + $"WHERE id = :id AND {DatabaseConstants.ReceivedAt} = :uri"); + cmd.Transaction = tx; + cmd.Parameters.Add(new OracleParameter("keepUntil", OracleDbType.TimeStampTZ) { Value = (object?)envelope.KeepUntil ?? DBNull.Value }); + cmd.With("id", envelope.Id); + cmd.With("uri", envelope.Destination?.ToString() ?? string.Empty); + await cmd.ExecuteNonQueryAsync(_cancellation); + } + + await tx.CommitAsync(_cancellation); + await conn.CloseAsync(); + } + + public async Task ReleaseIncomingAsync(int ownerId, Uri receivedAt) + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand( + $"UPDATE {SchemaName}.{DatabaseConstants.IncomingTable} SET " + + $"{DatabaseConstants.OwnerId} = 0 " + + $"WHERE {DatabaseConstants.OwnerId} = :ownerId AND {DatabaseConstants.ReceivedAt} = :uri"); + cmd.With("ownerId", ownerId); + cmd.With("uri", receivedAt.ToString()); + await cmd.ExecuteNonQueryAsync(_cancellation); + await conn.CloseAsync(); + } + + public async Task> LoadPageOfGloballyOwnedIncomingAsync(Uri listenerAddress, int limit) + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand( + $"SELECT {DatabaseConstants.IncomingFields} FROM {SchemaName}.{DatabaseConstants.IncomingTable} " + + $"WHERE owner_id = {TransportConstants.AnyNode} AND status = '{EnvelopeStatus.Incoming}' AND {DatabaseConstants.ReceivedAt} = :address " + + $"FETCH FIRST :limit ROWS ONLY"); + cmd.With("address", listenerAddress.ToString()); + cmd.With("limit", limit); + + var list = await cmd.FetchListAsync(r => OracleEnvelopeReader.ReadIncomingAsync(r), _cancellation); + await conn.CloseAsync(); + return list; + } + + public async Task ReassignIncomingAsync(int ownerId, IReadOnlyList incoming) + { + if (incoming.Count == 0) return; + + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var tx = (OracleTransaction)await conn.BeginTransactionAsync(_cancellation); + + foreach (var envelope in incoming) + { + var cmd = conn.CreateCommand( + $"UPDATE {SchemaName}.{DatabaseConstants.IncomingTable} SET {DatabaseConstants.OwnerId} = :owner " + + "WHERE id = :id"); + cmd.Transaction = tx; + cmd.With("owner", ownerId); + cmd.With("id", envelope.Id); + await cmd.ExecuteNonQueryAsync(_cancellation); + } + + await tx.CommitAsync(_cancellation); + await conn.CloseAsync(); + } + + // IMessageDatabase + public async Task StoreIncomingAsync(DbTransaction tx, Envelope[] envelopes) + { + foreach (var envelope in envelopes) + { + var data = envelope.Status == EnvelopeStatus.Handled + ? Array.Empty() + : EnvelopeSerializer.Serialize(envelope); + + var cmd = ((OracleConnection)tx.Connection!).CreateCommand( + $"INSERT INTO {SchemaName}.{DatabaseConstants.IncomingTable} ({DatabaseConstants.IncomingFields}) " + + "VALUES (:body, :id, :status, :ownerId, :executionTime, :attempts, :messageType, :receivedAt, :keepUntil)"); + cmd.Transaction = (OracleTransaction)tx; + + cmd.Parameters.Add(new OracleParameter("body", OracleDbType.Blob) { Value = data }); + cmd.With("id", envelope.Id); + cmd.With("status", envelope.Status.ToString()); + cmd.With("ownerId", envelope.OwnerId); + cmd.Parameters.Add(new OracleParameter("executionTime", OracleDbType.TimeStampTZ) { Value = (object?)envelope.ScheduledTime ?? DBNull.Value }); + cmd.With("attempts", envelope.Attempts); + cmd.With("messageType", envelope.MessageType!); + cmd.With("receivedAt", envelope.Destination?.ToString() ?? string.Empty); + cmd.Parameters.Add(new OracleParameter("keepUntil", OracleDbType.TimeStampTZ) { Value = (object?)envelope.KeepUntil ?? DBNull.Value }); + + try + { + await cmd.ExecuteNonQueryAsync(_cancellation); + } + catch (OracleException e) when (e.Number == 1) + { + // Idempotent + } + } + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Outgoing.cs b/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Outgoing.cs new file mode 100644 index 000000000..36c3c6986 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Outgoing.cs @@ -0,0 +1,144 @@ +using System.Data.Common; +using Oracle.ManagedDataAccess.Client; +using Weasel.Oracle; +using Wolverine.Oracle.Util; +using Wolverine.RDBMS; +using Wolverine.Runtime.Serialization; +using Wolverine.Transports; + +namespace Wolverine.Oracle; + +internal partial class OracleMessageStore +{ + public async Task> LoadOutgoingAsync(Uri destination) + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand( + $"SELECT {DatabaseConstants.OutgoingFields} FROM {SchemaName}.{DatabaseConstants.OutgoingTable} " + + $"WHERE owner_id = {TransportConstants.AnyNode} AND destination = :destination " + + $"FETCH FIRST {_durability.RecoveryBatchSize} ROWS ONLY"); + cmd.With("destination", destination.ToString()); + + var list = await cmd.FetchListAsync(r => OracleEnvelopeReader.ReadOutgoingAsync(r), _cancellation); + await conn.CloseAsync(); + return list; + } + + public async Task StoreOutgoingAsync(Envelope envelope, int ownerId) + { + var data = EnvelopeSerializer.Serialize(envelope); + + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand( + $"INSERT INTO {SchemaName}.{DatabaseConstants.OutgoingTable} ({DatabaseConstants.OutgoingFields}) " + + "VALUES (:body, :id, :ownerId, :destination, :deliverBy, :attempts, :messageType)"); + + cmd.Parameters.Add(new OracleParameter("body", OracleDbType.Blob) { Value = data }); + cmd.With("id", envelope.Id); + cmd.With("ownerId", ownerId); + cmd.With("destination", envelope.Destination!.ToString()); + cmd.Parameters.Add(new OracleParameter("deliverBy", OracleDbType.TimeStampTZ) { Value = (object?)envelope.DeliverBy ?? DBNull.Value }); + cmd.With("attempts", envelope.Attempts); + cmd.With("messageType", envelope.MessageType!); + + try + { + await cmd.ExecuteNonQueryAsync(_cancellation); + } + catch (OracleException e) when (e.Number == 1) + { + // Idempotent + } + finally + { + await conn.CloseAsync(); + } + } + + public async Task DeleteOutgoingAsync(Envelope[] envelopes) + { + if (HasDisposed) return; + if (envelopes.Length == 0) return; + + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand(""); + var placeholders = OracleCommandExtensions.WithEnvelopeIds(cmd, "id", envelopes); + cmd.CommandText = $"DELETE FROM {SchemaName}.{DatabaseConstants.OutgoingTable} WHERE id IN ({placeholders})"; + await cmd.ExecuteNonQueryAsync(_cancellation); + await conn.CloseAsync(); + } + + public async Task DeleteOutgoingAsync(Envelope envelope) + { + if (HasDisposed) return; + + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand( + $"DELETE FROM {SchemaName}.{DatabaseConstants.OutgoingTable} WHERE id = :id"); + cmd.With("id", envelope.Id); + await cmd.ExecuteNonQueryAsync(_cancellation); + await conn.CloseAsync(); + } + + public async Task DiscardAndReassignOutgoingAsync(Envelope[] discards, Envelope[] reassigned, int nodeId) + { + await using var conn = await _dataSource.OpenConnectionAsync(_cancellation); + + try + { + if (discards.Length > 0) + { + var deleteCmd = conn.CreateCommand(""); + var deletePlaceholders = OracleCommandExtensions.WithEnvelopeIds(deleteCmd, "id", discards); + deleteCmd.CommandText = + $"DELETE FROM {SchemaName}.{DatabaseConstants.OutgoingTable} WHERE id IN ({deletePlaceholders})"; + await deleteCmd.ExecuteNonQueryAsync(_cancellation); + } + + if (reassigned.Length > 0) + { + var reassignCmd = conn.CreateCommand(""); + var reassignPlaceholders = OracleCommandExtensions.WithEnvelopeIds(reassignCmd, "rid", reassigned); + reassignCmd.CommandText = + $"UPDATE {SchemaName}.{DatabaseConstants.OutgoingTable} SET owner_id = :node WHERE id IN ({reassignPlaceholders})"; + reassignCmd.With("node", nodeId); + await reassignCmd.ExecuteNonQueryAsync(_cancellation); + } + } + finally + { + await conn.CloseAsync(); + } + } + + // IMessageDatabase + public async Task StoreOutgoingAsync(DbTransaction tx, Envelope[] envelopes) + { + foreach (var envelope in envelopes) + { + var data = EnvelopeSerializer.Serialize(envelope); + + var cmd = ((OracleConnection)tx.Connection!).CreateCommand( + $"INSERT INTO {SchemaName}.{DatabaseConstants.OutgoingTable} ({DatabaseConstants.OutgoingFields}) " + + "VALUES (:body, :id, :ownerId, :destination, :deliverBy, :attempts, :messageType)"); + cmd.Transaction = (OracleTransaction)tx; + + cmd.Parameters.Add(new OracleParameter("body", OracleDbType.Blob) { Value = data }); + cmd.With("id", envelope.Id); + cmd.With("ownerId", envelope.OwnerId); + cmd.With("destination", envelope.Destination!.ToString()); + cmd.Parameters.Add(new OracleParameter("deliverBy", OracleDbType.TimeStampTZ) { Value = (object?)envelope.DeliverBy ?? DBNull.Value }); + cmd.With("attempts", envelope.Attempts); + cmd.With("messageType", envelope.MessageType!); + + try + { + await cmd.ExecuteNonQueryAsync(_cancellation); + } + catch (OracleException e) when (e.Number == 1) + { + // Idempotent + } + } + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Scheduled.cs b/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Scheduled.cs new file mode 100644 index 000000000..e0e03f9f7 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.Scheduled.cs @@ -0,0 +1,79 @@ +using Microsoft.Extensions.Logging; +using Oracle.ManagedDataAccess.Client; +using Weasel.Oracle; +using Wolverine.Oracle.Util; +using Wolverine.RDBMS; +using Wolverine.Runtime; + +namespace Wolverine.Oracle; + +internal partial class OracleMessageStore +{ + public async Task PollForScheduledMessagesAsync(IWolverineRuntime runtime, ILogger logger, + DurabilitySettings durabilitySettings, CancellationToken cancellationToken) + { + if (HasDisposed) return; + + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + try + { + var tx = (OracleTransaction)await conn.BeginTransactionAsync(cancellationToken); + + // Try to attain a row-level lock for scheduled jobs + var lockCmd = conn.CreateCommand( + $"SELECT lock_id FROM {SchemaName}.{Schema.LockTable.TableName} WHERE lock_id = :lockId FOR UPDATE NOWAIT"); + lockCmd.Transaction = tx; + lockCmd.With("lockId", _settings.ScheduledJobLockId); + + bool gotLock; + try + { + await lockCmd.ExecuteScalarAsync(cancellationToken); + gotLock = true; + } + catch (OracleException ex) when (ex.Number == 54) // ORA-00054: resource busy + { + gotLock = false; + await tx.RollbackAsync(cancellationToken); + } + + if (gotLock) + { + var builder = ToOracleCommandBuilder(); + builder.Append( + $"SELECT {DatabaseConstants.IncomingFields} FROM {SchemaName}.{DatabaseConstants.IncomingTable} WHERE status = '{EnvelopeStatus.Scheduled}' AND execution_time <= "); + builder.AppendParameter(DateTimeOffset.UtcNow); + builder.Append($" ORDER BY execution_time FETCH FIRST {_durability.RecoveryBatchSize} ROWS ONLY"); + var cmd = builder.Compile(); + cmd.Connection = conn; + cmd.Transaction = tx; + + var envelopes = await cmd.FetchListAsync( + reader => OracleEnvelopeReader.ReadIncomingAsync(reader, cancellationToken), cancellationToken); + + if (envelopes.Count == 0) + { + await tx.RollbackAsync(cancellationToken); + return; + } + + var ids = envelopes.Select(x => x.Id).ToArray(); + var reassignCmd = conn.CreateCommand(""); + reassignCmd.Transaction = tx; + var placeholders = OracleCommandExtensions.WithIdList(reassignCmd, "id", ids); + reassignCmd.CommandText = + $"UPDATE {SchemaName}.{DatabaseConstants.IncomingTable} SET owner_id = :owner, status = '{EnvelopeStatus.Incoming}' WHERE id IN ({placeholders})"; + reassignCmd.With("owner", durabilitySettings.AssignedNodeNumber); + await reassignCmd.ExecuteNonQueryAsync(_cancellation); + + await tx.CommitAsync(cancellationToken); + + await runtime.EnqueueDirectlyAsync(envelopes); + } + } + finally + { + await conn.CloseAsync(); + } + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.cs b/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.cs new file mode 100644 index 000000000..254ba1c8c --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/OracleMessageStore.cs @@ -0,0 +1,522 @@ +using System.Data.Common; +using ImTools; +using JasperFx; +using JasperFx.Core; +using JasperFx.Core.Reflection; +using JasperFx.MultiTenancy; +using JasperFx.Descriptors; +using Microsoft.Extensions.Logging; +using Oracle.ManagedDataAccess.Client; +using Weasel.Core; +using Weasel.Core.Migrations; +using Weasel.Oracle; +using Weasel.Oracle.Tables; +using Wolverine.Logging; +using Wolverine.Oracle.Sagas; +using Wolverine.Oracle.Schema; +using Wolverine.Persistence; +using Wolverine.Persistence.Durability; +using Wolverine.Persistence.Sagas; +using Wolverine.RDBMS; +using Wolverine.RDBMS.MultiTenancy; +using Wolverine.RDBMS.Sagas; +using Wolverine.RDBMS.Polling; +using Wolverine.RDBMS.Transport; +using Wolverine.Runtime; +using Wolverine.Runtime.Agents; +using Wolverine.Runtime.Serialization; +using Wolverine.Runtime.WorkerQueues; +using Wolverine.Transports; +using DbCommandBuilder = Weasel.Core.DbCommandBuilder; +using Table = Weasel.Oracle.Tables.Table; + +namespace Wolverine.Oracle; + +internal partial class OracleMessageStore : IMessageDatabase, IMessageInbox, IMessageOutbox, IMessageStoreAdmin, IDeadLetters, ISagaSupport +{ + private readonly OracleDataSource _dataSource; + private readonly DatabaseSettings _settings; + private readonly DurabilitySettings _durability; + private readonly ILogger _logger; + private readonly CancellationToken _cancellation; + private ImHashMap _sagaStorage = ImHashMap.Empty; + private readonly List _otherTables = new(); + private bool _hasDisposed; + private string _schemaName; + private INodeAgentPersistence? _nodes; + + public OracleMessageStore(DatabaseSettings databaseSettings, DurabilitySettings durability, + OracleDataSource dataSource, ILogger logger) + : this(databaseSettings, durability, dataSource, logger, Array.Empty()) + { + } + + public OracleMessageStore(DatabaseSettings databaseSettings, DurabilitySettings durability, + OracleDataSource dataSource, ILogger logger, IEnumerable sagaTypes) + { + _settings = databaseSettings; + _durability = durability; + _dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource)); + _logger = logger; + _cancellation = durability.Cancellation; + _schemaName = databaseSettings.SchemaName ?? durability.MessageStorageSchemaName ?? "WOLVERINE"; + _schemaName = _schemaName.ToUpperInvariant(); + + Role = databaseSettings.Role; + Settings = databaseSettings; + + AdvisoryLock = new OracleAdvisoryLock(_dataSource, logger, _schemaName); + + foreach (var sagaTableDefinition in sagaTypes) + { + var storage = typeof(OracleSagaSchema<,>).CloseAndBuildAs(sagaTableDefinition, + _settings, sagaTableDefinition.SagaType, sagaTableDefinition.IdMember.GetMemberType()); + _sagaStorage = _sagaStorage.AddOrUpdate(sagaTableDefinition.SagaType, storage); + } + + if (Role == MessageStoreRole.Main) + { + _nodes = new OracleNodePersistence(_settings, this, _dataSource); + } + + var descriptor = Describe(); + var parts = new List + { + "oracle", + descriptor.ServerName.Split(',')[0], + descriptor.DatabaseName, + _schemaName + }; + + if (Role == MessageStoreRole.Main) + { + Uri = new Uri("wolverine://messages/main"); + } + else + { + Uri = new Uri($"messagedb://{parts.Join("/")}"); + } + + Name = Uri.ToString(); + } + + public OracleDataSource OracleDataSource => _dataSource; + public DurabilitySettings Durability => _durability; + + // IMessageStore + public MessageStoreRole Role { get; set; } + public List TenantIds { get; } = new(); + public Uri Uri { get; internal set; } + public bool HasDisposed => _hasDisposed; + public IMessageInbox Inbox => this; + public IMessageOutbox Outbox => this; + public INodeAgentPersistence Nodes => _nodes!; + public IMessageStoreAdmin Admin => this; + public IDeadLetters DeadLetters => this; + public string Name { get; set; } + public IAdvisoryLock AdvisoryLock { get; } + + // IMessageDatabase + public DatabaseSettings Settings { get; } + public string SchemaName + { + get => _schemaName; + set => _schemaName = value.ToUpperInvariant(); + } + public DbDataSource DataSource => _dataSource; + public ILogger Logger => _logger; + + public void Initialize(IWolverineRuntime runtime) + { + // No-op; initialization happens in MigrateAsync + } + + public IAgent BuildAgent(IWolverineRuntime runtime) + { + return new DurabilityAgent(runtime, this); + } + + public IAgent StartScheduledJobs(IWolverineRuntime runtime) + { + var agent = new DurabilityAgent(runtime, this); + agent.StartScheduledJobPolling(); + return agent; + } + + public DatabaseDescriptor Describe() + { + var builder = new OracleConnectionStringBuilder(_dataSource.ConnectionString); + var descriptor = new DatabaseDescriptor + { + Engine = "Oracle", + ServerName = builder.DataSource ?? string.Empty, + DatabaseName = _schemaName, + Subject = GetType().FullNameInCode(), + SubjectUri = Uri + }; + + descriptor.TenantIds.AddRange(TenantIds); + + return descriptor; + } + + public Task DrainAsync() + { + return Task.CompletedTask; + } + + public void PromoteToMain(IWolverineRuntime runtime) + { + Role = MessageStoreRole.Main; + Uri = new Uri("wolverine://messages/main"); + _nodes ??= new OracleNodePersistence(_settings, this, _dataSource); + } + + public void DemoteToAncillary() + { + Role = MessageStoreRole.Ancillary; + } + + public async ValueTask DisposeAsync() + { + _hasDisposed = true; + await AdvisoryLock.DisposeAsync(); + } + + public OracleConnection CreateConnection() + { + return _dataSource.CreateConnection(); + } + + public async Task CreateCommand(string sql) + { + var conn = await _dataSource.OpenConnectionAsync(_cancellation); + var cmd = conn.CreateCommand(sql); + return cmd; + } + + public IEnumerable AllObjects() + { + yield return new OutgoingEnvelopeTable(_durability, SchemaName); + yield return new IncomingEnvelopeTable(_durability, SchemaName); + yield return new DeadLettersTable(_durability, SchemaName); + yield return new LockTable(SchemaName); + + if (Role == MessageStoreRole.Main) + { + var nodeTable = new Table(new OracleObjectName(SchemaName, DatabaseConstants.NodeTableName.ToUpperInvariant())); + nodeTable.AddColumn("id").AsPrimaryKey(); + nodeTable.AddColumn("node_number", "NUMBER(10) GENERATED BY DEFAULT AS IDENTITY").NotNull() + .AddIndex(idx => idx.IsUnique = true); + nodeTable.AddColumn("description", "VARCHAR2(4000)").NotNull(); + nodeTable.AddColumn("uri", "VARCHAR2(500)").NotNull(); + nodeTable.AddColumn("started") + .DefaultValueByExpression("SYS_EXTRACT_UTC(SYSTIMESTAMP)").NotNull(); + nodeTable.AddColumn("health_check").NotNull() + .DefaultValueByExpression("SYS_EXTRACT_UTC(SYSTIMESTAMP)"); + nodeTable.AddColumn("version", "VARCHAR2(4000)"); + nodeTable.AddColumn("capabilities", "VARCHAR2(4000)").AllowNulls(); + + yield return nodeTable; + + var assignmentTable = new Table(new OracleObjectName(SchemaName, DatabaseConstants.NodeAssignmentsTableName.ToUpperInvariant())); + assignmentTable.AddColumn("id", "VARCHAR2(500)").AsPrimaryKey(); + assignmentTable.AddColumn("node_id") + .ForeignKeyTo(nodeTable.Identifier, "id", onDelete: CascadeAction.Cascade); + assignmentTable.AddColumn("started") + .DefaultValueByExpression("SYS_EXTRACT_UTC(SYSTIMESTAMP)").NotNull(); + + yield return assignmentTable; + + if (_settings.CommandQueuesEnabled) + { + var queueTable = new Table(new OracleObjectName(SchemaName, DatabaseConstants.ControlQueueTableName.ToUpperInvariant())); + queueTable.AddColumn("id").AsPrimaryKey(); + queueTable.AddColumn("message_type", "VARCHAR2(4000)").NotNull(); + queueTable.AddColumn("node_id").NotNull(); + queueTable.AddColumn(DatabaseConstants.Body, "BLOB").NotNull(); + queueTable.AddColumn("posted").NotNull() + .DefaultValueByExpression("SYS_EXTRACT_UTC(SYSTIMESTAMP)"); + queueTable.AddColumn("expires"); + + yield return queueTable; + } + + if (_settings.AddTenantLookupTable) + { + var tenantTable = new Table(new OracleObjectName(SchemaName, DatabaseConstants.TenantsTableName.ToUpperInvariant())); + tenantTable.AddColumn("tenant_id", "VARCHAR2(100)").AsPrimaryKey(); + tenantTable.AddColumn("connection_string", "VARCHAR2(500)").NotNull(); + yield return tenantTable; + } + + var eventTable = new Table(new OracleObjectName(SchemaName, DatabaseConstants.NodeRecordTableName.ToUpperInvariant())); + eventTable.AddColumn("id", "NUMBER(10) GENERATED BY DEFAULT AS IDENTITY").AsPrimaryKey(); + eventTable.AddColumn("node_number").NotNull(); + eventTable.AddColumn("event_name", "VARCHAR2(500)").NotNull(); + eventTable.AddColumn("timestamp") + .DefaultValueByExpression("SYS_EXTRACT_UTC(SYSTIMESTAMP)").NotNull(); + eventTable.AddColumn("description", "VARCHAR2(500)").AllowNulls(); + yield return eventTable; + + var restrictionTable = new Table(new OracleObjectName(SchemaName, DatabaseConstants.AgentRestrictionsTableName.ToUpperInvariant())); + restrictionTable.AddColumn("id").AsPrimaryKey(); + restrictionTable.AddColumn("uri", "VARCHAR2(4000)").NotNull(); + restrictionTable.AddColumn("type", "VARCHAR2(4000)").NotNull(); + restrictionTable.AddColumn("node").NotNull().DefaultValue(0); + yield return restrictionTable; + } + + foreach (var table in _otherTables) + { + yield return table; + } + + foreach (var entry in _sagaStorage.Enumerate()) + { + yield return entry.Value.Table; + } + } + + public void AddTable(Table table) + { + _otherTables.Add(table); + } + + public OracleSagaSchema SagaSchemaFor() where T : Saga + { + if (_sagaStorage.TryFind(typeof(T), out var raw)) + { + if (raw is OracleSagaSchema sagaStorage) + { + return sagaStorage; + } + } + + var definition = new SagaTableDefinition(typeof(T), null); + var storage = new OracleSagaSchema(definition, _settings); + _sagaStorage = _sagaStorage.AddOrUpdate(typeof(T), storage); + + return storage; + } + + // IMessageDatabase - extra methods + public Weasel.Core.DbCommandBuilder ToCommandBuilder() + { + // The IMessageDatabase interface requires DbCommandBuilder, but we create an OracleCommandBuilder + // internally. Return a DbCommandBuilder that uses Oracle's OracleCommand as the underlying command. + // Our dead letter methods use ToOracleCommandBuilder() instead. + return new Weasel.Core.DbCommandBuilder(CreateConnection()); + } + + internal Weasel.Oracle.CommandBuilder ToOracleCommandBuilder() + { + return new Weasel.Oracle.CommandBuilder(); + } + + public Task EnqueueAsync(IDatabaseOperation operation) + { + // For Oracle, we execute operations directly since we can't batch + return Task.CompletedTask; + } + + public void WriteLoadScheduledEnvelopeSql(DbCommandBuilder builder, DateTimeOffset utcNow) + { + builder.Append( + $"SELECT {DatabaseConstants.IncomingFields} FROM {SchemaName}.{DatabaseConstants.IncomingTable} WHERE status = '{EnvelopeStatus.Scheduled}' AND execution_time <= "); + builder.AppendParameter(utcNow); + builder.Append($" ORDER BY execution_time FETCH FIRST {_durability.RecoveryBatchSize} ROWS ONLY"); + } + + public Task PollForMessagesFromExternalTablesAsync(IListener listener, IWolverineRuntime settings, + ExternalMessageTable externalTable, IReceiver receiver, CancellationToken token) + { + // Not implemented yet + return Task.CompletedTask; + } + + public async Task MigrateExternalMessageTable(ExternalMessageTable definition) + { + var table = AddExternalMessageTable(definition); + await using var conn = CreateConnection(); + await conn.OpenAsync(); + + var migration = await SchemaMigration.DetermineAsync(conn, CancellationToken.None, table); + if (migration.Difference != SchemaPatchDifference.None) + { + await new OracleMigrator().ApplyAllAsync(conn, migration, AutoCreate.CreateOrUpdate); + } + + await conn.CloseAsync(); + } + + public async Task PublishMessageToExternalTableAsync(ExternalMessageTable table, string messageTypeName, + byte[] json, CancellationToken token) + { + await using var conn = CreateConnection(); + await conn.OpenAsync(token); + + var cmd = conn.CreateCommand(""); + if (table.MessageTypeColumnName.IsEmpty()) + { + cmd.CommandText = + $"INSERT INTO {table.TableName.QualifiedName} ({table.IdColumnName}, {table.JsonBodyColumnName}) VALUES (:id, :json)"; + cmd.With("id", Guid.NewGuid()); + cmd.Parameters.Add(new OracleParameter("json", OracleDbType.Blob) { Value = json }); + } + else + { + cmd.CommandText = + $"INSERT INTO {table.TableName.QualifiedName} ({table.IdColumnName}, {table.JsonBodyColumnName}, {table.MessageTypeColumnName}) VALUES (:id, :json, :message)"; + cmd.With("id", Guid.NewGuid()); + cmd.Parameters.Add(new OracleParameter("json", OracleDbType.Blob) { Value = json }); + cmd.With("message", messageTypeName); + } + + await cmd.ExecuteNonQueryAsync(token); + await conn.CloseAsync(); + } + + public ISchemaObject AddExternalMessageTable(ExternalMessageTable definition) + { + var table = new Table(definition.TableName); + table.AddColumn(definition.IdColumnName).AsPrimaryKey(); + table.AddColumn(definition.JsonBodyColumnName, "BLOB").NotNull(); + if (definition.TimestampColumnName.IsNotEmpty()) + { + table.AddColumn(definition.TimestampColumnName) + .DefaultValueByExpression("SYS_EXTRACT_UTC(SYSTIMESTAMP)"); + } + + if (definition.MessageTypeColumnName.IsNotEmpty()) + { + table.AddColumn(definition.MessageTypeColumnName); + } + + return table; + } + + // ISagaSupport + public async ValueTask> EnrollAndFetchSagaStorage(MessageContext context) + where TSaga : Saga + { + var conn = CreateConnection(); + await conn.OpenAsync(_cancellation); + try + { + var tx = await conn.BeginTransactionAsync(_cancellation); + + var schema = SagaSchemaFor(); + + var transaction = new DatabaseEnvelopeTransaction(this, tx); + await context.EnlistInOutboxAsync(transaction); + return new DatabaseSagaStorage(conn, tx, schema); + } + catch (Exception) + { + await conn.CloseAsync(); + throw; + } + } + + // ITenantDatabaseRegistry + public IDatabaseProvider Provider => OracleProvider.Instance; + + private bool _hasAppliedDefaults; + + public async Task TryFindTenantConnectionString(string tenantId) + { + if (!_hasAppliedDefaults && _settings.AutoCreate != AutoCreate.None && _settings.TenantConnections != null) + { + await SeedDatabasesAsync(_settings.TenantConnections); + _hasAppliedDefaults = true; + } + + await using var conn = CreateConnection(); + await conn.OpenAsync(_cancellation); + + try + { + var cmd = conn.CreateCommand( + $"SELECT connection_string FROM {SchemaName}.{DatabaseConstants.TenantsTableName} WHERE tenant_id = :id"); + cmd.With("id", tenantId); + await using var reader = await cmd.ExecuteReaderAsync(_cancellation); + + if (await reader.ReadAsync(_cancellation)) + { + return await reader.GetFieldValueAsync(0, _cancellation); + } + + await reader.CloseAsync(); + return null!; + } + finally + { + await conn.CloseAsync(); + } + } + + public async Task>> LoadAllTenantConnectionStrings() + { + if (!_hasAppliedDefaults && _settings.AutoCreate != AutoCreate.None && _settings.TenantConnections != null) + { + await SeedDatabasesAsync(_settings.TenantConnections); + _hasAppliedDefaults = true; + } + + var list = new List>(); + + await using var conn = CreateConnection(); + await conn.OpenAsync(_cancellation); + + try + { + var cmd = conn.CreateCommand( + $"SELECT tenant_id, connection_string FROM {SchemaName}.{DatabaseConstants.TenantsTableName}"); + await using var reader = await cmd.ExecuteReaderAsync(_cancellation); + + while (await reader.ReadAsync(_cancellation)) + { + var tid = await reader.GetFieldValueAsync(0); + var cs = await reader.GetFieldValueAsync(1); + + list.Add(new Assignment(tid, cs)); + } + + await reader.CloseAsync(); + } + finally + { + await conn.CloseAsync(); + } + + return list; + } + + public async Task SeedDatabasesAsync(ITenantedSource tenantConnectionStrings) + { + await using var conn = CreateConnection(); + await conn.OpenAsync(_cancellation); + + try + { + foreach (var assignment in tenantConnectionStrings.AllActiveByTenant()) + { + var deleteCmd = conn.CreateCommand( + $"DELETE FROM {SchemaName}.{DatabaseConstants.TenantsTableName} WHERE tenant_id = :tid"); + deleteCmd.With("tid", assignment.TenantId); + await deleteCmd.ExecuteNonQueryAsync(_cancellation); + + var insertCmd = conn.CreateCommand( + $"INSERT INTO {SchemaName}.{DatabaseConstants.TenantsTableName} (tenant_id, connection_string) VALUES (:tid, :cs)"); + insertCmd.With("tid", assignment.TenantId); + insertCmd.With("cs", assignment.Value); + await insertCmd.ExecuteNonQueryAsync(_cancellation); + } + } + finally + { + await conn.CloseAsync(); + } + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/OracleNodePersistence.cs b/src/Persistence/Oracle/Wolverine.Oracle/OracleNodePersistence.cs new file mode 100644 index 000000000..897538ea2 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/OracleNodePersistence.cs @@ -0,0 +1,396 @@ +using System.Data; +using System.Data.Common; +using JasperFx.Core; +using Microsoft.Extensions.Logging; +using Oracle.ManagedDataAccess.Client; +using Weasel.Core; +using Weasel.Oracle; +using Wolverine.Oracle.Util; +using Wolverine.RDBMS; +using Wolverine.RDBMS.Durability; +using Wolverine.Runtime.Agents; +using Wolverine.Transports; + +namespace Wolverine.Oracle; + +internal class OracleNodePersistence : DatabaseConstants, INodeAgentPersistence +{ + public static int LeaderLockId = 9999999; + private readonly DbObjectName _assignmentTable; + private readonly IMessageDatabase _database; + private readonly OracleDataSource _dataSource; + private readonly int _lockId; + private readonly DbObjectName _nodeTable; + private readonly DatabaseSettings _settings; + private readonly DbObjectName _restrictionTable; + + public OracleNodePersistence(DatabaseSettings settings, OracleMessageStore database, OracleDataSource dataSource) + { + _settings = settings; + _database = database; + _dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource)); + var schemaName = (settings.SchemaName ?? "WOLVERINE").ToUpperInvariant(); + _nodeTable = new DbObjectName(schemaName, NodeTableName.ToUpperInvariant()); + _restrictionTable = new DbObjectName(schemaName, AgentRestrictionsTableName.ToUpperInvariant()); + _assignmentTable = new DbObjectName(schemaName, NodeAssignmentsTableName.ToUpperInvariant()); + + _lockId = schemaName.GetDeterministicHashCode(); + } + + public async Task ClearAllAsync(CancellationToken cancellationToken) + { + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + var cmd = conn.CreateCommand($"DELETE FROM {_nodeTable}"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + await conn.CloseAsync(); + } + + public async Task PersistAsync(WolverineNode node, CancellationToken cancellationToken) + { + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + + var capabilities = string.Join(",", node.Capabilities.Select(x => x.ToString())); + + var insertCmd = conn.CreateCommand( + $"INSERT INTO {_nodeTable} (id, uri, capabilities, description, version) VALUES (:id, :uri, :capabilities, :description, :version)"); + insertCmd.With("id", node.NodeId); + insertCmd.With("uri", (node.ControlUri ?? TransportConstants.LocalUri).ToString()); + insertCmd.With("capabilities", capabilities); + insertCmd.With("description", node.Description); + insertCmd.With("version", node.Version.ToString()); + + await insertCmd.ExecuteNonQueryAsync(cancellationToken); + + // Get the auto-generated node_number + var selectCmd = conn.CreateCommand($"SELECT node_number FROM {_nodeTable} WHERE id = :id"); + selectCmd.With("id", node.NodeId); + var result = await selectCmd.ExecuteScalarAsync(cancellationToken); + + await conn.CloseAsync(); + + return Convert.ToInt32(result); + } + + public async Task DeleteAsync(Guid nodeId, int assignedNodeNumber) + { + if (_database.HasDisposed) return; + + await using var conn = await _dataSource.OpenConnectionAsync(); + + var cmd1 = conn.CreateCommand($"DELETE FROM {_nodeTable} WHERE id = :id"); + cmd1.With("id", nodeId); + await cmd1.ExecuteNonQueryAsync(); + + var cmd2 = conn.CreateCommand( + $"UPDATE {_settings.SchemaName}.{IncomingTable} SET {OwnerId} = 0 WHERE {OwnerId} = :nodeNum"); + cmd2.With("nodeNum", assignedNodeNumber); + await cmd2.ExecuteNonQueryAsync(); + + var cmd3 = conn.CreateCommand( + $"UPDATE {_settings.SchemaName}.{OutgoingTable} SET {OwnerId} = 0 WHERE {OwnerId} = :nodeNum"); + cmd3.With("nodeNum", assignedNodeNumber); + await cmd3.ExecuteNonQueryAsync(); + + await conn.CloseAsync(); + } + + public async Task> LoadAllNodesAsync(CancellationToken cancellationToken) + { + var nodes = new List(); + + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + + var nodeCmd = conn.CreateCommand($"SELECT {NodeColumns} FROM {_nodeTable}"); + await using var nodeReader = await nodeCmd.ExecuteReaderAsync(cancellationToken); + while (await nodeReader.ReadAsync(cancellationToken)) + { + var node = await readNodeAsync(nodeReader); + nodes.Add(node); + } + await nodeReader.CloseAsync(); + + var dict = nodes.ToDictionary(x => x.NodeId); + + var assignCmd = conn.CreateCommand($"SELECT {Id}, {NodeId}, {Started} FROM {_assignmentTable}"); + await using var assignReader = await assignCmd.ExecuteReaderAsync(cancellationToken); + while (await assignReader.ReadAsync(cancellationToken)) + { + var agentId = new Uri(await assignReader.GetFieldValueAsync(0, cancellationToken)); + var nid = await OracleEnvelopeReader.ReadGuidAsync(assignReader, 1, cancellationToken); + + if (dict.TryGetValue(nid, out var node)) + { + node.ActiveAgents.Add(agentId); + } + } + await assignReader.CloseAsync(); + await conn.CloseAsync(); + + return nodes; + } + + public async Task PersistAgentRestrictionsAsync(IReadOnlyList restrictions, + CancellationToken cancellationToken) + { + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + + foreach (var restriction in restrictions) + { + if (restriction.Type == AgentRestrictionType.None) + { + var cmd = conn.CreateCommand($"DELETE FROM {_restrictionTable} WHERE id = :id"); + cmd.With("id", restriction.Id); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + else + { + var cmd = conn.CreateCommand( + $"MERGE INTO {_restrictionTable} t USING DUAL ON (t.id = :id) " + + "WHEN MATCHED THEN UPDATE SET t.node = :node " + + "WHEN NOT MATCHED THEN INSERT (id, uri, type, node) VALUES (:id, :uri, :type, :node)"); + cmd.With("id", restriction.Id); + cmd.With("uri", restriction.AgentUri.ToString()); + cmd.With("type", restriction.Type.ToString()); + cmd.With("node", restriction.NodeNumber); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + } + + await conn.CloseAsync(); + } + + public async Task LoadNodeAgentStateAsync(CancellationToken cancellationToken) + { + var nodes = new List(); + var restrictions = new List(); + + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + + var nodeCmd = conn.CreateCommand($"SELECT {NodeColumns} FROM {_nodeTable}"); + await using var nodeReader = await nodeCmd.ExecuteReaderAsync(cancellationToken); + while (await nodeReader.ReadAsync(cancellationToken)) + { + var node = await readNodeAsync(nodeReader); + nodes.Add(node); + } + await nodeReader.CloseAsync(); + + var dict = nodes.ToDictionary(x => x.NodeId); + + var assignCmd = conn.CreateCommand($"SELECT {Id}, {NodeId}, {Started} FROM {_assignmentTable}"); + await using var assignReader = await assignCmd.ExecuteReaderAsync(cancellationToken); + while (await assignReader.ReadAsync(cancellationToken)) + { + var agentId = new Uri(await assignReader.GetFieldValueAsync(0, cancellationToken)); + var nid = await OracleEnvelopeReader.ReadGuidAsync(assignReader, 1, cancellationToken); + + if (dict.TryGetValue(nid, out var node)) + { + node.ActiveAgents.Add(agentId); + } + } + await assignReader.CloseAsync(); + + var restrictCmd = conn.CreateCommand($"SELECT id, uri, type, node FROM {_restrictionTable}"); + await using var restrictReader = await restrictCmd.ExecuteReaderAsync(cancellationToken); + while (await restrictReader.ReadAsync(cancellationToken)) + { + var id = await OracleEnvelopeReader.ReadGuidAsync(restrictReader, 0, cancellationToken); + var uriString = await restrictReader.GetFieldValueAsync(1, cancellationToken); + var typeString = await restrictReader.GetFieldValueAsync(2, cancellationToken); + var nodeNumber = Convert.ToInt32(restrictReader.GetValue(3)); + + restrictions.Add(new AgentRestriction(id, new Uri(uriString), + Enum.Parse(typeString), nodeNumber)); + } + await restrictReader.CloseAsync(); + await conn.CloseAsync(); + + return new NodeAgentState(nodes, new AgentRestrictions(restrictions.ToArray())); + } + + public async Task LoadNodeAsync(Guid nodeId, CancellationToken cancellationToken) + { + if (_database.HasDisposed) return null; + + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + + WolverineNode? returnValue = null; + + var nodeCmd = conn.CreateCommand($"SELECT {NodeColumns} FROM {_nodeTable} WHERE id = :id"); + nodeCmd.With("id", nodeId); + await using var nodeReader = await nodeCmd.ExecuteReaderAsync(cancellationToken); + if (await nodeReader.ReadAsync(cancellationToken)) + { + returnValue = await readNodeAsync(nodeReader); + } + await nodeReader.CloseAsync(); + + if (returnValue != null) + { + var assignCmd = conn.CreateCommand( + $"SELECT {Id}, {NodeId}, {Started} FROM {_assignmentTable} WHERE node_id = :id"); + assignCmd.With("id", nodeId); + await using var assignReader = await assignCmd.ExecuteReaderAsync(cancellationToken); + while (await assignReader.ReadAsync(cancellationToken)) + { + var agentId = new Uri(await assignReader.GetFieldValueAsync(0, cancellationToken)); + returnValue.ActiveAgents.Add(agentId); + } + await assignReader.CloseAsync(); + } + + await conn.CloseAsync(); + return returnValue; + } + + public async Task AssignAgentsAsync(Guid nodeId, IReadOnlyList agents, CancellationToken cancellationToken) + { + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + + foreach (var agent in agents) + { + var cmd = conn.CreateCommand( + $"MERGE INTO {_assignmentTable} t USING DUAL ON (t.id = :id) " + + "WHEN MATCHED THEN UPDATE SET t.node_id = :node " + + "WHEN NOT MATCHED THEN INSERT (id, node_id) VALUES (:id, :node)"); + cmd.With("id", agent.ToString()); + cmd.With("node", nodeId); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + await conn.CloseAsync(); + } + + public async Task RemoveAssignmentAsync(Guid nodeId, Uri agentUri, CancellationToken cancellationToken) + { + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + var cmd = conn.CreateCommand($"DELETE FROM {_assignmentTable} WHERE id = :id AND node_id = :node"); + cmd.With("id", agentUri.ToString()); + cmd.With("node", nodeId); + await cmd.ExecuteNonQueryAsync(cancellationToken); + await conn.CloseAsync(); + } + + public async Task AddAssignmentAsync(Guid nodeId, Uri agentUri, CancellationToken cancellationToken) + { + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + var cmd = conn.CreateCommand( + $"MERGE INTO {_assignmentTable} t USING DUAL ON (t.id = :id) " + + "WHEN MATCHED THEN UPDATE SET t.node_id = :node " + + "WHEN NOT MATCHED THEN INSERT (id, node_id) VALUES (:id, :node)"); + cmd.With("id", agentUri.ToString()); + cmd.With("node", nodeId); + await cmd.ExecuteNonQueryAsync(cancellationToken); + await conn.CloseAsync(); + } + + public async Task OverwriteHealthCheckTimeAsync(Guid nodeId, DateTimeOffset lastHeartbeatTime) + { + await using var conn = await _dataSource.OpenConnectionAsync(); + var cmd = conn.CreateCommand($"UPDATE {_nodeTable} SET health_check = :now WHERE id = :id"); + cmd.With("id", nodeId); + cmd.With("now", lastHeartbeatTime); + await cmd.ExecuteNonQueryAsync(); + await conn.CloseAsync(); + } + + public async Task MarkHealthCheckAsync(WolverineNode node, CancellationToken token) + { + await using var conn = await _dataSource.OpenConnectionAsync(token); + var cmd = conn.CreateCommand( + $"UPDATE {_nodeTable} SET health_check = SYS_EXTRACT_UTC(SYSTIMESTAMP) WHERE id = :id"); + cmd.With("id", node.NodeId); + var count = await cmd.ExecuteNonQueryAsync(token); + + if (count == 0) + { + await conn.CloseAsync(); + await PersistAsync(node, token); + return; + } + + await conn.CloseAsync(); + } + + public Task LogRecordsAsync(params NodeRecord[] records) + { + if (records.Length == 0) return Task.CompletedTask; + + var op = new PersistNodeRecord(_settings, records); + return _database.EnqueueAsync(op); + } + + public async Task> FetchRecentRecordsAsync(int count) + { + if (count <= 0) throw new ArgumentOutOfRangeException(nameof(count), "Must be a positive number"); + + await using var conn = await _dataSource.OpenConnectionAsync(); + var cmd = conn.CreateCommand( + $"SELECT node_number, event_name, timestamp, description FROM {_settings.SchemaName}.{NodeRecordTableName} " + + $"ORDER BY id DESC FETCH FIRST :limit ROWS ONLY"); + cmd.With("limit", count); + + var list = new List(); + await using var reader = await cmd.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + list.Add(new NodeRecord + { + NodeNumber = Convert.ToInt32(reader.GetValue(0)), + RecordType = Enum.Parse(await reader.GetFieldValueAsync(1)), + Timestamp = await reader.GetFieldValueAsync(2), + Description = await reader.GetFieldValueAsync(3) + }); + } + await conn.CloseAsync(); + + return list; + } + + public bool HasLeadershipLock() + { + return _database.AdvisoryLock.HasLock(_lockId); + } + + public Task TryAttainLeadershipLockAsync(CancellationToken token) + { + return _database.AdvisoryLock.TryAttainLockAsync(_lockId, token); + } + + public Task ReleaseLeadershipLockAsync() + { + return _database.AdvisoryLock.ReleaseLockAsync(_lockId); + } + + private async Task readNodeAsync(DbDataReader reader) + { + var node = new WolverineNode + { + NodeId = OracleEnvelopeReader.ReadGuid(reader, 0), + AssignedNodeNumber = Convert.ToInt32(reader.GetValue(1)), + Description = await reader.GetFieldValueAsync(2), + ControlUri = (await reader.GetFieldValueAsync(3)).ToUri(), + Started = await reader.GetFieldValueAsync(4), + LastHealthCheck = await reader.GetFieldValueAsync(5) + }; + + if (!await reader.IsDBNullAsync(6)) + { + var rawVersion = await reader.GetFieldValueAsync(6); + node.Version = System.Version.Parse(rawVersion); + } + + if (!await reader.IsDBNullAsync(7)) + { + var capabilitiesStr = await reader.GetFieldValueAsync(7); + if (capabilitiesStr.IsNotEmpty()) + { + var capabilities = capabilitiesStr.Split(',', StringSplitOptions.RemoveEmptyEntries); + node.Capabilities.AddRange(capabilities.Select(x => new Uri(x.Trim()))); + } + } + + return node; + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/OracleTenantedMessageStore.cs b/src/Persistence/Oracle/Wolverine.Oracle/OracleTenantedMessageStore.cs new file mode 100644 index 000000000..0c9d6cdaf --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/OracleTenantedMessageStore.cs @@ -0,0 +1,123 @@ +using ImTools; +using JasperFx; +using JasperFx.Core; +using JasperFx.Descriptors; +using JasperFx.MultiTenancy; +using Microsoft.Extensions.Logging; +using Wolverine.Persistence.Durability; +using Wolverine.RDBMS; +using Wolverine.RDBMS.MultiTenancy; +using Wolverine.RDBMS.Sagas; +using Wolverine.Runtime; + +namespace Wolverine.Oracle; + +internal class OracleTenantedMessageStore : ITenantedMessageSource +{ + private readonly OracleBackedPersistence _persistence; + private readonly SagaTableDefinition[] _sagaTables; + private readonly IWolverineRuntime _runtime; + private ImHashMap _stores = ImHashMap.Empty; + + public OracleTenantedMessageStore(IWolverineRuntime runtime, OracleBackedPersistence persistence, + IEnumerable sagaTables) + { + _persistence = persistence; + _sagaTables = sagaTables.ToArray(); + _runtime = runtime; + } + + public DatabaseCardinality Cardinality => _persistence.ConnectionStringTenancy?.Cardinality ?? DatabaseCardinality.Single; + + public async ValueTask FindAsync(string tenantId) + { + if (_stores.TryFind(tenantId, out var store)) + { + return store; + } + + var connectionString = await _persistence.ConnectionStringTenancy!.FindAsync(tenantId); + store = buildTenantStoreForConnectionString(connectionString); + + store.TenantIds.Fill(tenantId); + + if (_runtime.Options.AutoBuildMessageStorageOnStartup != AutoCreate.None) + { + await store.Admin.MigrateAsync(); + } + + _stores = _stores.AddOrUpdate(tenantId, store); + return store; + } + + private OracleMessageStore buildTenantStoreForConnectionString(string connectionString) + { + var dataSource = new OracleDataSource(connectionString); + var settings = new DatabaseSettings + { + CommandQueuesEnabled = false, + ConnectionString = connectionString, + Role = MessageStoreRole.Tenant, + ScheduledJobLockId = _persistence.ScheduledJobLockId, + SchemaName = _persistence.EnvelopeStorageSchemaName + }; + + var store = new OracleMessageStore(settings, _runtime.Options.Durability, dataSource, + _runtime.LoggerFactory.CreateLogger(), _sagaTables); + store.Name = store.Describe().DatabaseUri().ToString(); + return store; + } + + public Task RefreshAsync() + { + return RefreshAsync(true); + } + + public Task RefreshLiteAsync() + { + return RefreshAsync(false); + } + + public async Task RefreshAsync(bool withMigration) + { + if (_persistence.ConnectionStringTenancy != null) + { + await _persistence.ConnectionStringTenancy.RefreshAsync(); + + foreach (var assignment in _persistence.ConnectionStringTenancy.AllActiveByTenant()) + { + if (!_stores.Contains(assignment.TenantId)) + { + var store = buildTenantStoreForConnectionString(assignment.Value); + store.TenantIds.Fill(assignment.TenantId); + + if (withMigration && _runtime.Options.AutoBuildMessageStorageOnStartup != AutoCreate.None) + { + await store.Admin.MigrateAsync(); + } + + _stores = _stores.AddOrUpdate(assignment.TenantId, store); + } + } + } + } + + public IReadOnlyList AllActive() + { + return _stores.Enumerate().Select(x => x.Value).ToList(); + } + + public IReadOnlyList> AllActiveByTenant() + { + return _stores.Enumerate().Select(x => new Assignment(x.Key, x.Value)).ToList(); + } + + public async ValueTask ConfigureDatabaseAsync(Func configureDatabase) + { + await RefreshAsync(); + foreach (var store in _stores.Enumerate().Select(x => x.Value).ToArray()) + { + await configureDatabase(store); + } + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Sagas/OracleSagaSchema.cs b/src/Persistence/Oracle/Wolverine.Oracle/Sagas/OracleSagaSchema.cs new file mode 100644 index 000000000..f38474491 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Sagas/OracleSagaSchema.cs @@ -0,0 +1,177 @@ +using System.Data.Common; +using System.Text.Json; +using JasperFx; +using JasperFx.Core.Reflection; +using Oracle.ManagedDataAccess.Client; +using Weasel.Core; +using Weasel.Core.Migrations; +using Weasel.Oracle; +using Weasel.Oracle.Tables; +using Wolverine.RDBMS; +using Wolverine.RDBMS.Sagas; + +namespace Wolverine.Oracle.Sagas; + +public class OracleSagaSchema : IDatabaseSagaSchema where T : Saga +{ + private readonly DatabaseSettings _settings; + private readonly string _insertSql; + private readonly string _updateSql; + private readonly string _deleteSql; + private readonly string _loadSql; + + public OracleSagaSchema(SagaTableDefinition definition, DatabaseSettings settings) + { + _settings = settings; + IdSource = LambdaBuilder.Getter(definition.IdMember); + + var schemaName = (settings.SchemaName ?? "WOLVERINE").ToUpperInvariant(); + var tableName = definition.TableName.ToUpperInvariant(); + + _insertSql = + $"INSERT INTO {schemaName}.{tableName} ({DatabaseConstants.Id}, {DatabaseConstants.Body}, {DatabaseConstants.Version}) VALUES (:id, :body, 1)"; + _updateSql = + $"UPDATE {schemaName}.{tableName} SET {DatabaseConstants.Body} = :body, {DatabaseConstants.Version} = :version + 1, last_modified = SYS_EXTRACT_UTC(SYSTIMESTAMP) WHERE {DatabaseConstants.Id} = :id AND {DatabaseConstants.Version} = :version"; + _loadSql = + $"SELECT body, version FROM {schemaName}.{tableName} WHERE {DatabaseConstants.Id} = :id"; + + _deleteSql = $"DELETE FROM {schemaName}.{tableName} WHERE id = :id"; + + var table = new Table(new OracleObjectName(schemaName, tableName)); + + // Determine ID column type + var idType = typeof(TId); + if (idType == typeof(Guid)) + { + table.AddColumn("id").AsPrimaryKey(); + } + else if (idType == typeof(int)) + { + table.AddColumn("id").AsPrimaryKey(); + } + else if (idType == typeof(long)) + { + table.AddColumn("id").AsPrimaryKey(); + } + else if (idType == typeof(string)) + { + table.AddColumn("id", "VARCHAR2(200)").AsPrimaryKey(); + } + else + { + table.AddColumn("id").AsPrimaryKey(); + } + + table.AddColumn(DatabaseConstants.Body, "CLOB").NotNull(); + table.AddColumn(DatabaseConstants.Version, "NUMBER(10)").DefaultValue(1).NotNull(); + table.AddColumn("created").DefaultValueByExpression("SYS_EXTRACT_UTC(SYSTIMESTAMP)").NotNull(); + table.AddColumn("last_modified").DefaultValueByExpression("SYS_EXTRACT_UTC(SYSTIMESTAMP)").NotNull(); + + Table = table; + } + + public void MarkAsChecked() => HasChecked = true; + + public bool HasChecked { get; private set; } + + public async Task EnsureStorageExistsAsync(CancellationToken cancellationToken) + { + if (HasChecked || _settings.AutoCreate == AutoCreate.None) return; + + await using var conn = new OracleConnection(_settings.ConnectionString); + await conn.OpenAsync(cancellationToken); + + var migration = await SchemaMigration.DetermineAsync(conn, cancellationToken, Table); + + if (migration.Difference != SchemaPatchDifference.None) + { + await new OracleMigrator().ApplyAllAsync(conn, migration, _settings.AutoCreate, ct: cancellationToken); + } + + HasChecked = true; + } + + public ISchemaObject Table { get; } + + public Func IdSource { get; } + + public async Task InsertAsync(T saga, DbTransaction transaction, CancellationToken cancellationToken) + { + var id = IdSource(saga); + if (id == null || id.Equals(default(TId))) + throw new ArgumentOutOfRangeException(nameof(saga), + "You must define the saga id when using the lightweight saga storage"); + + await EnsureStorageExistsAsync(cancellationToken); + + var cmd = ((OracleConnection)transaction.Connection!).CreateCommand(_insertSql, (OracleTransaction)transaction); + addIdParameter(cmd, "id", id); + cmd.Parameters.Add(new OracleParameter("body", OracleDbType.Clob) { Value = JsonSerializer.Serialize(saga) }); + await cmd.ExecuteNonQueryAsync(cancellationToken); + + saga.Version = 1; + } + + public async Task UpdateAsync(T saga, DbTransaction transaction, CancellationToken cancellationToken) + { + await EnsureStorageExistsAsync(cancellationToken); + + var id = IdSource(saga); + + var cmd = ((OracleConnection)transaction.Connection!).CreateCommand(_updateSql, (OracleTransaction)transaction); + cmd.Parameters.Add(new OracleParameter("body", OracleDbType.Clob) { Value = JsonSerializer.Serialize(saga) }); + addIdParameter(cmd, "id", id); + cmd.With("version", saga.Version); + var count = await cmd.ExecuteNonQueryAsync(cancellationToken); + + if (count == 0) + throw new SagaConcurrencyException( + $"Saga of type {saga.GetType().FullNameInCode()} and id {id} cannot be updated because of optimistic concurrency violations"); + + saga.Version++; + } + + public async Task DeleteAsync(T saga, DbTransaction transaction, CancellationToken cancellationToken) + { + await EnsureStorageExistsAsync(cancellationToken); + + var cmd = ((OracleConnection)transaction.Connection!).CreateCommand(_deleteSql, (OracleTransaction)transaction); + addIdParameter(cmd, "id", IdSource(saga)); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + public async Task LoadAsync(TId id, DbTransaction tx, CancellationToken cancellationToken) + { + await EnsureStorageExistsAsync(cancellationToken); + + var cmd = ((OracleConnection)tx.Connection!).CreateCommand(_loadSql, (OracleTransaction)tx); + addIdParameter(cmd, "id", id); + + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + + if (!await reader.ReadAsync(cancellationToken)) + { + return null; + } + + var body = await reader.GetFieldValueAsync(0, cancellationToken); + var saga = JsonSerializer.Deserialize(body); + saga!.Version = Convert.ToInt32(reader.GetValue(1)); + + await reader.CloseAsync(); + + return saga; + } + + private static void addIdParameter(OracleCommand cmd, string name, object? id) + { + if (id is Guid guidValue) + { + cmd.Parameters.Add(new OracleParameter(name, OracleDbType.Raw) { Value = guidValue.ToByteArray() }); + } + else + { + cmd.With(name, id!); + } + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Schema/DeadLettersTable.cs b/src/Persistence/Oracle/Wolverine.Oracle/Schema/DeadLettersTable.cs new file mode 100644 index 000000000..fe5029222 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Schema/DeadLettersTable.cs @@ -0,0 +1,40 @@ +using Weasel.Oracle; +using Weasel.Oracle.Tables; +using Wolverine.RDBMS; + +namespace Wolverine.Oracle.Schema; + +internal class DeadLettersTable : Table +{ + public DeadLettersTable(DurabilitySettings durability, string schemaName) : base( + new OracleObjectName(schemaName.ToUpperInvariant(), DatabaseConstants.DeadLetterTable.ToUpperInvariant())) + { + AddColumn(DatabaseConstants.Id).NotNull().AsPrimaryKey(); + + AddColumn(DatabaseConstants.ExecutionTime); + AddColumn(DatabaseConstants.Body, "BLOB").NotNull(); + + AddColumn(DatabaseConstants.MessageType, "VARCHAR2(500)").NotNull(); + + if (durability.MessageIdentity == MessageIdentity.IdOnly) + { + AddColumn(DatabaseConstants.ReceivedAt, "VARCHAR2(500)").NotNull(); + } + else + { + AddColumn(DatabaseConstants.ReceivedAt, "VARCHAR2(250)").AsPrimaryKey().NotNull(); + } + + AddColumn(DatabaseConstants.Source, "VARCHAR2(500)"); + AddColumn(DatabaseConstants.ExceptionType, "VARCHAR2(4000)"); + AddColumn(DatabaseConstants.ExceptionMessage, "VARCHAR2(4000)"); + + AddColumn(DatabaseConstants.SentAt); + AddColumn(DatabaseConstants.Replayable); + + if (durability.DeadLetterQueueExpirationEnabled) + { + AddColumn(DatabaseConstants.Expires).AllowNulls(); + } + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Schema/IncomingEnvelopeTable.cs b/src/Persistence/Oracle/Wolverine.Oracle/Schema/IncomingEnvelopeTable.cs new file mode 100644 index 000000000..58d7642c2 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Schema/IncomingEnvelopeTable.cs @@ -0,0 +1,38 @@ +using Weasel.Oracle; +using Weasel.Oracle.Tables; +using Wolverine.RDBMS; + +namespace Wolverine.Oracle.Schema; + +internal class IncomingEnvelopeTable : Table +{ + public IncomingEnvelopeTable(DurabilitySettings durability, string schemaName) : base( + new OracleObjectName(schemaName.ToUpperInvariant(), DatabaseConstants.IncomingTable.ToUpperInvariant())) + { + AddColumn(DatabaseConstants.Id).NotNull().AsPrimaryKey(); + AddColumn("status", "VARCHAR2(25)").NotNull(); + AddColumn(DatabaseConstants.OwnerId).NotNull(); + AddColumn(DatabaseConstants.ExecutionTime); + AddColumn(DatabaseConstants.Attempts).DefaultValue(0); + AddColumn(DatabaseConstants.Body, "BLOB").NotNull(); + + AddColumn(DatabaseConstants.MessageType, "VARCHAR2(500)").NotNull(); + + if (durability.MessageIdentity == MessageIdentity.IdOnly) + { + AddColumn(DatabaseConstants.ReceivedAt, "VARCHAR2(500)").NotNull(); + } + else + { + AddColumn(DatabaseConstants.ReceivedAt, "VARCHAR2(250)").AsPrimaryKey().NotNull(); + } + + AddColumn(DatabaseConstants.KeepUntil); + + if (durability.InboxStaleTime.HasValue) + { + AddColumn(DatabaseConstants.Timestamp) + .DefaultValueByExpression("SYS_EXTRACT_UTC(SYSTIMESTAMP)"); + } + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Schema/LockTable.cs b/src/Persistence/Oracle/Wolverine.Oracle/Schema/LockTable.cs new file mode 100644 index 000000000..0de52f621 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Schema/LockTable.cs @@ -0,0 +1,15 @@ +using Weasel.Oracle; +using Weasel.Oracle.Tables; + +namespace Wolverine.Oracle.Schema; + +internal class LockTable : Table +{ + public const string TableName = "WOLVERINE_LOCKS"; + + public LockTable(string schemaName) : base( + new OracleObjectName(schemaName.ToUpperInvariant(), TableName)) + { + AddColumn("lock_id").AsPrimaryKey(); + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Schema/OutgoingEnvelopeTable.cs b/src/Persistence/Oracle/Wolverine.Oracle/Schema/OutgoingEnvelopeTable.cs new file mode 100644 index 000000000..43cc41566 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Schema/OutgoingEnvelopeTable.cs @@ -0,0 +1,27 @@ +using Weasel.Oracle; +using Weasel.Oracle.Tables; +using Wolverine.RDBMS; + +namespace Wolverine.Oracle.Schema; + +internal class OutgoingEnvelopeTable : Table +{ + public OutgoingEnvelopeTable(DurabilitySettings durability, string schemaName) : base( + new OracleObjectName(schemaName.ToUpperInvariant(), DatabaseConstants.OutgoingTable.ToUpperInvariant())) + { + AddColumn(DatabaseConstants.Id).AsPrimaryKey(); + AddColumn(DatabaseConstants.OwnerId).NotNull(); + AddColumn(DatabaseConstants.Destination, "VARCHAR2(500)").NotNull(); + AddColumn(DatabaseConstants.DeliverBy); + AddColumn(DatabaseConstants.Body, "BLOB").NotNull(); + + AddColumn(DatabaseConstants.Attempts).DefaultValue(0); + AddColumn(DatabaseConstants.MessageType, "VARCHAR2(500)").NotNull(); + + if (durability.OutboxStaleTime.HasValue) + { + AddColumn(DatabaseConstants.Timestamp) + .DefaultValueByExpression("SYS_EXTRACT_UTC(SYSTIMESTAMP)"); + } + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Transport/IOracleQueueSender.cs b/src/Persistence/Oracle/Wolverine.Oracle/Transport/IOracleQueueSender.cs new file mode 100644 index 000000000..6a4c73117 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Transport/IOracleQueueSender.cs @@ -0,0 +1,8 @@ +using Wolverine.Transports.Sending; + +namespace Wolverine.Oracle.Transport; + +internal interface IOracleQueueSender : ISender +{ + Task ScheduleRetryAsync(Envelope envelope, CancellationToken cancellationToken); +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Transport/MultiTenantedQueueListener.cs b/src/Persistence/Oracle/Wolverine.Oracle/Transport/MultiTenantedQueueListener.cs new file mode 100644 index 000000000..853d714ad --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Transport/MultiTenantedQueueListener.cs @@ -0,0 +1,112 @@ +using ImTools; +using JasperFx.Core; +using Microsoft.Extensions.Logging; +using Wolverine.Runtime; +using Wolverine.Transports; +using MultiTenantedMessageStore = Wolverine.Persistence.Durability.MultiTenantedMessageStore; + +namespace Wolverine.Oracle.Transport; + +public class MultiTenantedQueueListener : IListener +{ + private readonly ILogger _logger; + private readonly OracleQueue _queue; + private readonly MultiTenantedMessageStore _stores; + private readonly IWolverineRuntime _runtime; + private readonly IReceiver _receiver; + private readonly CancellationTokenSource _cancellation; + + private ImHashMap _listeners = ImHashMap.Empty; + private Task? _activator; + + public MultiTenantedQueueListener(ILogger logger, OracleQueue queue, MultiTenantedMessageStore stores, IWolverineRuntime runtime, IReceiver receiver) + { + _logger = logger; + _queue = queue; + _stores = stores; + _runtime = runtime; + _receiver = receiver; + + Address = _queue.Uri; + + _cancellation = CancellationTokenSource.CreateLinkedTokenSource(runtime.Cancellation); + } + + public async Task StartAsync() + { + if (_queue.Parent.Store != null) + { + await startListening(_queue.Parent.Store); + } + + foreach (var store in _stores.Source.AllActive().OfType()) + { + await startListening(store); + } + + _activator = Task.Run(async () => + { + while (!_cancellation.IsCancellationRequested) + { + await Task.Delay(_runtime.Options.Durability.TenantCheckPeriod, _cancellation.Token); + + await _stores.Source.RefreshAsync(); + var databases = _stores.Source.AllActive(); + foreach (var store in databases.OfType()) + { + if (!_listeners.Contains(store.Name)) + { + await startListening(store); + } + } + } + }, _cancellation.Token); + } + + private async Task startListening(OracleMessageStore store) + { + var listener = new OracleQueueListener(_queue, _runtime, _receiver, store.OracleDataSource, store.Name); + _listeners = _listeners.AddOrUpdate(store.Name, listener); + await listener.StartAsync(); + + _logger.LogInformation("Started message listening for Oracle queue {QueueName} on database {Database}", _queue.Name, store.Name); + } + + public IHandlerPipeline? Pipeline => _receiver.Pipeline; + + ValueTask IChannelCallback.CompleteAsync(Envelope envelope) + { + return new ValueTask(); + } + + ValueTask IChannelCallback.DeferAsync(Envelope envelope) + { + return new ValueTask(); + } + + public async ValueTask DisposeAsync() + { + _cancellation.Cancel(); + _activator?.SafeDispose(); + foreach (var entry in _listeners.Enumerate()) + { + await entry.Value.DisposeAsync(); + } + } + + public Uri Address { get; set; } + + public async ValueTask StopAsync() + { + _cancellation.Cancel(); + foreach (var entry in _listeners.Enumerate()) + { + await entry.Value.StopAsync(); + } + } + + public bool IsListeningToDatabase(string databaseName) + { + return _listeners.Contains(databaseName); + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Transport/MultiTenantedQueueSender.cs b/src/Persistence/Oracle/Wolverine.Oracle/Transport/MultiTenantedQueueSender.cs new file mode 100644 index 000000000..1d00fdd1f --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Transport/MultiTenantedQueueSender.cs @@ -0,0 +1,93 @@ +using ImTools; +using JasperFx.Core; +using Wolverine.RDBMS; +using MultiTenantedMessageStore = Wolverine.Persistence.Durability.MultiTenantedMessageStore; + +namespace Wolverine.Oracle.Transport; + +internal class MultiTenantedQueueSender : IOracleQueueSender, IAsyncDisposable +{ + private readonly OracleQueue _queue; + private readonly OracleQueueSender _master; + private readonly MultiTenantedMessageStore _stores; + private ImHashMap _byDatabase = ImHashMap.Empty; + private readonly SemaphoreSlim _lock = new(1); + private readonly CancellationTokenSource _cancellation = new(); + + public MultiTenantedQueueSender(OracleQueue queue, MultiTenantedMessageStore stores) + { + _queue = queue; + _master = new OracleQueueSender(queue); + _stores = stores; + + Destination = _queue.Uri; + } + + public bool SupportsNativeScheduledSend => true; + public Uri Destination { get; } + + public Task PingAsync() + { + return _master.PingAsync(); + } + + public async ValueTask SendAsync(Envelope envelope) + { + var sender = await resolveSender(envelope); + await sender.SendAsync(envelope); + } + + public async Task ScheduleRetryAsync(Envelope envelope, CancellationToken cancellationToken) + { + var sender = await resolveSender(envelope); + await sender.ScheduleRetryAsync(envelope, cancellationToken); + } + + private async ValueTask resolveSender(Envelope envelope) + { + if (envelope.TenantId.IsEmpty() || envelope.TenantId == "*DEFAULT*") + { + return _master; + } + + if (_byDatabase.TryFind(envelope.TenantId, out var sender)) + { + return sender; + } + + await _lock.WaitAsync(_cancellation.Token); + try + { + var database = (IMessageDatabase)await _stores.GetDatabaseAsync(envelope.TenantId); + if (_byDatabase.TryFind(database.Name, out sender)) + { + // Database encountered before with a different tenant id + } + else + { + var oracleDataSource = (OracleDataSource)database.DataSource; + sender = new OracleQueueSender(_queue, oracleDataSource, database.Name); + _byDatabase = _byDatabase.AddOrUpdate(database.Name, sender); + + if (_queue.Parent.AutoProvision) + { + await _queue.EnsureSchemaExists(database.Name, oracleDataSource); + } + } + + _byDatabase = _byDatabase.AddOrUpdate(envelope.TenantId, sender); + } + finally + { + _lock.Release(); + } + + return sender; + } + + public ValueTask DisposeAsync() + { + _cancellation.Cancel(); + return new ValueTask(); + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleListenerConfiguration.cs b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleListenerConfiguration.cs new file mode 100644 index 000000000..7d4297cf6 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleListenerConfiguration.cs @@ -0,0 +1,39 @@ +using Wolverine.Configuration; +using Wolverine.ErrorHandling; + +namespace Wolverine.Oracle.Transport; + +public class OracleListenerConfiguration : ListenerConfiguration +{ + public OracleListenerConfiguration(OracleQueue endpoint) : base(endpoint) + { + } + + public OracleListenerConfiguration(Func source) : base(source) + { + } + + /// + /// The maximum number of messages to receive in a single batch when listening + /// in either buffered or durable modes. The default is 20. + /// + public OracleListenerConfiguration MaximumMessagesToReceive(int maximum) + { + add(e => e.MaximumMessagesToReceive = maximum); + return this; + } + + /// + /// Add circuit breaker exception handling to this listener + /// + public OracleListenerConfiguration CircuitBreaker(Action? configure = null) + { + add(e => + { + e.CircuitBreakerOptions = new CircuitBreakerOptions(); + configure?.Invoke(e.CircuitBreakerOptions); + }); + + return this; + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Transport/OraclePersistenceExpression.cs b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OraclePersistenceExpression.cs new file mode 100644 index 000000000..65e17fd71 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OraclePersistenceExpression.cs @@ -0,0 +1,52 @@ +using Wolverine.Configuration; +using Wolverine.Transports; + +namespace Wolverine.Oracle.Transport; + +public class OraclePersistenceExpression : BrokerExpression +{ + private readonly WolverineOptions _options; + + public OraclePersistenceExpression(OracleTransport transport, WolverineOptions options) : base(transport, options) + { + _options = options; + } + + protected override OracleListenerConfiguration createListenerExpression(OracleQueue listenerEndpoint) + { + return new OracleListenerConfiguration(listenerEndpoint); + } + + protected override OracleSubscriberConfiguration createSubscriberExpression(OracleQueue subscriberEndpoint) + { + return new OracleSubscriberConfiguration(subscriberEndpoint); + } + + /// + /// Configure the schema name for the transport queue and scheduled message tables + /// + public OraclePersistenceExpression TransportSchemaName(string schemaName) + { + Transport.TransportSchemaName = schemaName.ToUpperInvariant(); + return this; + } + + /// + /// Disable inbox and outbox usage on all Oracle Transport endpoints + /// + public OraclePersistenceExpression DisableInboxAndOutboxOnAll() + { + var policy = new LambdaEndpointPolicy((e, _) => + { + if (e.Role == EndpointRole.System) + { + return; + } + + e.Mode = EndpointMode.BufferedInMemory; + }); + + _options.Policies.Add(policy); + return this; + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueue.cs b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueue.cs new file mode 100644 index 000000000..623bf6eab --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueue.cs @@ -0,0 +1,300 @@ +using ImTools; +using JasperFx.Core; +using Microsoft.Extensions.Logging; +using Oracle.ManagedDataAccess.Client; +using Weasel.Oracle; +using Weasel.Oracle.Tables; +using Wolverine.Configuration; +using Wolverine.Runtime; +using Wolverine.Transports; +using Wolverine.Transports.Sending; + +namespace Wolverine.Oracle.Transport; + +public class OracleQueue : Endpoint, IBrokerQueue, IDatabaseBackedEndpoint +{ + internal static Uri ToUri(string name, string? databaseName) + { + return databaseName.IsEmpty() + ? new Uri($"{OracleTransport.ProtocolName}://{name}") + : new Uri($"{OracleTransport.ProtocolName}://{name}/{databaseName}"); + } + + private bool _hasInitialized; + private IOracleQueueSender? _sender; + private ImHashMap _checkedDatabases = ImHashMap.Empty; + private readonly Lazy _queueTable; + private readonly Lazy _scheduledMessageTable; + + public OracleQueue(string name, OracleTransport parent, EndpointRole role = EndpointRole.Application, + string? databaseName = null) : + base(ToUri(name, databaseName), role) + { + Parent = parent; + var queueTableName = $"wolverine_queue_{name}"; + var scheduledTableName = $"wolverine_queue_{name}_scheduled"; + + Mode = EndpointMode.Durable; + Name = name; + EndpointName = name; + + _queueTable = new Lazy(() => new QueueTable(Parent, queueTableName)); + _scheduledMessageTable = + new Lazy(() => new ScheduledMessageTable(Parent, scheduledTableName)); + } + + public string Name { get; } + + internal OracleTransport Parent { get; } + + internal Table QueueTable => _queueTable.Value; + + internal Table ScheduledTable => _scheduledMessageTable.Value; + + protected override bool supportsMode(EndpointMode mode) + { + return mode == EndpointMode.Durable || mode == EndpointMode.BufferedInMemory; + } + + /// + /// The maximum number of messages to receive in a single batch when listening + /// in either buffered or durable modes. The default is 20. + /// + public int MaximumMessagesToReceive { get; set; } = 20; + + public override async ValueTask BuildListenerAsync(IWolverineRuntime runtime, IReceiver receiver) + { + if (Parent.AutoProvision) + { + await SetupAsync(runtime.LoggerFactory.CreateLogger()); + } + + if (Parent.Databases != null) + { + var mtListener = new MultiTenantedQueueListener( + runtime.LoggerFactory.CreateLogger(), this, Parent.Databases, runtime, + receiver); + + await mtListener.StartAsync(); + return mtListener; + } + + var listener = new OracleQueueListener(this, runtime, receiver, DataSource, null); + await listener.StartAsync(); + return listener; + } + + private void buildSenderIfMissing() + { + if (Parent.Databases != null) + { + _sender = new MultiTenantedQueueSender(this, Parent.Databases); + } + else + { + _sender = new OracleQueueSender(this, DataSource, null); + } + } + + protected override ISender CreateSender(IWolverineRuntime runtime) + { + buildSenderIfMissing(); + return _sender!; + } + + public override async ValueTask InitializeAsync(ILogger logger) + { + if (_hasInitialized) + { + return; + } + + if (Parent.AutoProvision) + { + await SetupAsync(logger); + } + + if (Parent.AutoPurgeAllQueues) + { + await PurgeAsync(logger); + } + + _hasInitialized = true; + } + + internal OracleDataSource DataSource => Parent.Store?.OracleDataSource ?? throw new InvalidOperationException("The Oracle transport has not been successfully initialized"); + + public ValueTask SendAsync(Envelope envelope) + { + buildSenderIfMissing(); + return _sender!.SendAsync(envelope); + } + + private async ValueTask forEveryDatabase(Func action) + { + if (Parent?.Store?.OracleDataSource != null) + { + await action(Parent.Store.OracleDataSource, Parent.Store.Name); + } + + if (Parent?.Databases != null) + { + foreach (var database in Parent.Databases.ActiveDatabases().OfType()) + { + await action(database.OracleDataSource, database.Name); + } + } + } + + public ValueTask PurgeAsync(ILogger logger) + { + return forEveryDatabase(async (source, _) => + { + await using var conn = await source.OpenConnectionAsync(); + try + { + var cmd1 = conn.CreateCommand($"DELETE FROM {QueueTable.Identifier.QualifiedName}"); + await cmd1.ExecuteNonQueryAsync(); + + var cmd2 = conn.CreateCommand($"DELETE FROM {ScheduledTable.Identifier.QualifiedName}"); + await cmd2.ExecuteNonQueryAsync(); + } + finally + { + await conn.CloseAsync(); + } + }); + } + + public async ValueTask> GetAttributesAsync() + { + var count = await CountAsync(); + var scheduled = await ScheduledCountAsync(); + + return new Dictionary + { { "Name", Name }, { "Count", count.ToString() }, { "Scheduled", scheduled.ToString() } }; + } + + public async ValueTask CheckAsync() + { + var returnValue = true; + await forEveryDatabase(async (source, _) => + { + await using var conn = await source.OpenConnectionAsync(); + try + { + var queueDelta = await QueueTable.FindDeltaAsync(conn); + if (queueDelta.HasChanges()) + { + returnValue = false; + return; + } + + var scheduledDelta = await ScheduledTable.FindDeltaAsync(conn); + + returnValue = returnValue && !scheduledDelta.HasChanges(); + } + finally + { + await conn.CloseAsync(); + } + }); + + return returnValue; + } + + public async ValueTask TeardownAsync(ILogger logger) + { + await forEveryDatabase(async (source, _) => + { + // Retry to handle ORA-00054 (resource busy) when a listener is still polling + for (var attempt = 0; attempt < 3; attempt++) + { + try + { + await using var conn = await source.OpenConnectionAsync(); + + await QueueTable.Drop(conn); + await ScheduledTable.Drop(conn); + + await conn.CloseAsync(); + return; + } + catch (OracleException e) when (e.Number == 54 && attempt < 2) + { + await Task.Delay(500 * (attempt + 1)); + } + } + }); + } + + public async ValueTask SetupAsync(ILogger logger) + { + await forEveryDatabase(async (source, identifier) => + { + await EnsureSchemaExists(identifier, source); + }); + } + + internal async Task EnsureSchemaExists(string identifier, OracleDataSource source) + { + if (_checkedDatabases.Contains(identifier)) return; + + await using (var conn = await source.OpenConnectionAsync()) + { + await QueueTable.ApplyChangesAsync(conn); + await ScheduledTable.ApplyChangesAsync(conn); + + await conn.CloseAsync(); + } + + _checkedDatabases = _checkedDatabases.AddOrUpdate(identifier, true); + } + + public async Task CountAsync() + { + var count = 0L; + await forEveryDatabase(async (source, _) => + { + await using var conn = await source.OpenConnectionAsync(); + + try + { + var cmd = conn.CreateCommand($"SELECT COUNT(*) FROM {QueueTable.Identifier.QualifiedName}"); + count += Convert.ToInt64(await cmd.ExecuteScalarAsync()); + } + finally + { + await conn.CloseAsync(); + } + }); + + return count; + } + + public async Task ScheduledCountAsync() + { + var count = 0L; + await forEveryDatabase(async (source, _) => + { + await using var conn = await source.OpenConnectionAsync(); + try + { + var cmd = conn.CreateCommand($"SELECT COUNT(*) FROM {ScheduledTable.Identifier.QualifiedName}"); + count += Convert.ToInt64(await cmd.ExecuteScalarAsync()); + } + finally + { + await conn.CloseAsync(); + } + }); + + return count; + } + + public Task ScheduleRetryAsync(Envelope envelope, CancellationToken cancellation) + { + buildSenderIfMissing(); + return _sender!.ScheduleRetryAsync(envelope, cancellation); + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueueListener.cs b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueueListener.cs new file mode 100644 index 000000000..bf1fb57d6 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueueListener.cs @@ -0,0 +1,439 @@ +using JasperFx.Core; +using Microsoft.Extensions.Logging; +using Oracle.ManagedDataAccess.Client; +using Weasel.Oracle; +using Wolverine.Configuration; +using Wolverine.Oracle.Util; +using Wolverine.RDBMS; +using Wolverine.Runtime; +using Wolverine.Runtime.Serialization; +using Wolverine.Transports; + +namespace Wolverine.Oracle.Transport; + +internal class OracleQueueListener : IListener +{ + private readonly CancellationTokenSource _cancellation = new(); + private readonly OracleQueue _queue; + private readonly IReceiver _receiver; + private readonly OracleDataSource _dataSource; + private readonly string? _databaseName; + private readonly ILogger _logger; + private Task? _task; + private readonly DurabilitySettings _settings; + private Task? _scheduledTask; + private readonly OracleQueueSender _sender; + private readonly string _tryPopMessagesDirectlySql; + private readonly string _queueTableName; + private readonly string _queueName; + private readonly string _schemaName; + private readonly string _scheduledTableName; + + public OracleQueueListener(OracleQueue queue, IWolverineRuntime runtime, IReceiver receiver, + OracleDataSource dataSource, string? databaseName) + { + Address = OracleQueue.ToUri(queue.Name, databaseName); + _queue = queue; + _receiver = receiver; + _dataSource = dataSource; + _databaseName = databaseName; + _logger = runtime.LoggerFactory.CreateLogger(); + _settings = runtime.DurabilitySettings; + + _sender = new OracleQueueSender(queue, _dataSource, databaseName); + + _queueTableName = _queue.QueueTable.Identifier.QualifiedName; + _scheduledTableName = _queue.ScheduledTable.Identifier.QualifiedName; + _schemaName = _queue.Parent.MessageStorageSchemaName; + + _tryPopMessagesDirectlySql = + $"SELECT {DatabaseConstants.Id}, {DatabaseConstants.Body} FROM {_queueTableName} " + + "ORDER BY timestamp FETCH FIRST :count ROWS ONLY FOR UPDATE SKIP LOCKED"; + + _queueName = _queue.Name; + } + + public IHandlerPipeline? Pipeline => _receiver.Pipeline; + + public ValueTask CompleteAsync(Envelope envelope) + { + return ValueTask.CompletedTask; + } + + public async ValueTask DeferAsync(Envelope envelope) + { + await _sender.SendAsync(envelope, _cancellation.Token); + } + + public ValueTask DisposeAsync() + { + _cancellation.Cancel(); + _task.SafeDispose(); + _scheduledTask.SafeDispose(); + return ValueTask.CompletedTask; + } + + public Uri Address { get; } + + public ValueTask StopAsync() + { + _cancellation.Cancel(); + + _task?.SafeDispose(); + _scheduledTask?.SafeDispose(); + + return ValueTask.CompletedTask; + } + + private async Task lookForScheduledMessagesAsync() + { + await Task.Delay(_settings.ScheduledJobFirstExecution); + + var failedCount = 0; + + while (!_cancellation.Token.IsCancellationRequested) + { + try + { + var count = await MoveScheduledToReadyQueueAsync(_cancellation.Token); + if (count > 0) + { + _logger.LogInformation("Propagated {Number} scheduled messages to Oracle-backed queue {Queue}", count, _queueName); + } + + await DeleteExpiredAsync(CancellationToken.None); + + failedCount = 0; + + await Task.Delay(_settings.ScheduledJobPollingTime); + } + catch (Exception e) + { + if (e is TaskCanceledException && _cancellation.IsCancellationRequested) + { + break; + } + + failedCount++; + var pauseTime = failedCount > 5 ? 1.Seconds() : (failedCount * 100).Milliseconds(); + + _logger.LogError(e, "Error while trying to propagate scheduled messages from Oracle Queue {Name}", + _queueName); + + await Task.Delay(pauseTime); + } + } + } + + public async Task MoveScheduledToReadyQueueAsync(CancellationToken cancellationToken) + { + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + + long count = 0; + + try + { + await using var transaction = await conn.BeginTransactionAsync(cancellationToken); + + OracleCommand CreateCmd(string sql) + { + var cmd = conn.CreateCommand(sql); + cmd.Transaction = (OracleTransaction)transaction; + return cmd; + } + + // Select scheduled messages that are ready and lock them + var selectCmd = CreateCmd( + $"SELECT id, body, message_type, keep_until FROM {_scheduledTableName} " + + $"WHERE {DatabaseConstants.ExecutionTime} <= SYS_EXTRACT_UTC(SYSTIMESTAMP) " + + "FOR UPDATE SKIP LOCKED"); + + var idsToMove = new List(); + var bodies = new List(); + var messageTypes = new List(); + var keepUntils = new List(); + + await using (var reader = await selectCmd.ExecuteReaderAsync(cancellationToken)) + { + while (await reader.ReadAsync(cancellationToken)) + { + idsToMove.Add(await OracleEnvelopeReader.ReadGuidAsync(reader, 0, cancellationToken)); + bodies.Add((byte[])reader.GetValue(1)); + messageTypes.Add(await reader.GetFieldValueAsync(2, cancellationToken)); + keepUntils.Add(await reader.IsDBNullAsync(3, cancellationToken) ? null : await reader.GetFieldValueAsync(3, cancellationToken)); + } + } + + for (int i = 0; i < idsToMove.Count; i++) + { + // Insert into queue (skip if already exists) + try + { + var insertCmd = CreateCmd( + $"INSERT INTO {_queueTableName} (id, body, message_type, keep_until) VALUES (:id, :body, :type, :keepUntil)"); + insertCmd.With("id", idsToMove[i]); + insertCmd.Parameters.Add(new OracleParameter("body", OracleDbType.Blob) { Value = bodies[i] }); + insertCmd.With("type", messageTypes[i]); + insertCmd.Parameters.Add(new OracleParameter("keepUntil", OracleDbType.TimeStampTZ) { Value = (object?)keepUntils[i] ?? DBNull.Value }); + await insertCmd.ExecuteNonQueryAsync(cancellationToken); + } + catch (OracleException ex) when (ex.Number == 1) + { + // Already exists in queue, skip + } + + // Delete from scheduled + var deleteCmd = CreateCmd( + $"DELETE FROM {_scheduledTableName} WHERE id = :id"); + deleteCmd.With("id", idsToMove[i]); + await deleteCmd.ExecuteNonQueryAsync(cancellationToken); + } + + count = idsToMove.Count; + + await transaction.CommitAsync(cancellationToken); + } + finally + { + await conn.CloseAsync(); + } + + return count; + } + + private async Task listenForMessagesAsync() + { + var failedCount = 0; + + while (!_cancellation.Token.IsCancellationRequested) + { + try + { + var messages = _queue.Mode == EndpointMode.Durable + ? await TryPopDurablyAsync(_queue.MaximumMessagesToReceive, _settings, _logger, + _cancellation.Token) + : await TryPopAsync(_queue.MaximumMessagesToReceive, _logger, _cancellation.Token); + + failedCount = 0; + + if (messages.Any()) + { + await _receiver.ReceivedAsync(this, messages.ToArray()); + + if (messages.Count > _queue.MaximumMessagesToReceive) + { + await Task.Delay(250.Milliseconds()); + } + else + { + await Task.Delay(_settings.ScheduledJobPollingTime); + } + } + else + { + await Task.Delay(_settings.ScheduledJobPollingTime); + } + } + catch (Exception e) + { + if (e is TaskCanceledException && _cancellation.IsCancellationRequested) + { + break; + } + + failedCount++; + var pauseTime = failedCount > 5 ? 1.Seconds() : (failedCount * 100).Milliseconds(); + + _logger.LogError(e, "Error while trying to retrieve messages from Oracle Queue {Name}", + _queueName); + + await Task.Delay(pauseTime); + } + } + } + + public async Task> TryPopDurablyAsync(int count, DurabilitySettings settings, + ILogger logger, CancellationToken cancellationToken) + { + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + + try + { + await using var transaction = await conn.BeginTransactionAsync(cancellationToken); + + OracleCommand CreateCmd(string sql) + { + var cmd = conn.CreateCommand(sql); + cmd.Transaction = (OracleTransaction)transaction; + return cmd; + } + + // First, delete any messages that are already in the incoming table (deduplication) + await CreateCmd($"DELETE FROM {_queueTableName} WHERE id IN (SELECT id FROM {_schemaName}.{DatabaseConstants.IncomingTable})") + .ExecuteNonQueryAsync(cancellationToken); + + // Select messages to process with lock + var selectCmd = CreateCmd( + $"SELECT id, body, message_type, keep_until FROM {_queueTableName} " + + "ORDER BY timestamp FETCH FIRST :count ROWS ONLY FOR UPDATE SKIP LOCKED"); + selectCmd.With("count", count); + + var ids = new List(); + var bodyList = new List(); + var messageTypeList = new List(); + var keepUntilList = new List(); + + await using (var reader = await selectCmd.ExecuteReaderAsync(cancellationToken)) + { + while (await reader.ReadAsync(cancellationToken)) + { + ids.Add(await OracleEnvelopeReader.ReadGuidAsync(reader, 0, cancellationToken)); + bodyList.Add((byte[])reader.GetValue(1)); + messageTypeList.Add(await reader.GetFieldValueAsync(2, cancellationToken)); + keepUntilList.Add(await reader.IsDBNullAsync(3, cancellationToken) ? null : await reader.GetFieldValueAsync(3, cancellationToken)); + } + } + + if (ids.Count == 0) + { + await transaction.RollbackAsync(cancellationToken); + return Array.Empty(); + } + + var list = new List(); + + for (int i = 0; i < ids.Count; i++) + { + // Delete from queue + var deleteCmd = CreateCmd($"DELETE FROM {_queueTableName} WHERE id = :id"); + deleteCmd.With("id", ids[i]); + await deleteCmd.ExecuteNonQueryAsync(cancellationToken); + + // Insert into incoming + var insertCmd = CreateCmd( + $"INSERT INTO {_schemaName}.{DatabaseConstants.IncomingTable} (id, status, owner_id, body, message_type, received_at, keep_until) " + + "VALUES (:id, 'Incoming', :ownerId, :body, :messageType, :receivedAt, :keepUntil)"); + insertCmd.With("id", ids[i]); + insertCmd.With("ownerId", settings.AssignedNodeNumber); + insertCmd.Parameters.Add(new OracleParameter("body", OracleDbType.Blob) { Value = bodyList[i] }); + insertCmd.With("messageType", messageTypeList[i]); + insertCmd.With("receivedAt", Address.ToString()); + insertCmd.Parameters.Add(new OracleParameter("keepUntil", OracleDbType.TimeStampTZ) { Value = (object?)keepUntilList[i] ?? DBNull.Value }); + await insertCmd.ExecuteNonQueryAsync(cancellationToken); + + try + { + var e = EnvelopeSerializer.Deserialize(bodyList[i]); + list.Add(e); + } + catch (Exception e) + { + logger.LogError(e, "Error trying to deserialize Envelope data in Oracle Transport Queue {Queue}, discarding", _queueName); + var ping = Envelope.ForPing(Address); + list.Add(ping); + } + } + + await transaction.CommitAsync(cancellationToken); + + return list; + } + finally + { + await conn.CloseAsync(); + } + } + + public async Task> TryPopAsync(int count, ILogger logger, + CancellationToken cancellationToken) + { + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + + try + { + await using var transaction = await conn.BeginTransactionAsync(cancellationToken); + + OracleCommand CreateCmd(string sql) + { + var cmd = conn.CreateCommand(sql); + cmd.Transaction = (OracleTransaction)transaction; + return cmd; + } + + var idsToDelete = new List(); + var list = new List(); + + var selectCmd = CreateCmd(_tryPopMessagesDirectlySql); + selectCmd.With("count", count); + await using (var reader = await selectCmd.ExecuteReaderAsync(cancellationToken)) + { + while (await reader.ReadAsync(cancellationToken)) + { + var id = await OracleEnvelopeReader.ReadGuidAsync(reader, 0, cancellationToken); + var data = (byte[])reader.GetValue(1); + + idsToDelete.Add(id); + + try + { + var e = EnvelopeSerializer.Deserialize(data); + list.Add(e); + } + catch (Exception e) + { + logger.LogError(e, "Error trying to deserialize Envelope data in Oracle Transport Queue {Queue}, discarding", _queueName); + var ping = Envelope.ForPing(Address); + list.Add(ping); + } + } + } + + // Delete the messages we just read + foreach (var id in idsToDelete) + { + var deleteCmd = CreateCmd($"DELETE FROM {_queueTableName} WHERE id = :id"); + deleteCmd.With("id", id); + await deleteCmd.ExecuteNonQueryAsync(cancellationToken); + } + + await transaction.CommitAsync(cancellationToken); + + return list; + } + finally + { + await conn.CloseAsync(); + } + } + + public async Task DeleteExpiredAsync(CancellationToken cancellationToken) + { + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + + try + { + var cmd1 = conn.CreateCommand( + $"DELETE FROM {_queueTableName} WHERE {DatabaseConstants.KeepUntil} IS NOT NULL AND {DatabaseConstants.KeepUntil} <= SYS_EXTRACT_UTC(SYSTIMESTAMP)"); + await cmd1.ExecuteNonQueryAsync(cancellationToken); + + var cmd2 = conn.CreateCommand( + $"DELETE FROM {_scheduledTableName} WHERE {DatabaseConstants.KeepUntil} IS NOT NULL AND {DatabaseConstants.KeepUntil} <= SYS_EXTRACT_UTC(SYSTIMESTAMP)"); + await cmd2.ExecuteNonQueryAsync(cancellationToken); + } + finally + { + await conn.CloseAsync(); + } + } + + public async Task StartAsync() + { + if (_queue.Parent.AutoProvision) + { + await _queue.EnsureSchemaExists(_databaseName ?? string.Empty, _dataSource); + } + + _task = Task.Run(listenForMessagesAsync, _cancellation.Token); + _scheduledTask = Task.Run(lookForScheduledMessagesAsync, _cancellation.Token); + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueueSender.cs b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueueSender.cs new file mode 100644 index 000000000..70f04fc72 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueueSender.cs @@ -0,0 +1,272 @@ +using JasperFx.Core; +using Oracle.ManagedDataAccess.Client; +using Weasel.Oracle; +using Wolverine.Configuration; +using Wolverine.Oracle.Util; +using Wolverine.RDBMS; +using Wolverine.Runtime.Serialization; + +namespace Wolverine.Oracle.Transport; + +internal class OracleQueueSender : IOracleQueueSender +{ + private readonly OracleQueue _queue; + private readonly OracleDataSource _dataSource; + + private readonly string _writeDirectlyToQueueTableSql; + private readonly string _writeDirectlyToTheScheduledTable; + private readonly string _schemaName; + + // Strictly for testing + public OracleQueueSender(OracleQueue queue) : this(queue, queue.DataSource, null) + { + Destination = queue.Uri; + } + + public OracleQueueSender(OracleQueue queue, OracleDataSource dataSource, string? databaseName) + { + _queue = queue; + _dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource)); + + Destination = OracleQueue.ToUri(queue.Name, databaseName); + + _schemaName = queue.Parent.TransportSchemaName; + + _writeDirectlyToQueueTableSql = + $"INSERT INTO {queue.QueueTable.Identifier.QualifiedName} ({DatabaseConstants.Id}, {DatabaseConstants.Body}, {DatabaseConstants.MessageType}, {DatabaseConstants.KeepUntil}) VALUES (:id, :body, :type, :expires)"; + + _writeDirectlyToTheScheduledTable = + $"MERGE INTO {queue.ScheduledTable.Identifier.QualifiedName} t USING DUAL ON (t.{DatabaseConstants.Id} = :id) " + + $"WHEN MATCHED THEN UPDATE SET t.{DatabaseConstants.Body} = :body, t.{DatabaseConstants.MessageType} = :type, t.{DatabaseConstants.KeepUntil} = :expires, t.{DatabaseConstants.ExecutionTime} = :time " + + $"WHEN NOT MATCHED THEN INSERT ({DatabaseConstants.Id}, {DatabaseConstants.Body}, {DatabaseConstants.MessageType}, {DatabaseConstants.KeepUntil}, {DatabaseConstants.ExecutionTime}) VALUES (:id, :body, :type, :expires, :time)"; + } + + public bool SupportsNativeScheduledSend => true; + public Uri Destination { get; private set; } + + public async Task PingAsync() + { + try + { + await _queue.CheckAsync(); + return true; + } + catch (Exception) + { + return false; + } + } + + public async Task ScheduleRetryAsync(Envelope envelope, CancellationToken cancellationToken) + { + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + try + { + // Delete from incoming, write to scheduled + var deleteCmd = conn.CreateCommand( + $"DELETE FROM {_queue.Parent.MessageStorageSchemaName}.{DatabaseConstants.IncomingTable} WHERE id = :id"); + deleteCmd.With("id", envelope.Id); + await deleteCmd.ExecuteNonQueryAsync(cancellationToken); + + await writeToScheduledTableAsync(envelope, cancellationToken, conn); + } + finally + { + await conn.CloseAsync(); + } + } + + public async ValueTask SendAsync(Envelope envelope) + { + if (_queue.Mode == EndpointMode.Durable && envelope.WasPersistedInOutbox) + { + if (envelope.IsScheduledForLater(DateTimeOffset.UtcNow)) + { + await MoveFromOutgoingToScheduledAsync(envelope, CancellationToken.None); + } + else + { + await MoveFromOutgoingToQueueAsync(envelope, CancellationToken.None); + } + } + else + { + await SendAsync(envelope, CancellationToken.None); + } + } + + public async Task MoveFromOutgoingToQueueAsync(Envelope envelope, CancellationToken cancellationToken) + { + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + + try + { + var tx = (OracleTransaction)await conn.BeginTransactionAsync(cancellationToken); + + // Read from outgoing + var readCmd = conn.CreateCommand( + $"SELECT {DatabaseConstants.Id}, {DatabaseConstants.Body}, {DatabaseConstants.MessageType}, {DatabaseConstants.DeliverBy} FROM {_queue.Parent.MessageStorageSchemaName}.{DatabaseConstants.OutgoingTable} WHERE {DatabaseConstants.Id} = :id FOR UPDATE"); + readCmd.Transaction = tx; + readCmd.With("id", envelope.Id); + + await using var reader = await readCmd.ExecuteReaderAsync(cancellationToken); + if (await reader.ReadAsync(cancellationToken)) + { + var id = await OracleEnvelopeReader.ReadGuidAsync(reader, 0, cancellationToken); + var body = (byte[])reader.GetValue(1); + var messageType = await reader.GetFieldValueAsync(2, cancellationToken); + DateTimeOffset? keepUntil = await reader.IsDBNullAsync(3, cancellationToken) ? null : await reader.GetFieldValueAsync(3, cancellationToken); + + await reader.CloseAsync(); + + // Insert into queue + var insertCmd = conn.CreateCommand(_writeDirectlyToQueueTableSql); + insertCmd.Transaction = tx; + insertCmd.With("id", id); + insertCmd.Parameters.Add(new OracleParameter("body", OracleDbType.Blob) { Value = body }); + insertCmd.With("type", messageType); + insertCmd.Parameters.Add(new OracleParameter("expires", OracleDbType.TimeStampTZ) { Value = (object?)keepUntil ?? DBNull.Value }); + await insertCmd.ExecuteNonQueryAsync(cancellationToken); + + // Delete from outgoing + var deleteCmd = conn.CreateCommand( + $"DELETE FROM {_queue.Parent.MessageStorageSchemaName}.{DatabaseConstants.OutgoingTable} WHERE {DatabaseConstants.Id} = :id"); + deleteCmd.Transaction = tx; + deleteCmd.With("id", id); + await deleteCmd.ExecuteNonQueryAsync(cancellationToken); + } + else + { + await reader.CloseAsync(); + } + + await tx.CommitAsync(cancellationToken); + } + catch (OracleException e) when (e.Number == 1) // ORA-00001: unique constraint violated + { + // Making this idempotent + return; + } + finally + { + await conn.CloseAsync(); + } + } + + public async Task MoveFromOutgoingToScheduledAsync(Envelope envelope, CancellationToken cancellationToken) + { + if (!envelope.ScheduledTime.HasValue) + throw new InvalidOperationException("This envelope has no scheduled time"); + + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + + try + { + var tx = (OracleTransaction)await conn.BeginTransactionAsync(cancellationToken); + + // Read from outgoing + var readCmd = conn.CreateCommand( + $"SELECT {DatabaseConstants.Id}, {DatabaseConstants.Body}, {DatabaseConstants.MessageType}, {DatabaseConstants.DeliverBy} FROM {_queue.Parent.MessageStorageSchemaName}.{DatabaseConstants.OutgoingTable} WHERE {DatabaseConstants.Id} = :id FOR UPDATE"); + readCmd.Transaction = tx; + readCmd.With("id", envelope.Id); + + await using var reader = await readCmd.ExecuteReaderAsync(cancellationToken); + if (await reader.ReadAsync(cancellationToken)) + { + var id = await OracleEnvelopeReader.ReadGuidAsync(reader, 0, cancellationToken); + var body = (byte[])reader.GetValue(1); + var messageType = await reader.GetFieldValueAsync(2, cancellationToken); + DateTimeOffset? keepUntil = await reader.IsDBNullAsync(3, cancellationToken) ? null : await reader.GetFieldValueAsync(3, cancellationToken); + + await reader.CloseAsync(); + + // Insert into scheduled + var insertCmd = conn.CreateCommand(_writeDirectlyToTheScheduledTable); + insertCmd.Transaction = tx; + insertCmd.With("id", id); + insertCmd.Parameters.Add(new OracleParameter("body", OracleDbType.Blob) { Value = body }); + insertCmd.With("type", messageType); + insertCmd.Parameters.Add(new OracleParameter("expires", OracleDbType.TimeStampTZ) { Value = (object?)keepUntil ?? DBNull.Value }); + insertCmd.Parameters.Add(new OracleParameter("time", OracleDbType.TimeStampTZ) { Value = envelope.ScheduledTime!.Value }); + await insertCmd.ExecuteNonQueryAsync(cancellationToken); + + // Delete from outgoing + var deleteCmd = conn.CreateCommand( + $"DELETE FROM {_queue.Parent.MessageStorageSchemaName}.{DatabaseConstants.OutgoingTable} WHERE {DatabaseConstants.Id} = :id"); + deleteCmd.Transaction = tx; + deleteCmd.With("id", id); + await deleteCmd.ExecuteNonQueryAsync(cancellationToken); + } + else + { + await reader.CloseAsync(); + } + + await tx.CommitAsync(cancellationToken); + } + catch (OracleException e) when (e.Number == 1) // ORA-00001: unique constraint violated + { + // Clean up outgoing on duplicate + await using var cleanConn = await _dataSource.OpenConnectionAsync(cancellationToken); + try + { + var cleanCmd = cleanConn.CreateCommand( + $"DELETE FROM {_queue.Parent.MessageStorageSchemaName}.{DatabaseConstants.OutgoingTable} WHERE id = :id"); + cleanCmd.With("id", envelope.Id); + await cleanCmd.ExecuteNonQueryAsync(cancellationToken); + } + finally + { + await cleanConn.CloseAsync(); + } + } + finally + { + await conn.CloseAsync(); + } + } + + public async Task SendAsync(Envelope envelope, CancellationToken cancellationToken) + { + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); + try + { + if (envelope.IsScheduledForLater(DateTimeOffset.UtcNow)) + { + await writeToScheduledTableAsync(envelope, cancellationToken, conn); + } + else + { + try + { + var cmd = conn.CreateCommand(_writeDirectlyToQueueTableSql); + cmd.With("id", envelope.Id); + cmd.Parameters.Add(new OracleParameter("body", OracleDbType.Blob) { Value = EnvelopeSerializer.Serialize(envelope) }); + cmd.With("type", envelope.MessageType); + cmd.Parameters.Add(new OracleParameter("expires", OracleDbType.TimeStampTZ) { Value = (object?)envelope.DeliverBy ?? DBNull.Value }); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + catch (OracleException e) when (e.Number == 1) + { + // Making this idempotent + return; + } + } + } + finally + { + await conn.CloseAsync(); + } + } + + private async Task writeToScheduledTableAsync(Envelope envelope, CancellationToken cancellationToken, OracleConnection conn) + { + var cmd = conn.CreateCommand(_writeDirectlyToTheScheduledTable); + cmd.With("id", envelope.Id); + cmd.Parameters.Add(new OracleParameter("body", OracleDbType.Blob) { Value = EnvelopeSerializer.Serialize(envelope) }); + cmd.With("type", envelope.MessageType); + cmd.Parameters.Add(new OracleParameter("expires", OracleDbType.TimeStampTZ) { Value = (object?)envelope.DeliverBy ?? DBNull.Value }); + cmd.Parameters.Add(new OracleParameter("time", OracleDbType.TimeStampTZ) { Value = (object?)envelope.ScheduledTime ?? DBNull.Value }); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleSubscriberConfiguration.cs b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleSubscriberConfiguration.cs new file mode 100644 index 000000000..93d15eec5 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleSubscriberConfiguration.cs @@ -0,0 +1,10 @@ +using Wolverine.Configuration; + +namespace Wolverine.Oracle.Transport; + +public class OracleSubscriberConfiguration : SubscriberConfiguration +{ + public OracleSubscriberConfiguration(OracleQueue endpoint) : base(endpoint) + { + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleTransport.cs b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleTransport.cs new file mode 100644 index 000000000..b3c6d92fa --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleTransport.cs @@ -0,0 +1,115 @@ +using JasperFx; +using JasperFx.Core; +using Oracle.ManagedDataAccess.Client; +using Spectre.Console; +using Weasel.Oracle; +using Wolverine.Configuration; +using Wolverine.RDBMS.MultiTenancy; +using Wolverine.Runtime; +using Wolverine.Transports; +using MultiTenantedMessageStore = Wolverine.Persistence.Durability.MultiTenantedMessageStore; + +namespace Wolverine.Oracle.Transport; + +public class OracleTransport : BrokerTransport +{ + public const string ProtocolName = "oracle"; + + public OracleTransport() : base(ProtocolName, "Oracle Transport", [ProtocolName]) + { + Queues = new LightweightCache(name => new OracleQueue(name, this)); + } + + public override Uri ResourceUri => new Uri("oracle-transport://"); + + /// + /// Schema name for the queue and scheduled message tables + /// + public string TransportSchemaName { get; set; } = "WOLVERINE_QUEUES"; + + /// + /// Schema name for the message storage tables + /// + public string MessageStorageSchemaName { get; set; } = "WOLVERINE"; + + public LightweightCache Queues { get; } + + protected override IEnumerable endpoints() + { + return Queues; + } + + public override string SanitizeIdentifier(string identifier) + { + return identifier.Replace('-', '_').ToUpperInvariant(); + } + + protected override OracleQueue findEndpointByUri(Uri uri) + { + var queueName = uri.Host; + return Queues[queueName]; + } + + public override async ValueTask ConnectAsync(IWolverineRuntime runtime) + { + if (runtime.Storage is OracleMessageStore store) + { + Store = store; + } + else if (runtime.Storage is MultiTenantedMessageStore mt && mt.Main is OracleMessageStore s) + { + Store = s; + Databases = mt; + } + else + { + throw new ArgumentOutOfRangeException( + "The Oracle transport can only be used if Oracle is the backing message store"); + } + + // This is de facto a little environment test + await runtime.Storage.Admin.CheckConnectivityAsync(CancellationToken.None); + + AutoProvision = AutoProvision || runtime.Options.AutoBuildMessageStorageOnStartup != AutoCreate.None; + + foreach (var queue in Queues) + { + Store.AddTable(queue.QueueTable); + Store.AddTable(queue.ScheduledTable); + } + + MessageStorageSchemaName = Store.SchemaName; + } + + internal MultiTenantedMessageStore? Databases { get; set; } + + internal OracleMessageStore? Store { get; set; } + + public override IEnumerable DiagnosticColumns() + { + yield return new PropertyColumn("Name"); + yield return new PropertyColumn("Count", Justify.Right); + yield return new PropertyColumn("Scheduled", Justify.Right); + } + + public async Task SystemTimeAsync() + { + OracleDataSource? dataSource = null; + if (Store is OracleMessageStore store) + { + dataSource = store.OracleDataSource; + } + + await using var conn = await dataSource!.OpenConnectionAsync(); + try + { + var cmd = conn.CreateCommand("SELECT SYS_EXTRACT_UTC(SYSTIMESTAMP) FROM DUAL"); + var raw = await cmd.ExecuteScalarAsync(); + return (DateTimeOffset)raw!; + } + finally + { + await conn.CloseAsync(); + } + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Transport/QueueTable.cs b/src/Persistence/Oracle/Wolverine.Oracle/Transport/QueueTable.cs new file mode 100644 index 000000000..a5fbfe31e --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Transport/QueueTable.cs @@ -0,0 +1,18 @@ +using Weasel.Oracle; +using Weasel.Oracle.Tables; +using Wolverine.RDBMS; + +namespace Wolverine.Oracle.Transport; + +internal class QueueTable : Table +{ + public QueueTable(OracleTransport parent, string tableName) : base( + new OracleObjectName(parent.TransportSchemaName.ToUpperInvariant(), tableName.ToUpperInvariant())) + { + AddColumn(DatabaseConstants.Id).AsPrimaryKey(); + AddColumn(DatabaseConstants.Body, "BLOB").NotNull(); + AddColumn(DatabaseConstants.MessageType, "VARCHAR2(500)").NotNull(); + AddColumn(DatabaseConstants.KeepUntil); + AddColumn("timestamp").DefaultValueByExpression("SYS_EXTRACT_UTC(SYSTIMESTAMP)"); + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Transport/ScheduledMessageTable.cs b/src/Persistence/Oracle/Wolverine.Oracle/Transport/ScheduledMessageTable.cs new file mode 100644 index 000000000..053ce0e2b --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Transport/ScheduledMessageTable.cs @@ -0,0 +1,24 @@ +using Weasel.Oracle; +using Weasel.Oracle.Tables; +using Wolverine.RDBMS; + +namespace Wolverine.Oracle.Transport; + +internal class ScheduledMessageTable : Table +{ + public ScheduledMessageTable(OracleTransport parent, string tableName) : base( + new OracleObjectName(parent.TransportSchemaName.ToUpperInvariant(), tableName.ToUpperInvariant())) + { + AddColumn(DatabaseConstants.Id).AsPrimaryKey(); + AddColumn(DatabaseConstants.Body, "BLOB").NotNull(); + AddColumn(DatabaseConstants.MessageType, "VARCHAR2(500)").NotNull(); + AddColumn(DatabaseConstants.ExecutionTime).NotNull(); + AddColumn(DatabaseConstants.KeepUntil); + AddColumn("timestamp").DefaultValueByExpression("SYS_EXTRACT_UTC(SYSTIMESTAMP)"); + + Indexes.Add(new IndexDefinition($"idx_{tableName}_execution_time") + { + Columns = [DatabaseConstants.ExecutionTime] + }); + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Util/OracleCommandExtensions.cs b/src/Persistence/Oracle/Wolverine.Oracle/Util/OracleCommandExtensions.cs new file mode 100644 index 000000000..35b775dd9 --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Util/OracleCommandExtensions.cs @@ -0,0 +1,76 @@ +using System.Data.Common; +using Oracle.ManagedDataAccess.Client; + +namespace Wolverine.Oracle.Util; + +public static class OracleCommandExtensions +{ + /// + /// Adds envelope IDs as individual parameters for use in an IN clause. + /// Oracle uses :param syntax. Returns the parameter placeholder string (e.g., ":id0, :id1, :id2") + /// + internal static string WithEnvelopeIds(this DbCommand command, string name, Envelope[] envelopes) + { + if (envelopes.Length == 0) + { + return "NULL"; + } + + var placeholders = new string[envelopes.Length]; + for (var i = 0; i < envelopes.Length; i++) + { + var paramName = $"{name}_{i}"; + placeholders[i] = $":{paramName}"; + + var parameter = command.CreateParameter(); + parameter.ParameterName = paramName; + parameter.Value = envelopes[i].Id.ToByteArray(); + command.Parameters.Add(parameter); + } + + return string.Join(", ", placeholders); + } + + /// + /// Adds GUID IDs as individual parameters for use in an IN clause. + /// Returns the parameter placeholder string (e.g., ":id0, :id1, :id2") + /// + internal static string WithIdList(this DbCommand command, string name, IReadOnlyList ids) + { + if (ids.Count == 0) + { + return "NULL"; + } + + var placeholders = new string[ids.Count]; + for (var i = 0; i < ids.Count; i++) + { + var paramName = $"{name}_{i}"; + placeholders[i] = $":{paramName}"; + + var parameter = command.CreateParameter(); + parameter.ParameterName = paramName; + parameter.Value = ids[i].ToByteArray(); + command.Parameters.Add(parameter); + } + + return string.Join(", ", placeholders); + } + + /// + /// Execute a reader and map each row to T. + /// + internal static async Task> FetchListAsync(this OracleCommand command, + Func> transform, + CancellationToken cancellation = default) + { + var list = new List(); + await using var reader = await command.ExecuteReaderAsync(cancellation); + while (await reader.ReadAsync(cancellation)) + { + list.Add(await transform(reader)); + } + + return list; + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Util/OracleEnvelopeReader.cs b/src/Persistence/Oracle/Wolverine.Oracle/Util/OracleEnvelopeReader.cs new file mode 100644 index 000000000..fb8ab171f --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Util/OracleEnvelopeReader.cs @@ -0,0 +1,110 @@ +using System.Data.Common; +using Wolverine.Persistence.Durability; +using Wolverine.RDBMS; +using Wolverine.Runtime; +using Wolverine.Runtime.Serialization; + +namespace Wolverine.Oracle.Util; + +/// +/// Oracle-specific envelope readers. Oracle stores Guids as RAW(16) which returns byte[], +/// not Guid directly. These methods handle the byte[] → Guid conversion that the shared +/// DatabasePersistence readers don't handle. +/// +internal static class OracleEnvelopeReader +{ + public static async Task ReadIncomingAsync(DbDataReader reader, CancellationToken cancellation = default) + { + var body = await reader.GetFieldValueAsync(0, cancellation); + var envelope = body.Length > 0 ? EnvelopeSerializer.Deserialize(body) : new Envelope { Message = new PlaceHolder() }; + envelope.Id = ReadGuid(reader, 1); + envelope.Status = Enum.Parse(await reader.GetFieldValueAsync(2, cancellation)); + envelope.OwnerId = Convert.ToInt32(reader.GetValue(3)); + envelope.MessageType = await reader.GetFieldValueAsync(6, cancellation); + + var rawUri = await reader.GetFieldValueAsync(7, cancellation); + envelope.Destination = new Uri(rawUri); + + if (!await reader.IsDBNullAsync(4, cancellation)) + { + envelope.ScheduledTime = await reader.GetFieldValueAsync(4, cancellation); + } + + envelope.Attempts = Convert.ToInt32(reader.GetValue(5)); + + if (!await reader.IsDBNullAsync(8, cancellation)) + { + envelope.KeepUntil = await reader.GetFieldValueAsync(8, cancellation); + } + + return envelope; + } + + public static async Task ReadOutgoingAsync(DbDataReader reader, CancellationToken cancellation = default) + { + var body = await reader.GetFieldValueAsync(0, cancellation); + var envelope = EnvelopeSerializer.Deserialize(body); + envelope.OwnerId = Convert.ToInt32(reader.GetValue(2)); + + if (!await reader.IsDBNullAsync(4, cancellation)) + { + envelope.DeliverBy = await reader.GetFieldValueAsync(4, cancellation); + } + + envelope.Attempts = Convert.ToInt32(reader.GetValue(5)); + + return envelope; + } + + public static async Task ReadDeadLetterAsync(DbDataReader reader, CancellationToken cancellation = default) + { + var id = ReadGuid(reader, 0); + var executionTime = await reader.IsDBNullAsync(1, cancellation).ConfigureAwait(false) ? null : await reader.GetFieldValueAsync(1, cancellation); + var envelope = EnvelopeSerializer.Deserialize(await reader.GetFieldValueAsync(2, cancellation)); + var messageType = await reader.GetFieldValueAsync(3, cancellation); + var receivedAt = await reader.GetFieldValueAsync(4, cancellation); + var source = await reader.GetFieldValueAsync(5, cancellation); + var exceptionType = await reader.GetFieldValueAsync(6, cancellation); + var exceptionMessage = await reader.GetFieldValueAsync(7, cancellation); + var sentAt = await reader.GetFieldValueAsync(8, cancellation); + + // Oracle stores bool as NUMBER(1) - read as decimal and convert + var replayableValue = reader.GetValue(9); + var replayable = Convert.ToInt32(replayableValue) != 0; + + return new DeadLetterEnvelope( + id, + executionTime, + envelope, + messageType, + receivedAt, + source, + exceptionType, + exceptionMessage, + sentAt, + replayable + ); + } + + /// + /// Read a Guid from an Oracle RAW(16) column. Oracle returns byte[] for RAW columns. + /// + internal static Guid ReadGuid(DbDataReader reader, int ordinal) + { + var value = reader.GetValue(ordinal); + if (value is Guid g) return g; + if (value is byte[] bytes) return new Guid(bytes); + throw new InvalidCastException($"Cannot convert {value?.GetType().Name ?? "null"} to Guid at ordinal {ordinal}"); + } + + /// + /// Async version: Read a Guid from an Oracle RAW(16) column. + /// + internal static async Task ReadGuidAsync(DbDataReader reader, int ordinal, CancellationToken cancellation = default) + { + var value = await reader.GetFieldValueAsync(ordinal, cancellation); + if (value is Guid g) return g; + if (value is byte[] bytes) return new Guid(bytes); + throw new InvalidCastException($"Cannot convert {value?.GetType().Name ?? "null"} to Guid at ordinal {ordinal}"); + } +} diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Wolverine.Oracle.csproj b/src/Persistence/Oracle/Wolverine.Oracle/Wolverine.Oracle.csproj new file mode 100644 index 000000000..6a0c7acaa --- /dev/null +++ b/src/Persistence/Oracle/Wolverine.Oracle/Wolverine.Oracle.csproj @@ -0,0 +1,25 @@ + + + + Oracle Saga, Message, and Outbox storage for Wolverine applications + WolverineFx.Oracle + false + false + false + false + false + + + + + + + + + + + + + + + diff --git a/src/Persistence/Wolverine.RDBMS/AssemblyAttributes.cs b/src/Persistence/Wolverine.RDBMS/AssemblyAttributes.cs index b1fe7aab6..c7d4004cc 100644 --- a/src/Persistence/Wolverine.RDBMS/AssemblyAttributes.cs +++ b/src/Persistence/Wolverine.RDBMS/AssemblyAttributes.cs @@ -5,4 +5,6 @@ [assembly: InternalsVisibleTo("PostgresqlTests")] [assembly: InternalsVisibleTo("MartenTests")] [assembly: InternalsVisibleTo("SqliteTests")] +[assembly: InternalsVisibleTo("Wolverine.Oracle")] +[assembly: InternalsVisibleTo("OracleTests")] [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] // Castle Core proxies for NSubstitute \ No newline at end of file diff --git a/src/Servers.cs b/src/Servers.cs index 5caf36bf3..2210f0735 100644 --- a/src/Servers.cs +++ b/src/Servers.cs @@ -10,4 +10,7 @@ public class Servers public static readonly string MySqlConnectionString = "Server=localhost;Port=3306;Database=wolverine;User=root;Password=P@55w0rd;"; + + public static readonly string OracleConnectionString = + "User Id=wolverine;Password=wolverine;Data Source=localhost:1521/FREEPDB1"; } \ No newline at end of file diff --git a/wolverine.sln b/wolverine.sln index 24449047a..0f06c1097 100644 --- a/wolverine.sln +++ b/wolverine.sln @@ -346,6 +346,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wolverine.Sqlite", "src\Per EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SqliteTests", "src\Persistence\SqliteTests\SqliteTests.csproj", "{57B8F129-50EA-4803-AEB7-FE655B6D1B81}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wolverine.Oracle", "src\Persistence\Oracle\Wolverine.Oracle\Wolverine.Oracle.csproj", "{B1294A26-2C75-42A8-8A9F-9758664F6988}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OracleTests", "src\Persistence\Oracle\OracleTests\OracleTests.csproj", "{C86E3CE6-3A97-451F-945D-61B3D5070160}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1928,6 +1932,30 @@ Global {57B8F129-50EA-4803-AEB7-FE655B6D1B81}.Release|x64.Build.0 = Release|Any CPU {57B8F129-50EA-4803-AEB7-FE655B6D1B81}.Release|x86.ActiveCfg = Release|Any CPU {57B8F129-50EA-4803-AEB7-FE655B6D1B81}.Release|x86.Build.0 = Release|Any CPU + {B1294A26-2C75-42A8-8A9F-9758664F6988}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B1294A26-2C75-42A8-8A9F-9758664F6988}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B1294A26-2C75-42A8-8A9F-9758664F6988}.Debug|x64.ActiveCfg = Debug|Any CPU + {B1294A26-2C75-42A8-8A9F-9758664F6988}.Debug|x64.Build.0 = Debug|Any CPU + {B1294A26-2C75-42A8-8A9F-9758664F6988}.Debug|x86.ActiveCfg = Debug|Any CPU + {B1294A26-2C75-42A8-8A9F-9758664F6988}.Debug|x86.Build.0 = Debug|Any CPU + {B1294A26-2C75-42A8-8A9F-9758664F6988}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B1294A26-2C75-42A8-8A9F-9758664F6988}.Release|Any CPU.Build.0 = Release|Any CPU + {B1294A26-2C75-42A8-8A9F-9758664F6988}.Release|x64.ActiveCfg = Release|Any CPU + {B1294A26-2C75-42A8-8A9F-9758664F6988}.Release|x64.Build.0 = Release|Any CPU + {B1294A26-2C75-42A8-8A9F-9758664F6988}.Release|x86.ActiveCfg = Release|Any CPU + {B1294A26-2C75-42A8-8A9F-9758664F6988}.Release|x86.Build.0 = Release|Any CPU + {C86E3CE6-3A97-451F-945D-61B3D5070160}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C86E3CE6-3A97-451F-945D-61B3D5070160}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C86E3CE6-3A97-451F-945D-61B3D5070160}.Debug|x64.ActiveCfg = Debug|Any CPU + {C86E3CE6-3A97-451F-945D-61B3D5070160}.Debug|x64.Build.0 = Debug|Any CPU + {C86E3CE6-3A97-451F-945D-61B3D5070160}.Debug|x86.ActiveCfg = Debug|Any CPU + {C86E3CE6-3A97-451F-945D-61B3D5070160}.Debug|x86.Build.0 = Debug|Any CPU + {C86E3CE6-3A97-451F-945D-61B3D5070160}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C86E3CE6-3A97-451F-945D-61B3D5070160}.Release|Any CPU.Build.0 = Release|Any CPU + {C86E3CE6-3A97-451F-945D-61B3D5070160}.Release|x64.ActiveCfg = Release|Any CPU + {C86E3CE6-3A97-451F-945D-61B3D5070160}.Release|x64.Build.0 = Release|Any CPU + {C86E3CE6-3A97-451F-945D-61B3D5070160}.Release|x86.ActiveCfg = Release|Any CPU + {C86E3CE6-3A97-451F-945D-61B3D5070160}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -2087,6 +2115,8 @@ Global {4E69232F-1E78-4486-9406-EDB991DFDC9A} = {7A9E0EAE-9ABF-40F6-9DB9-8FB1243F4210} {1F419B33-AD3A-4F30-8083-AC37F7F33F12} = {7A9E0EAE-9ABF-40F6-9DB9-8FB1243F4210} {57B8F129-50EA-4803-AEB7-FE655B6D1B81} = {7A9E0EAE-9ABF-40F6-9DB9-8FB1243F4210} + {B1294A26-2C75-42A8-8A9F-9758664F6988} = {7A9E0EAE-9ABF-40F6-9DB9-8FB1243F4210} + {C86E3CE6-3A97-451F-945D-61B3D5070160} = {7A9E0EAE-9ABF-40F6-9DB9-8FB1243F4210} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {30422362-0D90-4DBE-8C97-DD2B5B962768} From 14f2d027b5cf5684fcfc603b549b3e64a43be97f Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Tue, 10 Feb 2026 20:43:03 -0600 Subject: [PATCH 2/2] Switch Weasel.Oracle from project reference to NuGet 8.6.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the local project references to Weasel.Oracle and Weasel.Core with the published Weasel.Oracle 8.6.1 NuGet package which includes the Oracle-specific Guid→byte[] conversions and BindByName support. Co-Authored-By: Claude Opus 4.6 --- .../Oracle/Wolverine.Oracle/Wolverine.Oracle.csproj | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Wolverine.Oracle.csproj b/src/Persistence/Oracle/Wolverine.Oracle/Wolverine.Oracle.csproj index 6a0c7acaa..edbf03770 100644 --- a/src/Persistence/Oracle/Wolverine.Oracle/Wolverine.Oracle.csproj +++ b/src/Persistence/Oracle/Wolverine.Oracle/Wolverine.Oracle.csproj @@ -12,13 +12,11 @@ - - - +