From a142a3e0dab39793dcbe9787e12c4a6c54463d6f Mon Sep 17 00:00:00 2001 From: emeraldleaf Date: Sat, 23 May 2026 08:22:42 -0600 Subject: [PATCH] fix(integration-tests): unblock OrderService integration tests in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OrderService.Tests.Integration suite has hung in CI on every run since it was added 9 days ago (commit 76b2f2f, 2026-05-15). Step #6 ("Integration tests — Order") runs for ~19m14s before the job hits the runner's max and gets cancelled. The suite has never been observed green in CI. Two distinct bugs, in sequence — the first one hides the second. ROOT CAUSE #1 — Wolverine AutoProvision hangs against fake ASB endpoint Each service's Program.cs calls `.AutoProvision()` on the Azure Service Bus transport: opts.UseAzureServiceBus(connectionString).AutoProvision(); AutoProvision is a host-startup hook that actively connects to the Service Bus namespace to create topics/subscriptions if they don't exist. The integration test factory uses a syntactically-valid-but-fake connection string ("sb://fake.servicebus.windows.net/...") so Wolverine can register without throwing, and calls `DisableAllExternalWolverineTransports()` via ConfigureTestServices to turn off the actual message routing. But — DisableAllExternalWolverineTransports() is a DI registration; AutoProvision runs at host startup via a transport bootstrapper that fires BEFORE ConfigureTestServices is applied. So AutoProvision tries to reach the fake endpoint, the Azure SDK retries with backoff, and the host startup hangs until the runner kills the job at 20 minutes. Variant pattern: same .AutoProvision() call exists in OrderService, PaymentService, ShippingService, NotificationService. CatalogService is the working reference — it uses Wolverine but no Azure Service Bus transport, so no AutoProvision call, so no hang. Fix: gate .AutoProvision() on a config flag (defaults to true so dev/prod are unchanged). Test factory sets the flag to false: builder.UseSetting("Wolverine:AutoProvision", "false"); Applied to all four services even though only OrderService has integration tests today; preventing the same trap when integration tests get wired up for Payment / Shipping / Notification. ROOT CAUSE #2 — Dispose order crashes the test host AFTER tests pass With root cause #1 fixed, the 4 tests passed in 38 seconds — then the test host crashed because OrderApiFactory.DisposeAsync stops the SQL Server container BEFORE calling base.DisposeAsync. Wolverine's DurableReceiver (a BackgroundService that polls the wolverine.* outbox tables) is still running during that window; every heartbeat hits "connection refused" and the unhandled SqlException bubbles up: Total tests: Unknown Passed: 4 Test Run Aborted. Fix: dispose host first (await base.DisposeAsync()), then SQL container. Lets Wolverine's background services exit gracefully before the DB is yanked. VERIFICATION Locally with Docker Desktop: Test Run Successful. Total tests: 4 Passed: 4 Total time: 16.2220 Seconds The full saga runs end-to-end: PlaceOrder → OrderPlacedEvent published, second test asserts no orphan row on validation failure, PaymentCompletedEvent transitions Placed → Paid (idempotently), RowVersion concurrency token fires. Co-Authored-By: Claude Opus 4.7 --- NotificationService/Program.cs | 13 +++++++++++-- OrderService/Program.cs | 19 +++++++++++++++++-- PaymentService/Program.cs | 13 +++++++++++-- ShippingService/Program.cs | 13 +++++++++++-- .../OrderApiFactory.cs | 17 ++++++++++++++++- 5 files changed, 66 insertions(+), 9 deletions(-) diff --git a/NotificationService/Program.cs b/NotificationService/Program.cs index 237b7c2b..15fcb5e5 100644 --- a/NotificationService/Program.cs +++ b/NotificationService/Program.cs @@ -14,8 +14,17 @@ builder.Host.UseWolverine(opts => { var connectionString = builder.Configuration.GetConnectionString("messaging")!; - opts.UseAzureServiceBus(connectionString) - .AutoProvision(); + var azureServiceBus = opts.UseAzureServiceBus(connectionString); + + // AutoProvision creates topics/subscriptions at host startup. Test harnesses use a + // fake ASB connection string that would hang the connection attempt. Gate on a config + // flag so tests can disable provisioning while leaving the rest of the + // Development-gated code (EF migration, OpenAPI) intact. See OrderService/Program.cs + // for the full rationale. + if (builder.Configuration.GetValue("Wolverine:AutoProvision", defaultValue: true)) + { + azureServiceBus.AutoProvision(); + } // Listen to events from other services opts.ListenToAzureServiceBusSubscription("order-events/notify-orders-sub"); diff --git a/OrderService/Program.cs b/OrderService/Program.cs index ae139059..0387f261 100644 --- a/OrderService/Program.cs +++ b/OrderService/Program.cs @@ -19,8 +19,23 @@ builder.Host.UseWolverine(opts => { var connectionString = builder.Configuration.GetConnectionString("messaging")!; - opts.UseAzureServiceBus(connectionString) - .AutoProvision(); + var azureServiceBus = opts.UseAzureServiceBus(connectionString); + + // AutoProvision creates topics/subscriptions on the Service Bus namespace at host + // startup. That's a real network operation against the configured endpoint, and it + // happens even when Wolverine's external transports are otherwise stubbed — + // provisioning fires before DisableAllExternalWolverineTransports() from a test + // factory's ConfigureTestServices takes effect. Integration tests use a fake ASB + // connection string ("sb://fake.servicebus.windows.net/...") so AutoProvision hangs + // trying to resolve/connect, eventually timing out at the CI runner's job limit. + // + // Gate on a config flag (not environment) so tests can disable provisioning while + // leaving the rest of the Development-gated code (EF migration, OpenAPI, Scalar) + // intact. Defaults to true so dev/prod behavior is unchanged. + if (builder.Configuration.GetValue("Wolverine:AutoProvision", defaultValue: true)) + { + azureServiceBus.AutoProvision(); + } // Publish outgoing events to their topics opts.PublishMessage().ToAzureServiceBusTopic("order-events"); diff --git a/PaymentService/Program.cs b/PaymentService/Program.cs index 69963554..eaa2b5de 100644 --- a/PaymentService/Program.cs +++ b/PaymentService/Program.cs @@ -32,8 +32,17 @@ builder.Host.UseWolverine(opts => { var connectionString = builder.Configuration.GetConnectionString("messaging")!; - opts.UseAzureServiceBus(connectionString) - .AutoProvision(); + var azureServiceBus = opts.UseAzureServiceBus(connectionString); + + // AutoProvision creates topics/subscriptions at host startup. Test harnesses use a + // fake ASB connection string that would hang the connection attempt. Gate on a config + // flag so tests can disable provisioning while leaving the rest of the + // Development-gated code (EF migration, OpenAPI) intact. See OrderService/Program.cs + // for the full rationale. + if (builder.Configuration.GetValue("Wolverine:AutoProvision", defaultValue: true)) + { + azureServiceBus.AutoProvision(); + } // Publish outgoing events to their topics opts.PublishMessage().ToAzureServiceBusTopic("payment-events"); diff --git a/ShippingService/Program.cs b/ShippingService/Program.cs index fb6238f6..4d3ee103 100644 --- a/ShippingService/Program.cs +++ b/ShippingService/Program.cs @@ -17,8 +17,17 @@ builder.Host.UseWolverine(opts => { var connectionString = builder.Configuration.GetConnectionString("messaging")!; - opts.UseAzureServiceBus(connectionString) - .AutoProvision(); + var azureServiceBus = opts.UseAzureServiceBus(connectionString); + + // AutoProvision creates topics/subscriptions at host startup. Test harnesses use a + // fake ASB connection string that would hang the connection attempt. Gate on a config + // flag so tests can disable provisioning while leaving the rest of the + // Development-gated code (EF migration, OpenAPI) intact. See OrderService/Program.cs + // for the full rationale. + if (builder.Configuration.GetValue("Wolverine:AutoProvision", defaultValue: true)) + { + azureServiceBus.AutoProvision(); + } // Publish outgoing events to their topics opts.PublishMessage().ToAzureServiceBusTopic("shipping-events"); diff --git a/tests/OrderService.Tests.Integration/OrderApiFactory.cs b/tests/OrderService.Tests.Integration/OrderApiFactory.cs index 51844555..f9e97c2d 100644 --- a/tests/OrderService.Tests.Integration/OrderApiFactory.cs +++ b/tests/OrderService.Tests.Integration/OrderApiFactory.cs @@ -61,14 +61,29 @@ public async Task InitializeAsync() public override async ValueTask DisposeAsync() { - await _sqlServer.DisposeAsync(); + // Dispose the host BEFORE the SQL container. Wolverine's DurableReceiver + // background service polls the wolverine.* outbox tables on a heartbeat; if SQL + // Server is torn down while the receiver is still running, every heartbeat hits + // "connection refused" and the unhandled exceptions crash the test host AFTER + // all tests passed. base.DisposeAsync() runs the host's StopAsync, which lets + // Wolverine's background services exit gracefully before we yank the DB. await base.DisposeAsync(); + await _sqlServer.DisposeAsync(); } protected override void ConfigureWebHost(IWebHostBuilder builder) { ArgumentNullException.ThrowIfNull(builder); + // Disable Wolverine's .AutoProvision() in Program.cs. AutoProvision tries to + // create topics/subscriptions on the configured Service Bus namespace at host + // startup; against our fake "sb://fake.servicebus.windows.net/..." connection + // string it hangs trying to resolve/connect, eventually killing the test job + // at the runner's job limit. DisableAllExternalWolverineTransports() below + // handles the message-routing path; this handles broker-provisioning, which + // runs *before* ConfigureTestServices fires. + builder.UseSetting("Wolverine:AutoProvision", "false"); + // Real SQL Server for the order DB + Wolverine outbox tables. builder.UseSetting("ConnectionStrings:orders-db", _sqlServer.GetConnectionString());