From 85146f0b5f3ddb8d4d38aafea6a1e077a9e707eb Mon Sep 17 00:00:00 2001 From: Pascal Senn Date: Tue, 31 Mar 2026 21:33:27 +0200 Subject: [PATCH 01/14] Add retry/redeliver to mocha --- src/Mocha/src/Demo/Demo.Billing/Program.cs | 2 + src/Mocha/src/Demo/Demo.Catalog/Program.cs | 2 + src/Mocha/src/Demo/Demo.Shipping/Program.cs | 2 + .../IMessageBusHostBuilderExtensions.cs | 59 ++ src/Mocha/src/Mocha/Headers/MessageHeaders.cs | 11 + .../Consume/ConsumerMiddlewares.cs | 5 + .../Consume/Retry/ConsumerRetryMiddleware.cs | 185 +++++++ .../Consume/Retry/ExceptionPolicyBuilder.cs | 35 ++ .../Consume/Retry/ExceptionRule.cs | 33 ++ .../Consume/Retry/ExceptionRuleMatcher.cs | 52 ++ .../Consume/Retry/RetryBackoffType.cs | 22 + .../Retry/RetryConfigurationExtensions.cs | 90 +++ .../Middlewares/Consume/Retry/RetryFeature.cs | 75 +++ .../Middlewares/Consume/Retry/RetryOptions.cs | 102 ++++ .../Middlewares/Consume/Retry/RetryState.cs | 21 + .../MiddlewareConfigurationExtensions.cs | 6 + .../Middlewares/Receive/ReceiveMiddlewares.cs | 5 + .../Redelivery/ReceiveRedeliveryMiddleware.cs | 247 +++++++++ .../RedeliveryConfigurationExtensions.cs | 93 ++++ .../Receive/Redelivery/RedeliveryFeature.cs | 70 +++ .../Receive/Redelivery/RedeliveryOptions.cs | 92 ++++ .../Behaviors/RedeliveryTests.cs | 274 ++++++++++ .../Behaviors/RetryTests.cs | 516 ++++++++++++++++++ website/src/docs/mocha/v1/reliability.md | 451 ++++++++++++++- 24 files changed, 2438 insertions(+), 12 deletions(-) create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRule.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRuleMatcher.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryBackoffType.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryConfigurationExtensions.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryFeature.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryOptions.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryState.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryConfigurationExtensions.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryFeature.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryOptions.cs create mode 100644 src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs create mode 100644 src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs diff --git a/src/Mocha/src/Demo/Demo.Billing/Program.cs b/src/Mocha/src/Demo/Demo.Billing/Program.cs index 98757e06b61..9fd76fc4bb9 100644 --- a/src/Mocha/src/Demo/Demo.Billing/Program.cs +++ b/src/Mocha/src/Demo/Demo.Billing/Program.cs @@ -31,6 +31,8 @@ builder .Services.AddMessageBus() .AddInstrumentation() + .AddRetry() + .AddRedelivery() .AddBilling() .AddBatchHandler(opts => { diff --git a/src/Mocha/src/Demo/Demo.Catalog/Program.cs b/src/Mocha/src/Demo/Demo.Catalog/Program.cs index f6c2583b767..d81542bb287 100644 --- a/src/Mocha/src/Demo/Demo.Catalog/Program.cs +++ b/src/Mocha/src/Demo/Demo.Catalog/Program.cs @@ -31,6 +31,8 @@ builder .Services.AddMessageBus() .AddInstrumentation() + .AddRetry() + .AddRedelivery() .AddCatalog() .AddEntityFramework(p => { diff --git a/src/Mocha/src/Demo/Demo.Shipping/Program.cs b/src/Mocha/src/Demo/Demo.Shipping/Program.cs index eb08a0c7865..a6f0553fb7d 100644 --- a/src/Mocha/src/Demo/Demo.Shipping/Program.cs +++ b/src/Mocha/src/Demo/Demo.Shipping/Program.cs @@ -28,6 +28,8 @@ builder .Services.AddMessageBus() .AddInstrumentation() + .AddRetry() + .AddRedelivery() .AddShipping() .AddEntityFramework(p => { diff --git a/src/Mocha/src/Mocha/Extensions/IMessageBusHostBuilderExtensions.cs b/src/Mocha/src/Mocha/Extensions/IMessageBusHostBuilderExtensions.cs index 62045a64ad4..e282d6ba9e9 100644 --- a/src/Mocha/src/Mocha/Extensions/IMessageBusHostBuilderExtensions.cs +++ b/src/Mocha/src/Mocha/Extensions/IMessageBusHostBuilderExtensions.cs @@ -29,6 +29,26 @@ public static IMessageBusHostBuilder AddEventHandler< return builder; } + /// + /// Registers an event handler with the message bus and adds it to the service collection, + /// with additional consumer configuration. + /// + /// The event handler type. + /// The host builder. + /// The action to configure the consumer descriptor. + /// The builder for method chaining. + public static IMessageBusHostBuilder AddEventHandler< + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] THandler>( + this IMessageBusHostBuilder builder, + Action configure) + where THandler : class, IEventHandler + { + builder.Services.TryAddScoped(); + builder.ConfigureMessageBus(h => h.AddHandler(configure)); + + return builder; + } + /// /// Registers a handler using pre-built configuration from the source generator. /// @@ -94,6 +114,25 @@ public static IMessageBusHostBuilder AddRequestHandler(this IMessageBu return builder; } + /// + /// Registers a request handler with the message bus and adds it to the service collection, + /// with additional consumer configuration. + /// + /// The request handler type. + /// The host builder. + /// The action to configure the consumer descriptor. + /// The builder for method chaining. + public static IMessageBusHostBuilder AddRequestHandler( + this IMessageBusHostBuilder builder, + Action configure) + where THandler : class, IEventRequestHandler + { + builder.Services.TryAddScoped(); + builder.ConfigureMessageBus(h => h.AddHandler(configure)); + + return builder; + } + /// /// Registers a consumer with the message bus and adds it to the service collection. /// @@ -111,6 +150,26 @@ public static IMessageBusHostBuilder AddConsumer< return builder; } + /// + /// Registers a consumer with the message bus and adds it to the service collection, + /// with additional consumer configuration. + /// + /// The consumer type implementing . + /// The host builder. + /// The action to configure the consumer descriptor. + /// The builder for method chaining. + public static IMessageBusHostBuilder AddConsumer< + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] TConsumer>( + this IMessageBusHostBuilder builder, + Action configure) + where TConsumer : class, IConsumer + { + builder.Services.TryAddScoped(); + builder.ConfigureMessageBus(h => h.AddHandler(configure)); + + return builder; + } + /// /// Registers additional services into the internal service collection used by the message bus. /// This is the message bus equivalent of Hot Chocolate's ConfigureSchemaServices. diff --git a/src/Mocha/src/Mocha/Headers/MessageHeaders.cs b/src/Mocha/src/Mocha/Headers/MessageHeaders.cs index 6b22af15f64..ce37e775144 100644 --- a/src/Mocha/src/Mocha/Headers/MessageHeaders.cs +++ b/src/Mocha/src/Mocha/Headers/MessageHeaders.cs @@ -20,6 +20,17 @@ internal static class MessageHeaders /// public static readonly ContextDataKey MessageKind = new("message-kind"); + /// + /// Defines header keys used by the retry infrastructure. + /// + public static class Retry + { + /// + /// The header key for tracking the number of delayed redelivery attempts. + /// + public static readonly ContextDataKey DelayedRetryCount = new("delayed-retry-count"); + } + /// /// Defines header keys for fault information attached to messages that failed processing. /// diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/ConsumerMiddlewares.cs b/src/Mocha/src/Mocha/Middlewares/Consume/ConsumerMiddlewares.cs index 01dda0cc482..de1e4af845b 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/ConsumerMiddlewares.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/ConsumerMiddlewares.cs @@ -7,6 +7,11 @@ namespace Mocha; /// public static class ConsumerMiddlewares { + /// + /// The retry middleware configuration that retries failed handler invocations with configurable backoff. + /// + public static readonly ConsumerMiddlewareConfiguration Retry = ConsumerRetryMiddleware.Create(); + /// /// The instrumentation middleware configuration that emits telemetry for consumer operations. /// diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs new file mode 100644 index 00000000000..218c550f05f --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs @@ -0,0 +1,185 @@ +using Microsoft.Extensions.DependencyInjection; +using Polly; +using Polly.Retry; + +namespace Mocha; + +/// +/// A consumer middleware that implements in-process retry using Polly, replaying the handler +/// with configurable backoff strategies when transient failures occur. +/// +internal sealed class ConsumerRetryMiddleware( + ResiliencePipeline resiliencePipeline) +{ + public async ValueTask InvokeAsync(IConsumeContext context, ConsumerDelegate next) + { + // Read delayed retry count from headers (set by redelivery middleware). + var delayedRetryCount = 0; + + if (context.Headers.TryGetValue(MessageHeaders.Retry.DelayedRetryCount.Key, out var headerValue) + && headerValue is int count) + { + delayedRetryCount = count; + } + + // Expose retry state to handlers via features. + // ImmediateRetryCount starts at -1 so the first increment in the callback yields 0. + var retryState = new RetryState + { + DelayedRetryCount = delayedRetryCount, + ImmediateRetryCount = -1 + }; + context.Features.Set(retryState); + + await resiliencePipeline.ExecuteAsync( + static (state, _) => + { + state.retryState.ImmediateRetryCount++; + return state.next(state.context); + }, + (context, next, retryState), + context.CancellationToken); + } + + public static ConsumerMiddlewareConfiguration Create() + => new( + static (context, next) => + { + // Feature values are resolved from consumer -> bus to support overrides. + var enabled = context.GetConfiguration(f => f.Enabled) ?? true; + + if (!enabled) + { + return next; + } + + var intervals = context.GetConfiguration(f => f.Intervals); + var maxRetryAttempts = intervals?.Length + ?? context.GetConfiguration(f => f.MaxRetryAttempts) + ?? RetryOptions.Defaults.MaxRetryAttempts; + + var delay = context.GetConfiguration(f => f.Delay) + ?? RetryOptions.Defaults.Delay; + + var maxDelay = context.GetConfiguration(f => f.MaxDelay) + ?? RetryOptions.Defaults.MaxDelay; + + var backoffType = context.GetConfiguration(f => f.BackoffType) + ?? RetryOptions.Defaults.BackoffType; + + var useJitter = context.GetConfiguration(f => f.UseJitter) + ?? RetryOptions.Defaults.UseJitter; + + // Resolve exception rules (atomically from first scope that has them). + var exceptionRules = ResolveExceptionRules(context); + + // Map RetryBackoffType to Polly's DelayBackoffType. + var pollyBackoffType = backoffType switch + { + RetryBackoffType.Constant => DelayBackoffType.Constant, + RetryBackoffType.Linear => DelayBackoffType.Linear, + RetryBackoffType.Exponential => DelayBackoffType.Exponential, + _ => DelayBackoffType.Exponential + }; + + var strategyOptions = new RetryStrategyOptions + { + MaxRetryAttempts = maxRetryAttempts, + Delay = delay, + MaxDelay = maxDelay, + BackoffType = pollyBackoffType, + UseJitter = useJitter + }; + + // Map Intervals to custom DelayGenerator. + if (intervals is { Length: > 0 }) + { + strategyOptions.DelayGenerator = args => + { + var index = Math.Min(args.AttemptNumber, intervals.Length - 1); + return new ValueTask(intervals[index]); + }; + } + + // Map On().Ignore() rules to ShouldHandle predicate. + if (exceptionRules.Count > 0) + { + strategyOptions.ShouldHandle = args => + { + if (args.Outcome.Exception is { } ex + && ExceptionRuleMatcher.ShouldIgnore(exceptionRules, ex)) + { + return new ValueTask(false); + } + + return new ValueTask(args.Outcome.Exception is not null); + }; + } + + var pipeline = new ResiliencePipelineBuilder() + .AddRetry(strategyOptions) + .Build(); + + var middleware = new ConsumerRetryMiddleware(pipeline); + + return ctx => middleware.InvokeAsync(ctx, next); + }, + "Retry"); + + private static IReadOnlyList ResolveExceptionRules(ConsumerMiddlewareFactoryContext context) + { + var busFeatures = context.Services.GetRequiredService(); + + // Consumer rules take precedence if present. + if (context.Consumer.Configuration?.Features is { } consumerFeatures + && consumerFeatures.TryGet(out RetryFeature? consumerFeature) + && consumerFeature.ExceptionRules.Count > 0) + { + return consumerFeature.ExceptionRules; + } + + if (busFeatures.TryGet(out RetryFeature? busFeature) + && busFeature.ExceptionRules.Count > 0) + { + return busFeature.ExceptionRules; + } + + return []; + } +} + +file static class Extensions +{ + /// + /// Resolves configuration with the most specific scope taking precedence. + /// + public static T? GetConfiguration( + this ConsumerMiddlewareFactoryContext context, + Func selector) + { + var busFeatures = context.Services.GetRequiredService(); + + // consumer -> bus (most specific first) + if (context.Consumer.Configuration?.Features is { } consumerFeatures) + { + var value = consumerFeatures.GetFeatureValue(selector); + + if (value is not null) + { + return value; + } + } + + return busFeatures.GetFeatureValue(selector); + } + + private static T? GetFeatureValue(this IFeatureCollection features, Func selector) + { + if (features.TryGet(out RetryFeature? feature)) + { + return selector(feature); + } + + return default; + } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs new file mode 100644 index 00000000000..42f03e6f37e --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs @@ -0,0 +1,35 @@ +namespace Mocha; + +/// +/// Fluent builder for configuring per-exception retry/redelivery behavior. +/// +/// The exception type to configure behavior for. +public sealed class ExceptionPolicyBuilder where TException : Exception +{ + private readonly List _rules; + private readonly Func? _predicate; + + internal ExceptionPolicyBuilder(List rules, Func? predicate) + { + _rules = rules; + _predicate = predicate; + } + + /// + /// Excludes this exception type from retry/redelivery. The exception propagates + /// immediately without being retried. + /// + public void Ignore() + { + Func? wrappedPredicate = _predicate is not null + ? ex => ex is TException typed && _predicate(typed) + : null; + + _rules.Add(new ExceptionRule + { + ExceptionType = typeof(TException), + Predicate = wrappedPredicate, + Action = ExceptionAction.Ignore + }); + } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRule.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRule.cs new file mode 100644 index 00000000000..38587bf61b7 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRule.cs @@ -0,0 +1,33 @@ +namespace Mocha; + +/// +/// Internal representation of an exception filtering rule. +/// +internal sealed class ExceptionRule +{ + /// + /// Gets the exception type this rule applies to. + /// + public required Type ExceptionType { get; init; } + + /// + /// Gets the optional predicate that further filters the exception. + /// + public required Func? Predicate { get; init; } + + /// + /// Gets the action to take when this rule matches. + /// + public required ExceptionAction Action { get; init; } +} + +/// +/// Actions that can be taken for a matched exception rule. +/// +internal enum ExceptionAction +{ + /// + /// Don't retry/redeliver this exception. + /// + Ignore +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRuleMatcher.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRuleMatcher.cs new file mode 100644 index 00000000000..955ed57b36e --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRuleMatcher.cs @@ -0,0 +1,52 @@ +namespace Mocha; + +/// +/// Evaluates exception rules to determine if an exception should be ignored. +/// Most-specific-type-wins: if both DbException.Ignore() and NpgsqlException rules exist, +/// NpgsqlException rule takes priority for NpgsqlException instances. +/// +internal static class ExceptionRuleMatcher +{ + /// + /// Determines whether the given exception should be ignored based on the configured rules. + /// + /// The list of exception rules to evaluate. + /// The exception to match against. + /// true if the exception should be ignored; otherwise, false. + public static bool ShouldIgnore(IReadOnlyList rules, Exception exception) + { + ExceptionRule? bestMatch = null; + var bestDepth = int.MaxValue; + + foreach (var rule in rules) + { + if (!rule.ExceptionType.IsInstanceOfType(exception)) + { + continue; + } + + if (rule.Predicate is not null && !rule.Predicate(exception)) + { + continue; + } + + // Calculate inheritance depth: most specific type = smallest depth + var depth = 0; + var type = exception.GetType(); + + while (type != null && type != rule.ExceptionType) + { + depth++; + type = type.BaseType; + } + + if (depth < bestDepth) + { + bestDepth = depth; + bestMatch = rule; + } + } + + return bestMatch?.Action == ExceptionAction.Ignore; + } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryBackoffType.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryBackoffType.cs new file mode 100644 index 00000000000..0fe169a38b5 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryBackoffType.cs @@ -0,0 +1,22 @@ +namespace Mocha; + +/// +/// Specifies the backoff strategy for delay calculations between retry attempts. +/// +public enum RetryBackoffType +{ + /// + /// Constant delay between retries. Every attempt waits the same . + /// + Constant, + + /// + /// Linearly increasing delay. Delay = * (attempt + 1). + /// + Linear, + + /// + /// Exponentially increasing delay. Delay = * 2^attempt. + /// + Exponential +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryConfigurationExtensions.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryConfigurationExtensions.cs new file mode 100644 index 00000000000..26c2e465252 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryConfigurationExtensions.cs @@ -0,0 +1,90 @@ +using Mocha.Features; + +namespace Mocha; + +/// +/// Provides extension methods for configuring the retry middleware on message bus builders and consumer descriptors. +/// +public static class RetryConfigurationExtensions +{ + /// + /// Adds retry to the message bus consumer pipeline with default settings. + /// + /// The message bus builder. + /// The builder for method chaining. + public static IMessageBusBuilder AddRetry( + this IMessageBusBuilder builder) + { + builder.ConfigureFeature(f => f.GetOrSet()); + builder.UseConsume(ConsumerMiddlewares.Retry, before: "Instrumentation"); + return builder; + } + + /// + /// Adds retry to the message bus consumer pipeline. + /// + /// The message bus builder. + /// The action to configure retry options. + /// The builder for method chaining. + public static IMessageBusBuilder AddRetry( + this IMessageBusBuilder builder, + Action configure) + { + builder.ConfigureFeature(f => f.GetOrSet().Configure(configure)); + builder.UseConsume(ConsumerMiddlewares.Retry, before: "Instrumentation"); + return builder; + } + + /// + /// Adds retry to the host-level consumer pipeline with default settings. + /// + /// The host builder. + /// The builder for method chaining. + public static IMessageBusHostBuilder AddRetry( + this IMessageBusHostBuilder builder) + { + builder.ConfigureMessageBus(x => x.AddRetry()); + return builder; + } + + /// + /// Adds retry to the host-level consumer pipeline. + /// + /// The host builder. + /// The action to configure retry options. + /// The builder for method chaining. + public static IMessageBusHostBuilder AddRetry( + this IMessageBusHostBuilder builder, + Action configure) + { + builder.ConfigureMessageBus(x => x.AddRetry(configure)); + return builder; + } + + /// + /// Adds retry configuration to a specific consumer with default settings. + /// + /// The consumer descriptor to configure. + /// The descriptor for method chaining. + public static IConsumerDescriptor AddRetry( + this IConsumerDescriptor descriptor) + { + descriptor.Extend().Configuration.Features.GetOrSet(); + return descriptor; + } + + /// + /// Adds retry configuration to a specific consumer. + /// + /// The consumer descriptor to configure. + /// The action to configure retry options. + /// The descriptor for method chaining. + public static IConsumerDescriptor AddRetry( + this IConsumerDescriptor descriptor, + Action configure) + { + descriptor.Extend().Configuration.Features + .GetOrSet().Configure(configure); + return descriptor; + } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryFeature.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryFeature.cs new file mode 100644 index 00000000000..690d675ab0d --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryFeature.cs @@ -0,0 +1,75 @@ +using Mocha.Features; + +namespace Mocha; + +/// +/// A feature that exposes the retry configuration for a consumer. +/// +public sealed class RetryFeature : ISealable +{ + private readonly RetryOptions _options = new(); + + /// + public bool IsReadOnly { get; private set; } + + /// + /// Gets whether retry is enabled, or null if not configured. + /// + public bool? Enabled => _options.Enabled; + + /// + /// Gets the maximum retry attempts, or null if not configured. + /// + public int? MaxRetryAttempts => _options.MaxRetryAttempts; + + /// + /// Gets the base delay between retries, or null if not configured. + /// + public TimeSpan? Delay => _options.Delay; + + /// + /// Gets the maximum delay cap, or null if not configured. + /// + public TimeSpan? MaxDelay => _options.MaxDelay; + + /// + /// Gets the backoff strategy, or null if not configured. + /// + public RetryBackoffType? BackoffType => _options.BackoffType; + + /// + /// Gets whether jitter is enabled, or null if not configured. + /// + public bool? UseJitter => _options.UseJitter; + + /// + /// Gets the explicit retry intervals, or null if not configured. + /// + public TimeSpan[]? Intervals => _options.Intervals; + + /// + /// Gets the exception rules configured for this feature. + /// + internal IReadOnlyList ExceptionRules => _options.ExceptionRules; + + /// + public void Seal() + { + IsReadOnly = true; + } + + /// + /// Applies configuration to the retry options. + /// + /// An action that modifies the retry options. + /// Thrown if the feature has been sealed. + public void Configure(Action configure) + { + if (IsReadOnly) + { + throw ThrowHelper.FeatureIsReadOnly(); + } + + configure(_options); + } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryOptions.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryOptions.cs new file mode 100644 index 00000000000..9dc179c58c1 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryOptions.cs @@ -0,0 +1,102 @@ +namespace Mocha; + +/// +/// Options for configuring the retry middleware that retries failed message handler invocations +/// with configurable backoff strategies. +/// +public class RetryOptions +{ + /// + /// Gets or sets whether retry is enabled. Null inherits from parent scope; defaults to true. + /// + public bool? Enabled { get; set; } + + /// + /// Gets or sets the maximum retry attempts (not counting the initial attempt). + /// + public int? MaxRetryAttempts { get; set; } + + /// + /// Gets or sets the base delay between retries. Interpretation depends on . + /// + public TimeSpan? Delay { get; set; } + + /// + /// Gets or sets the maximum delay cap. Prevents exponential backoff from growing unbounded. + /// + public TimeSpan? MaxDelay { get; set; } + + /// + /// Gets or sets the backoff strategy: Constant, Linear, or Exponential. + /// + public RetryBackoffType? BackoffType { get; set; } + + /// + /// Gets or sets whether to add jitter to delay calculations. + /// + public bool? UseJitter { get; set; } + + /// + /// Gets or sets explicit retry intervals. When set, overrides , + /// , and . + /// The number of elements determines the number of retries. + /// + public TimeSpan[]? Intervals { get; set; } + + private List? _exceptionRules; + + internal IReadOnlyList ExceptionRules => _exceptionRules ?? (IReadOnlyList)[]; + + /// + /// Configures behavior for a specific exception type. + /// + /// The exception type to configure. + /// A builder for configuring the exception behavior. + public ExceptionPolicyBuilder On() where TException : Exception + => On(null); + + /// + /// Configures behavior for a specific exception type matching a predicate. + /// + /// The exception type to configure. + /// An optional predicate to further filter the exception. + /// A builder for configuring the exception behavior. + public ExceptionPolicyBuilder On(Func? predicate) + where TException : Exception + { + _exceptionRules ??= []; + var builder = new ExceptionPolicyBuilder(_exceptionRules, predicate); + return builder; + } + + /// + /// Provides the default values for retry options. + /// + public static class Defaults + { + /// + /// The default maximum retry attempts. + /// + public static int MaxRetryAttempts = 3; + + /// + /// The default base delay between retries. + /// + public static TimeSpan Delay = TimeSpan.FromMilliseconds(200); + + /// + /// The default maximum delay cap. + /// + public static TimeSpan MaxDelay = TimeSpan.FromSeconds(30); + + /// + /// The default backoff strategy. + /// + public static RetryBackoffType BackoffType = RetryBackoffType.Exponential; + + /// + /// The default jitter setting. + /// + public static bool UseJitter = true; + } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryState.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryState.cs new file mode 100644 index 00000000000..dad6f16da19 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryState.cs @@ -0,0 +1,21 @@ +namespace Mocha; + +/// +/// Provides retry state information to message handlers via +/// context.Features.Get<RetryState>(). +/// Null if AddRetry is not configured. +/// +public sealed class RetryState +{ + /// + /// Number of immediate retries already attempted for this delivery round. + /// 0 on the first (original) attempt. + /// + public int ImmediateRetryCount { get; internal set; } + + /// + /// Number of delayed redeliveries already attempted. + /// Read from the Mocha-Retry-DelayedRetryCount header. + /// + public int DelayedRetryCount { get; internal set; } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Extensions/MiddlewareConfigurationExtensions.cs b/src/Mocha/src/Mocha/Middlewares/Extensions/MiddlewareConfigurationExtensions.cs index d9f0f1d3bff..16f27b0b22a 100644 --- a/src/Mocha/src/Mocha/Middlewares/Extensions/MiddlewareConfigurationExtensions.cs +++ b/src/Mocha/src/Mocha/Middlewares/Extensions/MiddlewareConfigurationExtensions.cs @@ -220,6 +220,12 @@ public void Prepend(ConsumerMiddlewareConfiguration configuration, string? befor configurations.Add(pipeline => { + // Skip if a middleware with the same key is already registered. + if (configuration.Key is not null && pipeline.Exists(m => m.Key == configuration.Key)) + { + return; + } + var index = pipeline.FindIndex(m => m.Key == before); if (index == -1) { diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/ReceiveMiddlewares.cs b/src/Mocha/src/Mocha/Middlewares/Receive/ReceiveMiddlewares.cs index 629e336a64f..68f256fc943 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/ReceiveMiddlewares.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/ReceiveMiddlewares.cs @@ -49,6 +49,11 @@ public static class ReceiveMiddlewares public static readonly ReceiveMiddlewareConfiguration MessageTypeSelection = MessageTypeSelectionMiddleware.Create(); + /// + /// The redelivery middleware configuration that reschedules failed messages for later delivery. + /// + public static readonly ReceiveMiddlewareConfiguration Redelivery = ReceiveRedeliveryMiddleware.Create(); + /// /// The routing middleware configuration that dispatches messages to the appropriate consumer. /// diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs new file mode 100644 index 00000000000..bc26d6f41f4 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs @@ -0,0 +1,247 @@ +using Microsoft.Extensions.DependencyInjection; +using Mocha.Features; +using Mocha.Middlewares; + +namespace Mocha; + +/// +/// A receive middleware that reschedules failed messages for later delivery, releasing the +/// concurrency slot while the message waits for its next attempt. +/// +/// +/// This middleware implements Tier 2 (delayed redelivery) of the retry model. On failure it +/// increments the delayed-retry-count header and dispatches the original envelope back +/// to the same endpoint with a scheduled delivery time. Request/reply messages are excluded +/// because the caller would time out waiting for a response. +/// +internal sealed class ReceiveRedeliveryMiddleware( + int maxAttempts, + TimeSpan[]? intervals, + TimeSpan resolvedBaseDelay, + TimeSpan resolvedMaxDelay, + bool resolvedUseJitter, + IReadOnlyList exceptionRules, + TimeProvider timeProvider, + IMessagingPools pools) +{ + public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next) + { + // Read the current delayed retry count from headers. + var delayedRetryCount = 0; + + if (context.Headers.TryGetValue(MessageHeaders.Retry.DelayedRetryCount.Key, out var headerValue)) + { + delayedRetryCount = headerValue switch + { + int i => i, + long l => (int)l, + _ => 0 + }; + } + + try + { + await next(context); + } + catch (Exception ex) + { + // Request/reply messages must not be redelivered -- the caller is waiting. + if (context.Envelope?.ResponseAddress is not null) + { + throw; + } + + // Check exception rules: if the exception is explicitly ignored, rethrow. + if (exceptionRules.Count > 0 && ExceptionRuleMatcher.ShouldIgnore(exceptionRules, ex)) + { + throw; + } + + // Check if redelivery attempts remain. + if (delayedRetryCount >= maxAttempts) + { + throw; + } + + // Calculate the delay for this redelivery attempt. + var delay = CalculateDelay(delayedRetryCount); + var scheduledTime = timeProvider.GetUtcNow().Add(delay); + + // Update the header on the envelope so the next delivery round sees the incremented count. + var envelope = context.Envelope; + + if (envelope is null) + { + throw; + } + + envelope.Headers?.Set(MessageHeaders.Retry.DelayedRetryCount.Key, delayedRetryCount + 1); + + // Dispatch the envelope back to the same endpoint with the scheduled time. + // Use the Source address (queue/topic) rather than the endpoint address, because + // transports resolve dispatch endpoints from the topology resource address. + var dispatchEndpoint = context.Runtime.GetDispatchEndpoint(context.Endpoint.Source.Address); + var dispatchContext = pools.DispatchContext.Get(); + + try + { + dispatchContext.Initialize( + context.Services, + dispatchEndpoint, + context.Runtime, + context.MessageType, + context.CancellationToken); + + dispatchContext.Envelope = envelope; + dispatchContext.ScheduledTime = scheduledTime; + + await dispatchEndpoint.ExecuteAsync(dispatchContext); + } + finally + { + pools.DispatchContext.Return(dispatchContext); + } + + // Mark the message as consumed so Fault/DeadLetter don't also handle it. + context.Features.GetOrSet().MessageConsumed = true; + } + } + + private TimeSpan CalculateDelay(int attempt) + { + TimeSpan baseDelay; + + if (intervals is { Length: > 0 }) + { + // Explicit intervals: use array index, clamp to last. + baseDelay = intervals[Math.Min(attempt, intervals.Length - 1)]; + } + else + { + // Calculated: BaseDelay * (attempt + 1). + baseDelay = resolvedBaseDelay * (attempt + 1); + } + + // Cap by MaxDelay. + if (baseDelay > resolvedMaxDelay) + { + baseDelay = resolvedMaxDelay; + } + + // Add jitter: +/- 25%. + if (resolvedUseJitter) + { + var jitterRange = baseDelay.TotalMilliseconds * 0.25; + var jitter = (Random.Shared.NextDouble() * 2 - 1) * jitterRange; + baseDelay = TimeSpan.FromMilliseconds(Math.Max(0, baseDelay.TotalMilliseconds + jitter)); + } + + return baseDelay; + } + + public static ReceiveMiddlewareConfiguration Create() + => new( + static (context, next) => + { + // Feature values are resolved from endpoint -> transport -> bus to support overrides. + var enabled = context.GetConfiguration(f => f.Enabled) ?? true; + + if (!enabled) + { + return next; + } + + var intervals = context.GetConfiguration(f => f.Intervals) + ?? RedeliveryOptions.Defaults.Intervals; + + var maxAttempts = intervals is { Length: > 0 } + ? intervals.Length + : context.GetConfiguration(f => f.MaxAttempts) ?? 0; + + if (maxAttempts <= 0 && intervals is not { Length: > 0 }) + { + return next; + } + + var baseDelay = context.GetConfiguration(f => f.BaseDelay) + ?? TimeSpan.FromMinutes(5); + + var maxDelay = context.GetConfiguration(f => f.MaxDelay) + ?? RedeliveryOptions.Defaults.MaxDelay; + + var useJitter = context.GetConfiguration(f => f.UseJitter) + ?? RedeliveryOptions.Defaults.UseJitter; + + // Resolve exception rules (atomically from first scope that has them). + var exceptionRules = ResolveExceptionRules(context); + + var timeProvider = context.Services.GetRequiredService(); + var pools = context.Services.GetRequiredService(); + + var middleware = new ReceiveRedeliveryMiddleware( + maxAttempts, + intervals, + baseDelay, + maxDelay, + useJitter, + exceptionRules, + timeProvider, + pools); + + return ctx => middleware.InvokeAsync(ctx, next); + }, + "Redelivery"); + + private static IReadOnlyList ResolveExceptionRules(ReceiveMiddlewareFactoryContext context) + { + var busFeatures = context.Services.GetRequiredService(); + + // Endpoint rules take precedence if present. + if (context.Endpoint.Features.TryGet(out RedeliveryFeature? endpointFeature) + && endpointFeature.ExceptionRules.Count > 0) + { + return endpointFeature.ExceptionRules; + } + + if (context.Transport.Features.TryGet(out RedeliveryFeature? transportFeature) + && transportFeature.ExceptionRules.Count > 0) + { + return transportFeature.ExceptionRules; + } + + if (busFeatures.TryGet(out RedeliveryFeature? busFeature) + && busFeature.ExceptionRules.Count > 0) + { + return busFeature.ExceptionRules; + } + + return []; + } +} + +file static class Extensions +{ + /// + /// Resolves configuration with the most specific scope taking precedence. + /// + public static T? GetConfiguration( + this ReceiveMiddlewareFactoryContext context, + Func selector) + { + var busFeatures = context.Services.GetRequiredService(); + + return context.Endpoint.Features.GetFeatureValue(selector) + ?? context.Transport.Features.GetFeatureValue(selector) + ?? busFeatures.GetFeatureValue(selector); + } + + private static T? GetFeatureValue(this IFeatureCollection features, Func selector) + { + if (features.TryGet(out RedeliveryFeature? feature)) + { + return selector(feature); + } + + return default; + } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryConfigurationExtensions.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryConfigurationExtensions.cs new file mode 100644 index 00000000000..897e914260a --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryConfigurationExtensions.cs @@ -0,0 +1,93 @@ +using Mocha.Features; + +namespace Mocha; + +/// +/// Provides extension methods for configuring the redelivery middleware on message bus builders and descriptors. +/// +public static class RedeliveryConfigurationExtensions +{ + /// + /// Adds redelivery to the message bus receive pipeline with default settings. + /// + /// The message bus builder. + /// The builder for method chaining. + public static IMessageBusBuilder AddRedelivery( + this IMessageBusBuilder builder) + { + builder.ConfigureFeature(f => f.GetOrSet()); + builder.UseReceive(ReceiveMiddlewares.Redelivery, after: "Fault"); + return builder; + } + + /// + /// Adds redelivery to the message bus receive pipeline. + /// + /// The message bus builder. + /// The action to configure redelivery options. + /// The builder for method chaining. + public static IMessageBusBuilder AddRedelivery( + this IMessageBusBuilder builder, + Action configure) + { + builder.ConfigureFeature(f => f.GetOrSet().Configure(configure)); + builder.UseReceive(ReceiveMiddlewares.Redelivery, after: "Fault"); + return builder; + } + + /// + /// Adds redelivery to the host-level receive pipeline with default settings. + /// + /// The host builder. + /// The builder for method chaining. + public static IMessageBusHostBuilder AddRedelivery( + this IMessageBusHostBuilder builder) + { + builder.ConfigureMessageBus(x => x.AddRedelivery()); + return builder; + } + + /// + /// Adds redelivery to the host-level receive pipeline. + /// + /// The host builder. + /// The action to configure redelivery options. + /// The builder for method chaining. + public static IMessageBusHostBuilder AddRedelivery( + this IMessageBusHostBuilder builder, + Action configure) + { + builder.ConfigureMessageBus(x => x.AddRedelivery(configure)); + return builder; + } + + /// + /// Adds redelivery configuration to a specific descriptor (e.g., receive endpoint or transport) with default settings. + /// + /// The descriptor type that supports receive middleware. + /// The descriptor to configure. + /// The descriptor for method chaining. + public static TDescriptor AddRedelivery( + this TDescriptor descriptor) + where TDescriptor : IReceiveMiddlewareProvider + { + descriptor.Extend().Configuration.Features.GetOrSet(); + return descriptor; + } + + /// + /// Adds redelivery configuration to a specific descriptor (e.g., receive endpoint or transport). + /// + /// The descriptor type that supports receive middleware. + /// The descriptor to configure. + /// The action to configure redelivery options. + /// The descriptor for method chaining. + public static TDescriptor AddRedelivery( + this TDescriptor descriptor, + Action configure) + where TDescriptor : IReceiveMiddlewareProvider + { + descriptor.Extend().Configuration.Features.GetOrSet().Configure(configure); + return descriptor; + } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryFeature.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryFeature.cs new file mode 100644 index 00000000000..96129f88db8 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryFeature.cs @@ -0,0 +1,70 @@ +using Mocha.Features; + +namespace Mocha; + +/// +/// A feature that exposes the redelivery configuration for a receive endpoint. +/// +public sealed class RedeliveryFeature : ISealable +{ + private readonly RedeliveryOptions _options = new(); + + /// + public bool IsReadOnly { get; private set; } + + /// + /// Gets whether redelivery is enabled, or null if not configured. + /// + public bool? Enabled => _options.Enabled; + + /// + /// Gets the maximum redelivery attempts, or null if not configured. + /// + public int? MaxAttempts => _options.MaxAttempts; + + /// + /// Gets the base delay for backoff calculation, or null if not configured. + /// + public TimeSpan? BaseDelay => _options.BaseDelay; + + /// + /// Gets the maximum delay cap, or null if not configured. + /// + public TimeSpan? MaxDelay => _options.MaxDelay; + + /// + /// Gets whether jitter is enabled, or null if not configured. + /// + public bool? UseJitter => _options.UseJitter; + + /// + /// Gets the explicit redelivery intervals, or null if not configured. + /// + public TimeSpan[]? Intervals => _options.Intervals; + + /// + /// Gets the exception rules configured for this feature. + /// + internal IReadOnlyList ExceptionRules => _options.ExceptionRules; + + /// + public void Seal() + { + IsReadOnly = true; + } + + /// + /// Applies configuration to the redelivery options. + /// + /// An action that modifies the redelivery options. + /// Thrown if the feature has been sealed. + public void Configure(Action configure) + { + if (IsReadOnly) + { + throw ThrowHelper.FeatureIsReadOnly(); + } + + configure(_options); + } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryOptions.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryOptions.cs new file mode 100644 index 00000000000..f6a70102a88 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryOptions.cs @@ -0,0 +1,92 @@ +namespace Mocha; + +/// +/// Options for configuring the redelivery middleware that reschedules failed messages +/// for later delivery with configurable delay strategies. +/// +public class RedeliveryOptions +{ + /// + /// Gets or sets whether redelivery is enabled. Null inherits from parent scope; defaults to true. + /// + public bool? Enabled { get; set; } + + /// + /// Gets or sets the maximum number of redelivery attempts. + /// + public int? MaxAttempts { get; set; } + + /// + /// Gets or sets the base delay for backoff calculation. Actual delay = BaseDelay * (attempt + 1). + /// + public TimeSpan? BaseDelay { get; set; } + + /// + /// Gets or sets the maximum delay cap for any single redelivery delay. + /// + public TimeSpan? MaxDelay { get; set; } + + /// + /// Gets or sets whether to add jitter to delay calculations. + /// + public bool? UseJitter { get; set; } + + /// + /// Gets or sets explicit redelivery intervals. When set, overrides + /// and . The number of elements determines the number of + /// redelivery attempts. + /// + public TimeSpan[]? Intervals { get; set; } + + private List? _exceptionRules; + + internal IReadOnlyList ExceptionRules => _exceptionRules ?? (IReadOnlyList)[]; + + /// + /// Configures behavior for a specific exception type. + /// + /// The exception type to configure. + /// A builder for configuring the exception behavior. + public ExceptionPolicyBuilder On() where TException : Exception + => On(null); + + /// + /// Configures behavior for a specific exception type matching a predicate. + /// + /// The exception type to configure. + /// An optional predicate to further filter the exception. + /// A builder for configuring the exception behavior. + public ExceptionPolicyBuilder On(Func? predicate) + where TException : Exception + { + _exceptionRules ??= []; + var builder = new ExceptionPolicyBuilder(_exceptionRules, predicate); + return builder; + } + + /// + /// Provides the default values for redelivery options. + /// + public static class Defaults + { + /// + /// The default redelivery intervals: 5 minutes, 15 minutes, 30 minutes. + /// + public static TimeSpan[] Intervals = + [ + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(15), + TimeSpan.FromMinutes(30) + ]; + + /// + /// The default maximum delay cap. + /// + public static TimeSpan MaxDelay = TimeSpan.FromHours(1); + + /// + /// The default jitter setting. + /// + public static bool UseJitter = true; + } +} diff --git a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs new file mode 100644 index 00000000000..92cc7600382 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs @@ -0,0 +1,274 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.InMemory.Tests.Helpers; + +namespace Mocha.Transport.InMemory.Tests.Behaviors; + +public sealed class RedeliveryTests +{ + private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(10); + + [Fact] + public async Task Redelivery_Should_ScheduleRedelivery_When_HandlerFails() + { + // arrange + var counter = new InvocationCounter(); + var recorder = new MessageRecorder(); + + await using var provider = await new ServiceCollection() + .AddSingleton(counter) + .AddSingleton(recorder) + .AddMessageBus() + .AddRedelivery(redeliver => + { + redeliver.Intervals = + [ + TimeSpan.FromMilliseconds(1), + TimeSpan.FromMilliseconds(1), + TimeSpan.FromMilliseconds(1) + ]; + }) + .AddEventHandler() + .AddInMemory() + .BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-REDELIVER" }, CancellationToken.None); + + // assert - handler fails on first delivery, succeeds on redelivery + Assert.True( + await recorder.WaitAsync(s_timeout), + "Handler did not record the message after redelivery"); + + Assert.Equal(2, counter.Count); + } + + [Fact] + public async Task Redelivery_Should_SkipRedelivery_When_ExceptionIsIgnored() + { + // arrange + var counter = new InvocationCounter(); + + await using var provider = await new ServiceCollection() + .AddSingleton(counter) + .AddMessageBus() + .AddRedelivery(redeliver => + { + redeliver.Intervals = [TimeSpan.FromMilliseconds(1)]; + redeliver.On().Ignore(); + }) + .AddEventHandler() + .AddInMemory() + .BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-IGNORED" }, CancellationToken.None); + + // assert - only 1 invocation, exception propagates without redelivery + await Task.Delay(500); + Assert.Equal(1, counter.Count); + } + + [Fact] + public async Task Redelivery_Should_PassThrough_When_Disabled() + { + // arrange + var counter = new InvocationCounter(); + + await using var provider = await new ServiceCollection() + .AddSingleton(counter) + .AddMessageBus() + .AddRedelivery(redeliver => + { + redeliver.Enabled = false; + redeliver.Intervals = [TimeSpan.FromMilliseconds(1)]; + }) + .AddEventHandler() + .AddInMemory() + .BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-DISABLED" }, CancellationToken.None); + + // assert - only 1 invocation, redelivery disabled + await Task.Delay(500); + Assert.Equal(1, counter.Count); + } + + [Fact] + public async Task Redelivery_Should_PropagateToFault_When_AllAttemptsExhausted() + { + // arrange - 2 redelivery intervals = 2 redeliveries max, 3 total attempts + var counter = new InvocationCounter(); + + await using var provider = await new ServiceCollection() + .AddSingleton(counter) + .AddMessageBus() + .AddRedelivery(redeliver => + { + redeliver.Intervals = + [ + TimeSpan.FromMilliseconds(1), + TimeSpan.FromMilliseconds(1) + ]; + }) + .AddEventHandler() + .AddInMemory() + .BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-EXHAUST" }, CancellationToken.None); + + // assert - 1 original + 2 redeliveries = 3 total invocations + Assert.True( + await counter.WaitForCountAsync(3, s_timeout), + $"Expected 3 invocations (1 original + 2 redeliveries), but got {counter.Count}"); + } + + [Fact] + public async Task Redelivery_Should_UseEndpointOverride_When_EndpointConfigured() + { + // arrange - bus-level: 1 redelivery, but the endpoint overrides to disabled + var counter = new InvocationCounter(); + var builder = new ServiceCollection() + .AddSingleton(counter) + .AddScoped() + .AddMessageBus() + .AddRedelivery(redeliver => redeliver.Intervals = [TimeSpan.FromMilliseconds(1)]); + + // Override at transport level to disable redelivery. + builder.ConfigureMessageBus(b => b.AddHandler()); + + await using var provider = await builder + .AddInMemory(t => t.AddRedelivery(redeliver => redeliver.Enabled = false)) + .BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-OVERRIDE" }, CancellationToken.None); + + // assert - redelivery disabled at transport level: only 1 invocation + await Task.Delay(500); + Assert.Equal(1, counter.Count); + } + + [Fact] + public async Task Redelivery_Should_UseDefaults_When_ParameterlessAddRedelivery() + { + // arrange - parameterless AddRedelivery uses defaults: 3 intervals + var counter = new InvocationCounter(); + + await using var provider = await new ServiceCollection() + .AddSingleton(counter) + .AddMessageBus() + .AddRedelivery() + .AddEventHandler() + .AddInMemory() + .BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-DEFAULT" }, CancellationToken.None); + + // assert - default is 3 intervals: 1 original + 3 redeliveries = 4 total + Assert.True( + await counter.WaitForCountAsync(4, s_timeout), + $"Expected 4 invocations (1 original + 3 default redeliveries), but got {counter.Count}"); + } + + // ============================================================ + // Test Helpers + // ============================================================ + + private sealed class InvocationCounter + { + private int _count; + private readonly SemaphoreSlim _semaphore = new(0); + + public int Count => _count; + + public void Increment() + { + Interlocked.Increment(ref _count); + _semaphore.Release(); + } + + public async Task WaitForCountAsync(int targetCount, TimeSpan timeout) + { + for (var i = 0; i < targetCount; i++) + { + if (!await _semaphore.WaitAsync(timeout)) + { + return false; + } + } + + return true; + } + } + + // ============================================================ + // Test Handlers + // ============================================================ + + /// + /// Throws on the first invocation, succeeds on subsequent invocations. + /// + private sealed class ThrowThenSucceedHandler(InvocationCounter counter, MessageRecorder recorder) + : IEventHandler + { + public ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + var invocation = counter.Count; + counter.Increment(); + + if (invocation == 0) + { + throw new InvalidOperationException("Transient failure"); + } + + recorder.Record(message); + return default; + } + } + + /// + /// Always throws an InvalidOperationException. + /// + private sealed class AlwaysThrowingHandler(InvocationCounter counter) : IEventHandler + { + public ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + counter.Increment(); + throw new InvalidOperationException("Always fails"); + } + } + + /// + /// Always throws an InvalidOperationException (for the Ignore test). + /// + private sealed class ThrowInvalidOperationHandler(InvocationCounter counter) : IEventHandler + { + public ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + counter.Increment(); + throw new InvalidOperationException("Should be ignored"); + } + } +} diff --git a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs new file mode 100644 index 00000000000..c16aa1550e1 --- /dev/null +++ b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs @@ -0,0 +1,516 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Mocha.Transport.InMemory.Tests.Helpers; + +namespace Mocha.Transport.InMemory.Tests.Behaviors; + +public sealed class RetryTests +{ + private static readonly TimeSpan s_timeout = TimeSpan.FromSeconds(10); + + [Fact] + public async Task Retry_Should_RetryHandler_When_HandlerThrowsTransientException() + { + // arrange + var counter = new RetryInvocationCounter(); + var recorder = new MessageRecorder(); + await using var provider = await new ServiceCollection() + .AddSingleton(counter) + .AddSingleton(recorder) + .AddMessageBus() + .AddRetry(retry => + { + retry.MaxRetryAttempts = 3; + retry.Delay = TimeSpan.FromMilliseconds(1); + retry.BackoffType = RetryBackoffType.Constant; + retry.UseJitter = false; + }) + .AddEventHandler() + .AddInMemory() + .BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-1" }, CancellationToken.None); + + // assert - handler succeeds on 2nd attempt, so the message is recorded + Assert.True( + await recorder.WaitAsync(s_timeout), + "Handler did not record the message after retry"); + + Assert.Equal(2, counter.Count); + } + + [Fact] + public async Task Retry_Should_PropagateToFault_When_AllRetriesExhausted() + { + // arrange + var counter = new RetryInvocationCounter(); + await using var provider = await new ServiceCollection() + .AddSingleton(counter) + .AddMessageBus() + .AddRetry(retry => + { + retry.MaxRetryAttempts = 3; + retry.Delay = TimeSpan.FromMilliseconds(1); + retry.BackoffType = RetryBackoffType.Constant; + retry.UseJitter = false; + }) + .AddEventHandler() + .AddInMemory() + .BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-FAIL" }, CancellationToken.None); + + // assert - 1 original + 3 retries = 4 total invocations + Assert.True( + await counter.WaitForCountAsync(4, s_timeout), + $"Expected 4 invocations (1 original + 3 retries), but got {counter.Count}"); + } + + [Fact] + public async Task Retry_Should_SkipRetry_When_ExceptionIsIgnored() + { + // arrange + var counter = new RetryInvocationCounter(); + await using var provider = await new ServiceCollection() + .AddSingleton(counter) + .AddMessageBus() + .AddRetry(retry => + { + retry.MaxRetryAttempts = 3; + retry.Delay = TimeSpan.FromMilliseconds(1); + retry.BackoffType = RetryBackoffType.Constant; + retry.UseJitter = false; + retry.On().Ignore(); + }) + .AddEventHandler() + .AddInMemory() + .BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-IGNORED" }, CancellationToken.None); + + // assert - only 1 invocation, exception propagates without retry + await Task.Delay(500); + Assert.Equal(1, counter.Count); + } + + [Fact] + public async Task Retry_Should_SkipRetry_When_PredicateMatchesIgnoredException() + { + // arrange + var matchingCounter = new RetryInvocationCounter(); + var nonMatchingCounter = new RetryInvocationCounter(); + + // Test 1: matching predicate (ParamName == "test") - should NOT retry + await using var matchingProvider = await new ServiceCollection() + .AddSingleton(matchingCounter) + .AddMessageBus() + .AddRetry(retry => + { + retry.MaxRetryAttempts = 3; + retry.Delay = TimeSpan.FromMilliseconds(1); + retry.BackoffType = RetryBackoffType.Constant; + retry.UseJitter = false; + retry.On(ex => ex.ParamName == "test").Ignore(); + }) + .AddEventHandler() + .AddInMemory() + .BuildServiceProvider(); + + using var matchingScope = matchingProvider.CreateScope(); + var matchingBus = matchingScope.ServiceProvider.GetRequiredService(); + + await matchingBus.PublishAsync(new OrderCreated { OrderId = "ORD-MATCH" }, CancellationToken.None); + + // assert - matching predicate: no retry, only 1 invocation + await Task.Delay(500); + Assert.Equal(1, matchingCounter.Count); + + // Test 2: non-matching predicate (ParamName == "other") - SHOULD retry + await using var nonMatchingProvider = await new ServiceCollection() + .AddSingleton(nonMatchingCounter) + .AddMessageBus() + .AddRetry(retry => + { + retry.MaxRetryAttempts = 3; + retry.Delay = TimeSpan.FromMilliseconds(1); + retry.BackoffType = RetryBackoffType.Constant; + retry.UseJitter = false; + retry.On(ex => ex.ParamName == "other").Ignore(); + }) + .AddEventHandler() + .AddInMemory() + .BuildServiceProvider(); + + using var nonMatchingScope = nonMatchingProvider.CreateScope(); + var nonMatchingBus = nonMatchingScope.ServiceProvider.GetRequiredService(); + + await nonMatchingBus.PublishAsync(new OrderCreated { OrderId = "ORD-NOMATCH" }, CancellationToken.None); + + // assert - non-matching predicate: should retry, 4 total invocations + Assert.True( + await nonMatchingCounter.WaitForCountAsync(4, s_timeout), + $"Expected 4 invocations for non-matching predicate, but got {nonMatchingCounter.Count}"); + } + + [Fact] + public async Task Retry_Should_UseConsumerOverride_When_ConsumerHasDifferentConfig() + { + // arrange - bus-level: 2 retries, consumer-level: 5 retries + var counter = new RetryInvocationCounter(); + var builder = new ServiceCollection() + .AddSingleton(counter) + .AddScoped() + .AddMessageBus() + .AddRetry(retry => + { + retry.MaxRetryAttempts = 2; + retry.Delay = TimeSpan.FromMilliseconds(1); + retry.BackoffType = RetryBackoffType.Constant; + retry.UseJitter = false; + }); + + builder.ConfigureMessageBus(b => + { + b.AddHandler(consumer => + { + consumer.AddRetry(retry => + { + retry.MaxRetryAttempts = 5; + retry.Delay = TimeSpan.FromMilliseconds(1); + retry.BackoffType = RetryBackoffType.Constant; + retry.UseJitter = false; + }); + }); + }); + + await using var provider = await builder.AddInMemory().BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-OVERRIDE" }, CancellationToken.None); + + // assert - consumer override: 1 original + 5 retries = 6 total invocations + Assert.True( + await counter.WaitForCountAsync(6, s_timeout), + $"Expected 6 invocations (1 original + 5 consumer-level retries), but got {counter.Count}"); + } + + [Fact] + public async Task Retry_Should_ExposeRetryState_When_HandlerAccessesFeatures() + { + // arrange + var stateCapture = new RetryStateCapture(); + var builder = new ServiceCollection() + .AddSingleton(stateCapture) + .AddScoped() + .AddMessageBus() + .AddRetry(retry => + { + retry.MaxRetryAttempts = 2; + retry.Delay = TimeSpan.FromMilliseconds(1); + retry.BackoffType = RetryBackoffType.Constant; + retry.UseJitter = false; + }); + + builder.ConfigureMessageBus(b => b.AddHandler()); + + await using var provider = await builder.AddInMemory().BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-STATE" }, CancellationToken.None); + + // assert - 3 invocations (1 original + 2 retries), all fail + Assert.True( + await stateCapture.WaitForCountAsync(3, s_timeout), + $"Expected 3 invocations, but got {stateCapture.CapturedStates.Count}"); + + var states = stateCapture.CapturedStates.OrderBy(s => s).ToList(); + Assert.Equal(0, states[0]); // first attempt + Assert.Equal(1, states[1]); // first retry + Assert.Equal(2, states[2]); // second retry + } + + [Fact] + public async Task Retry_Should_PassThrough_When_DisabledForConsumer() + { + // arrange + var counter = new RetryInvocationCounter(); + var builder = new ServiceCollection() + .AddSingleton(counter) + .AddScoped() + .AddMessageBus() + .AddRetry(retry => + { + retry.MaxRetryAttempts = 3; + retry.Delay = TimeSpan.FromMilliseconds(1); + retry.BackoffType = RetryBackoffType.Constant; + retry.UseJitter = false; + }); + + builder.ConfigureMessageBus(b => + b.AddHandler(consumer => + consumer.AddRetry(retry => retry.Enabled = false))); + + await using var provider = await builder.AddInMemory().BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-DISABLED" }, CancellationToken.None); + + // assert - retry disabled: only 1 invocation + await Task.Delay(500); + Assert.Equal(1, counter.Count); + } + + [Fact] + public async Task Retry_Should_UseExplicitIntervals_When_IntervalsConfigured() + { + // arrange + var counter = new RetryInvocationCounter(); + await using var provider = await new ServiceCollection() + .AddSingleton(counter) + .AddMessageBus() + .AddRetry(retry => + { + retry.Intervals = + [ + TimeSpan.FromMilliseconds(10), + TimeSpan.FromMilliseconds(20), + TimeSpan.FromMilliseconds(30) + ]; + }) + .AddEventHandler() + .AddInMemory() + .BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-INTERVALS" }, CancellationToken.None); + + // assert - Intervals.Length = 3 retries, so 4 total invocations + Assert.True( + await counter.WaitForCountAsync(4, s_timeout), + $"Expected 4 invocations (1 original + 3 interval-based retries), but got {counter.Count}"); + } + + [Fact] + public async Task Retry_Should_RespectInheritance_When_BaseExceptionIgnored() + { + // arrange - ignore ArgumentException, handler throws ArgumentNullException (subclass) + var counter = new RetryInvocationCounter(); + await using var provider = await new ServiceCollection() + .AddSingleton(counter) + .AddMessageBus() + .AddRetry(retry => + { + retry.MaxRetryAttempts = 3; + retry.Delay = TimeSpan.FromMilliseconds(1); + retry.BackoffType = RetryBackoffType.Constant; + retry.UseJitter = false; + retry.On().Ignore(); + }) + .AddEventHandler() + .AddInMemory() + .BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-INHERIT" }, CancellationToken.None); + + // assert - ArgumentNullException is a subclass of ArgumentException, so it's ignored: only 1 invocation + await Task.Delay(500); + Assert.Equal(1, counter.Count); + } + + [Fact] + public async Task Retry_Should_UseDefaults_When_ParameterlessAddRetry() + { + // arrange - default: MaxRetryAttempts = 3 + var counter = new RetryInvocationCounter(); + await using var provider = await new ServiceCollection() + .AddSingleton(counter) + .AddMessageBus() + .AddRetry() + .AddEventHandler() + .AddInMemory() + .BuildServiceProvider(); + + using var scope = provider.CreateScope(); + var bus = scope.ServiceProvider.GetRequiredService(); + + // act + await bus.PublishAsync(new OrderCreated { OrderId = "ORD-DEFAULT" }, CancellationToken.None); + + // assert - default is 3 retries: 1 original + 3 retries = 4 total + Assert.True( + await counter.WaitForCountAsync(4, s_timeout), + $"Expected 4 invocations (1 original + 3 default retries), but got {counter.Count}"); + } + + // ============================================================ + // Test Helpers + // ============================================================ + + private sealed class RetryInvocationCounter + { + private int _count; + private readonly SemaphoreSlim _semaphore = new(0); + + public int Count => _count; + + public void Increment() + { + Interlocked.Increment(ref _count); + _semaphore.Release(); + } + + public async Task WaitForCountAsync(int targetCount, TimeSpan timeout) + { + for (var i = 0; i < targetCount; i++) + { + if (!await _semaphore.WaitAsync(timeout)) + { + return false; + } + } + + return true; + } + } + + private sealed class RetryStateCapture + { + private readonly SemaphoreSlim _semaphore = new(0); + + public ConcurrentBag CapturedStates { get; } = []; + + public void Record(int immediateRetryCount) + { + CapturedStates.Add(immediateRetryCount); + _semaphore.Release(); + } + + public async Task WaitForCountAsync(int targetCount, TimeSpan timeout) + { + for (var i = 0; i < targetCount; i++) + { + if (!await _semaphore.WaitAsync(timeout)) + { + return false; + } + } + + return true; + } + } + + // ============================================================ + // Test Handlers + // ============================================================ + + /// + /// Throws on the first invocation, succeeds on subsequent invocations. + /// + private sealed class ThrowOnceHandler(RetryInvocationCounter counter, MessageRecorder recorder) + : IEventHandler + { + public ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + var invocation = counter.Count; + counter.Increment(); + + if (invocation == 0) + { + throw new InvalidOperationException("Transient failure"); + } + + recorder.Record(message); + return default; + } + } + + /// + /// Always throws an InvalidOperationException. + /// + private sealed class AlwaysThrowingHandler(RetryInvocationCounter counter) : IEventHandler + { + public ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + counter.Increment(); + throw new InvalidOperationException("Always fails"); + } + } + + /// + /// Always throws an InvalidOperationException (for the Ignore test). + /// + private sealed class ThrowInvalidOperationHandler(RetryInvocationCounter counter) : IEventHandler + { + public ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + counter.Increment(); + throw new InvalidOperationException("Should be ignored"); + } + } + + /// + /// Always throws an ArgumentException with ParamName = "test". + /// + private sealed class ThrowMatchingArgumentHandler(RetryInvocationCounter counter) : IEventHandler + { + public ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + counter.Increment(); + throw new ArgumentException("Argument error", "test"); + } + } + + /// + /// Always throws an ArgumentNullException (subclass of ArgumentException). + /// + private sealed class ThrowArgumentNullHandler(RetryInvocationCounter counter) : IEventHandler + { + public ValueTask HandleAsync(OrderCreated message, CancellationToken cancellationToken) + { + counter.Increment(); + throw new ArgumentNullException("param", "Null argument"); + } + } + + /// + /// Consumer that captures RetryState from the context features on each invocation, + /// then always throws to force retries. + /// + private sealed class RetryStateCapturingConsumer(RetryStateCapture capture) : IConsumer + { + public ValueTask ConsumeAsync(IConsumeContext context) + { + var retryState = context.Features.Get(); + capture.Record(retryState?.ImmediateRetryCount ?? -1); + throw new InvalidOperationException("Fail to trigger retry"); + } + } +} diff --git a/website/src/docs/mocha/v1/reliability.md b/website/src/docs/mocha/v1/reliability.md index 7bca2410f75..cd131e23117 100644 --- a/website/src/docs/mocha/v1/reliability.md +++ b/website/src/docs/mocha/v1/reliability.md @@ -1,6 +1,6 @@ --- title: "Reliability" -description: "Configure fault handling, dead-letter routing, message expiry, concurrency limits, circuit breakers, the transactional outbox, and the idempotent inbox in Mocha to build resilient messaging pipelines." +description: "Configure retry policies, delayed redelivery, fault handling, dead-letter routing, message expiry, concurrency limits, circuit breakers, the transactional outbox, and the idempotent inbox in Mocha to build resilient messaging pipelines." --- Messaging systems fail. Handlers throw exceptions, brokers go offline, databases lock up, and messages arrive faster than consumers can process them. Mocha's reliability features handle these failures at the infrastructure level so your handler code stays focused on business logic. @@ -8,6 +8,8 @@ Messaging systems fail. Handlers throw exceptions, brokers go offline, databases ```csharp builder.Services .AddMessageBus() + .AddRetry() + .AddRedelivery() .AddCircuitBreaker(opts => { opts.FailureRatio = 0.5; @@ -24,7 +26,7 @@ builder.Services .AddRabbitMQ(); ``` -That configuration adds circuit breaking, concurrency limiting, transactional outbox, idempotent inbox, and database transaction wrapping - all as middleware in the receive and dispatch pipelines. +That configuration adds immediate retry, delayed redelivery, circuit breaking, concurrency limiting, transactional outbox, idempotent inbox, and database transaction wrapping - all as middleware in the receive and dispatch pipelines. # Delivery guarantees @@ -48,15 +50,17 @@ TransportCircuitBreaker -> Instrumentation -> DeadLetter -> Fault - -> CircuitBreaker - -> Expiry - -> MessageTypeSelection - -> Routing - -> Consumer pipeline - -> Transaction middleware (BEGIN) - -> Inbox (claim inside transaction) - -> Your handler - -> Transaction middleware (COMMIT/ROLLBACK) + -> Redelivery ← schedules for later if retries exhausted + -> CircuitBreaker + -> Expiry + -> MessageTypeSelection + -> Routing + -> Consumer pipeline + -> Retry ← immediate in-process retries (Polly) + -> Transaction middleware (BEGIN) + -> Inbox (claim inside transaction) + -> Your handler + -> Transaction middleware (COMMIT/ROLLBACK) ``` Each middleware can intercept failures from downstream, transform them, or short-circuit the pipeline. The reliability middlewares - dead-letter, fault, circuit breaker, expiry, inbox, and concurrency limiter - are all enabled by default with sensible defaults. You tune them when the defaults do not match your workload. @@ -146,6 +150,427 @@ await bus.SendAsync( For time-sensitive commands, a short expiry prevents stale operations from executing after their validity window. +# Retry failed messages + +When a handler throws a transient exception - a database timeout, an HTTP 503, temporary lock contention - retrying the same operation a few hundred milliseconds later often succeeds. The retry middleware re-runs the handler in-process using [Polly](https://github.com/App-vNext/Polly) without releasing the concurrency slot or leaving the consumer pipeline. + +Retry lives in the **consumer pipeline**, not the receive pipeline. This matters for two reasons: + +1. **Only handler failures are retried.** Deserialization errors, unknown message types, and routing failures are almost always permanent. Retrying them wastes time. +2. **Multi-consumer correctness.** When one endpoint routes a message to multiple consumers, only the failing consumer retries. The others are unaffected. + +## Add retry with defaults + +```csharp +builder.Services + .AddMessageBus() + .AddRetry() + .AddEventHandler() + .AddRabbitMQ(); +``` + +With no configuration, retry uses exponential backoff: 3 attempts, 200 ms base delay, jitter enabled, 30-second maximum delay. These defaults handle the majority of transient failures without tuning. + +## Customize retry behavior + +```csharp +builder.Services + .AddMessageBus() + .AddRetry(retry => + { + retry.MaxRetryAttempts = 5; + retry.Delay = TimeSpan.FromSeconds(1); + retry.BackoffType = RetryBackoffType.Exponential; + retry.UseJitter = true; + retry.MaxDelay = TimeSpan.FromSeconds(60); + }) + .AddEventHandler() + .AddRabbitMQ(); +``` + +## Use explicit retry intervals + +When you need precise control over each delay, set `Intervals` directly. This overrides `Delay`, `BackoffType`, and `MaxRetryAttempts` - the array length becomes the number of retries. + +```csharp +builder.Services + .AddMessageBus() + .AddRetry(retry => + { + retry.Intervals = + [ + TimeSpan.FromMilliseconds(100), + TimeSpan.FromMilliseconds(500), + TimeSpan.FromSeconds(2) + ]; + }) + .AddEventHandler() + .AddRabbitMQ(); +``` + +## Filter exceptions + +Not every exception should be retried. Validation errors, authorization failures, and other permanent errors waste retry budget. Use `On().Ignore()` to exclude specific exception types from retry. The exception propagates immediately to the fault middleware. + +```csharp +builder.Services + .AddMessageBus() + .AddRetry(retry => + { + retry.MaxRetryAttempts = 3; + + // Never retry validation failures + retry.On().Ignore(); + + // Never retry non-transient database errors + retry.On(ex => !ex.IsTransient).Ignore(); + }) + .AddEventHandler() + .AddRabbitMQ(); +``` + +Exception filtering respects inheritance. `On().Ignore()` also ignores `NpgsqlException` and any other `DbException` subclass. When multiple rules match, the most specific type wins - the same precedence as C# `catch` blocks. + +## Override retry per consumer + +The bus-level retry configuration applies to all consumers. Override it for specific consumers that need different behavior. See [Handlers and Consumers](/docs/mocha/v1/handlers-and-consumers) for the full consumer configuration API. + +```csharp +builder.Services + .AddMessageBus() + .AddRetry() // bus-level: 3 retries for all consumers + .AddEventHandler(consumer => + { + // This consumer: 10 retries with longer delay + consumer.AddRetry(retry => + { + retry.MaxRetryAttempts = 10; + retry.Delay = TimeSpan.FromSeconds(2); + }); + }) + .AddEventHandler(consumer => + { + // This consumer: no retries + consumer.AddRetry(retry => retry.Enabled = false); + }) + .AddRabbitMQ(); +``` + +Properties cascade: consumer overrides bus, bus overrides defaults. Properties not set at the consumer level (like `BackoffType`) fall through to the bus configuration, then to `RetryOptions.Defaults`. + +## Retry options reference + +| Property | Type | Default | Description | +| ------------------ | ------------------- | ------------- | ------------------------------------------------------------------------------------------------ | +| `Enabled` | `bool?` | `true` | Set to `false` to disable retry at this scope. `null` inherits from parent. | +| `MaxRetryAttempts` | `int?` | `3` | Number of retries after the initial attempt. Total handler invocations = `MaxRetryAttempts + 1`. | +| `Delay` | `TimeSpan?` | 200 ms | Base delay between retries. Interpretation depends on `BackoffType`. | +| `MaxDelay` | `TimeSpan?` | 30 s | Upper bound on any single retry delay. Prevents exponential backoff from growing unbounded. | +| `BackoffType` | `RetryBackoffType?` | `Exponential` | Delay calculation strategy: `Constant`, `Linear`, or `Exponential`. | +| `UseJitter` | `bool?` | `true` | Adds random variance to delays, preventing synchronized retry storms across consumers. | +| `Intervals` | `TimeSpan[]?` | `null` | Explicit delay per retry. When set, overrides `Delay`, `BackoffType`, and `MaxRetryAttempts`. | + +### RetryBackoffType values + +| Value | Formula | Example (Delay = 200 ms) | +| ------------- | ----------------------- | ------------------------ | +| `Constant` | `Delay` | 200 ms, 200 ms, 200 ms | +| `Linear` | `Delay × (attempt + 1)` | 400 ms, 600 ms, 800 ms | +| `Exponential` | `Delay × 2^attempt` | 400 ms, 800 ms, 1600 ms | + +# Redeliver failed messages + +Retry handles transient blips that resolve in milliseconds. Some failures take longer - a downstream service is deploying, a database is recovering from a failover, an external API is rate-limiting. For these, you need **redelivery**: reschedule the message for later delivery through the transport. + +Redelivery lives in the **receive pipeline**, not the consumer pipeline. This matters for two reasons: + +1. **Single decision point.** When an endpoint has multiple consumers, you want one redelivery decision per message, not one per consumer. +2. **Releases the concurrency slot.** Retry holds the slot while Polly waits. Redelivery returns the message to the transport and frees the slot for other messages. + +Redelivery uses Mocha's [scheduling infrastructure](/docs/mocha/v1/scheduling) to schedule the message for future delivery. The message re-enters the full receive pipeline when it arrives - fresh routing, fresh consumer invocations, fresh retry attempts. + +```text +Receive pipeline: + + Fault + -> Redelivery + -> CircuitBreaker + -> ... (rest of pipeline) + +All retries exhausted → exception propagates to Redelivery + → redelivery attempts remaining → schedule for later delivery + → redelivery exhausted → exception propagates to Fault → error endpoint +``` + +## Add redelivery with defaults + +```csharp +builder.Services + .AddMessageBus() + .AddRedelivery() + .AddEventHandler() + .AddRabbitMQ(); +``` + +The default redelivery schedule is three attempts at 5 minutes, 15 minutes, and 30 minutes with jitter enabled. + +## Use custom redelivery intervals + +```csharp +builder.Services + .AddMessageBus() + .AddRedelivery(redeliver => + { + redeliver.Intervals = + [ + TimeSpan.FromSeconds(30), + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(30) + ]; + }) + .AddEventHandler() + .AddRabbitMQ(); +``` + +The array length determines the number of redelivery attempts. Each element is the delay before that attempt. + +## Use calculated redelivery delays + +Instead of explicit intervals, configure a base delay and maximum. The actual delay for each attempt is `BaseDelay × (attempt + 1)`, capped at `MaxDelay`. + +```csharp +builder.Services + .AddMessageBus() + .AddRedelivery(redeliver => + { + redeliver.MaxAttempts = 5; + redeliver.BaseDelay = TimeSpan.FromMinutes(2); + redeliver.MaxDelay = TimeSpan.FromHours(1); + redeliver.UseJitter = true; + }) + .AddEventHandler() + .AddRabbitMQ(); +``` + +## Filter exceptions for redelivery + +The same `On().Ignore()` pattern applies to redelivery. Exceptions that match an ignore rule propagate immediately to the fault middleware without scheduling a redelivery. + +```csharp +builder.Services + .AddMessageBus() + .AddRedelivery(redeliver => + { + redeliver.Intervals = + [ + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(15), + TimeSpan.FromMinutes(30) + ]; + + // Validation failures are permanent - don't redeliver + redeliver.On().Ignore(); + + // Missing configuration won't fix itself + redeliver.On().Ignore(); + }) + .AddEventHandler() + .AddRabbitMQ(); +``` + +## Override redelivery per endpoint + +The bus-level redelivery configuration applies to all endpoints. Override it at the transport or endpoint level for more granular control. + +```csharp +builder.Services + .AddMessageBus() + .AddRedelivery() // bus-level default + .AddRabbitMQ(transport => + { + transport.AddRedelivery(redeliver => + { + // Override for this transport: longer intervals + redeliver.Intervals = + [ + TimeSpan.FromMinutes(10), + TimeSpan.FromMinutes(30), + TimeSpan.FromHours(1) + ]; + }); + }); +``` + +Disable redelivery for a specific transport or endpoint by setting `Enabled` to `false`: + +```csharp +transport.AddRedelivery(redeliver => redeliver.Enabled = false); +``` + +Properties cascade: endpoint overrides transport, transport overrides bus, bus overrides defaults. + +## Redelivery options reference + +| Property | Type | Default | Description | +| ------------- | ------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Enabled` | `bool?` | `true` | Set to `false` to disable redelivery at this scope. `null` inherits from parent. | +| `MaxAttempts` | `int?` | - | Maximum redelivery attempts. Inferred from `Intervals.Length` when `Intervals` is set. Required when using calculated delays without `Intervals`. | +| `BaseDelay` | `TimeSpan?` | 5 min | Base delay for calculated backoff: `BaseDelay × (attempt + 1)`. | +| `MaxDelay` | `TimeSpan?` | 1 h | Upper bound on any single redelivery delay. | +| `UseJitter` | `bool?` | `true` | Adds random variance to delays. | +| `Intervals` | `TimeSpan[]?` | [5 min, 15 min, 30 min] | Explicit delay per redelivery attempt. When set, overrides `BaseDelay` and `MaxAttempts`. | + +# Combine retry and redelivery + +The two tiers compose naturally. When both are configured, a message goes through immediate retry first. If all retries are exhausted, the exception propagates up from the consumer pipeline to the receive pipeline, where redelivery catches it and schedules the message for later delivery. On the next delivery round, the full retry cycle runs again. + +```text +Message arrives + │ + ├─ Consumer pipeline: Retry (Polly) + │ ├─ Attempt 1 → handler throws → retry + │ ├─ Attempt 2 → handler throws → retry + │ ├─ Attempt 3 → handler throws → retry + │ └─ Attempt 4 → handler throws → retries exhausted, exception propagates + │ + ├─ Receive pipeline: Redelivery + │ └─ Schedule message for delivery in 5 minutes + │ + ├─ ... 5 minutes later, message re-enters pipeline ... + │ + ├─ Consumer pipeline: Retry (Polly) + │ ├─ Attempt 1 → handler throws → retry + │ ├─ ... + │ └─ Attempt 4 → retries exhausted, exception propagates + │ + ├─ Receive pipeline: Redelivery + │ └─ Schedule message for delivery in 15 minutes + │ + ├─ ... continues until redelivery attempts exhausted ... + │ + └─ Fault middleware → error endpoint (dead letter) +``` + +The total number of handler invocations before a message reaches the error endpoint: + +``` +Total attempts = (MaxRetryAttempts + 1) × (redelivery attempts + 1) +``` + +With the defaults (3 retries, 3 redeliveries): `(3 + 1) × (3 + 1) = 16` total handler invocations. + +```csharp +builder.Services + .AddMessageBus() + .AddRetry(retry => + { + retry.MaxRetryAttempts = 5; + retry.Delay = TimeSpan.FromSeconds(1); + retry.BackoffType = RetryBackoffType.Exponential; + }) + .AddRedelivery(redeliver => + { + redeliver.Intervals = + [ + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(15), + TimeSpan.FromMinutes(30) + ]; + }) + .AddEventHandler() + .AddRabbitMQ(); +``` + +With this configuration: `(5 + 1) × (3 + 1) = 24` total handler invocations before dead-letter. + +:::note Request/reply messages skip redelivery +Redelivery does not apply to request/reply messages. The caller is waiting synchronously for a response - scheduling the message for delivery minutes later would cause a timeout. When a request/reply handler fails after retry exhaustion, the exception propagates directly to the fault middleware, which sends a `NotAcknowledgedEvent` back to the caller. Immediate retry still applies to request/reply handlers. +::: + +# Inspect retry state in handlers + +Handlers can access the current retry state through the context features. This is useful for graceful degradation - trying a primary path on the first attempt and falling back to an alternative on subsequent retries. + +```csharp +public class PaymentConsumer(ILogger logger) : IConsumer +{ + public async ValueTask ConsumeAsync(IConsumeContext context) + { + var state = context.Features.Get(); + + if (state is { ImmediateRetryCount: > 2 }) + { + logger.LogWarning( + "Primary gateway failed after {Retries} retries, using fallback", + state.ImmediateRetryCount); + await ProcessViaFallbackGatewayAsync(context.Message); + return; + } + + if (state is { DelayedRetryCount: > 0 }) + { + logger.LogWarning( + "Processing after {Redeliveries} redeliveries", + state.DelayedRetryCount); + } + + await ProcessPaymentAsync(context.Message); + } +} +``` + +`RetryState` is `null` when `AddRetry()` is not configured. When present, it exposes two properties: + +| Property | Type | Description | +| --------------------- | ----- | ------------------------------------------------------------------------------------------ | +| `ImmediateRetryCount` | `int` | Number of immediate retries so far in this delivery round. `0` on the initial attempt. | +| `DelayedRetryCount` | `int` | Number of redelivery rounds completed. Read from the `delayed-retry-count` message header. | + +## Retry and sagas + +[Sagas](/docs/mocha/v1/sagas) are consumers, so retry and redelivery apply automatically — no special configuration needed. Each saga handler invocation runs inside a saga transaction. If the handler throws, the transaction is never committed and all state changes are discarded. On the next retry attempt, the saga state is loaded fresh from the store. + +This means retry is safe for sagas by default. You do not need to worry about partial state mutations leaking between retry attempts. + +Redelivery is also safe. When a redelivered message arrives, a new transaction starts and the saga state is loaded at whatever state the last successful commit left it. + +# Troubleshoot retry and redelivery + +## "My handler runs more times than expected" + +Check both retry and redelivery. With both configured, the total handler invocations are `(MaxRetryAttempts + 1) × (redelivery attempts + 1)`. Three retries with three redeliveries means 16 total invocations, not 6. Use the [formula above](#combine-retry-and-redelivery) to calculate the exact count. + +## "Retry does not work for my consumer" + +`AddRetry()` must be called at the bus level (or on the specific consumer) to register the retry middleware in the consumer pipeline. Adding retry configuration to a consumer without a bus-level `AddRetry()` call has no effect - the middleware is not in the pipeline. + +## "Redelivery fails at startup" + +Redelivery uses Mocha's [scheduling infrastructure](/docs/mocha/v1/scheduling). If your transport does not support scheduling or the scheduling store is not configured, redelivery cannot schedule messages for later delivery. Check that your transport is configured with scheduling support. + +## "Validation exceptions are being retried" + +Use `On().Ignore()` to exclude permanent failures from retry and redelivery: + +```csharp +builder.Services + .AddMessageBus() + .AddRetry(retry => + { + retry.On().Ignore(); + }) + .AddRedelivery(redeliver => + { + redeliver.On().Ignore(); + }) + .AddEventHandler() + .AddRabbitMQ(); +``` + +## "Request/reply messages are not redelivered" + +This is by design. Redelivery skips request/reply messages because the caller is waiting for a synchronous response. Immediate retry still applies. See the [note above](#combine-retry-and-redelivery) for details. + # Limit concurrency The concurrency limiter middleware restricts how many messages a receive endpoint processes in parallel. @@ -551,8 +976,10 @@ The inbox cleanup worker is a background hosted service (`IHostedService`). It r # Next steps -Your messaging pipeline now handles failures, limits concurrency, breaks circuits on sustained errors, guarantees delivery through the outbox, and deduplicates messages through the inbox. To monitor your messaging system, see [Observability](/docs/mocha/v1/observability). +Your messaging pipeline now retries transient failures, redelivers after sustained errors, limits concurrency, breaks circuits on repeated failures, guarantees delivery through the outbox, and deduplicates messages through the inbox. To monitor your messaging system, see [Observability](/docs/mocha/v1/observability). +- [**Handlers and Consumers**](/docs/mocha/v1/handlers-and-consumers) - Configure per-consumer retry overrides and understand handler exception behavior. +- [**Scheduling**](/docs/mocha/v1/scheduling) - Configure the scheduling infrastructure that redelivery uses for delayed message delivery. - [**Middleware and Pipelines**](/docs/mocha/v1/middleware-and-pipelines) - Write custom middleware, control pipeline ordering, and understand the three pipeline stages. - [**Sagas**](/docs/mocha/v1/sagas) - Coordinate multi-step workflows with state machine sagas that use compensation when steps fail. - [**Observability**](/docs/mocha/v1/observability) - Trace message flows across services and monitor pipeline health with OpenTelemetry. From 0dce73420a65ce6608dbe9d4da61aa3ea706b0b0 Mon Sep 17 00:00:00 2001 From: Pascal Senn Date: Tue, 31 Mar 2026 23:21:22 +0200 Subject: [PATCH 02/14] Add retry/redeliver to mocha --- .../Consume/Retry/ConsumerRetryMiddleware.cs | 32 ++++++------------- .../{RetryState.cs => RetryRuntimeFeature.cs} | 4 +-- .../Behaviors/RetryTests.cs | 2 +- 3 files changed, 13 insertions(+), 25 deletions(-) rename src/Mocha/src/Mocha/Middlewares/Consume/Retry/{RetryState.cs => RetryRuntimeFeature.cs} (86%) diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs index 218c550f05f..eb37daa3278 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs @@ -8,8 +8,7 @@ namespace Mocha; /// A consumer middleware that implements in-process retry using Polly, replaying the handler /// with configurable backoff strategies when transient failures occur. /// -internal sealed class ConsumerRetryMiddleware( - ResiliencePipeline resiliencePipeline) +internal sealed class ConsumerRetryMiddleware(ResiliencePipeline resiliencePipeline) { public async ValueTask InvokeAsync(IConsumeContext context, ConsumerDelegate next) { @@ -24,11 +23,7 @@ public async ValueTask InvokeAsync(IConsumeContext context, ConsumerDelegate nex // Expose retry state to handlers via features. // ImmediateRetryCount starts at -1 so the first increment in the callback yields 0. - var retryState = new RetryState - { - DelayedRetryCount = delayedRetryCount, - ImmediateRetryCount = -1 - }; + var retryState = new RetryRuntimeFeature { DelayedRetryCount = delayedRetryCount, ImmediateRetryCount = -1 }; context.Features.Set(retryState); await resiliencePipeline.ExecuteAsync( @@ -54,21 +49,18 @@ public static ConsumerMiddlewareConfiguration Create() } var intervals = context.GetConfiguration(f => f.Intervals); - var maxRetryAttempts = intervals?.Length + var maxRetryAttempts = + intervals?.Length ?? context.GetConfiguration(f => f.MaxRetryAttempts) ?? RetryOptions.Defaults.MaxRetryAttempts; - var delay = context.GetConfiguration(f => f.Delay) - ?? RetryOptions.Defaults.Delay; + var delay = context.GetConfiguration(f => f.Delay) ?? RetryOptions.Defaults.Delay; - var maxDelay = context.GetConfiguration(f => f.MaxDelay) - ?? RetryOptions.Defaults.MaxDelay; + var maxDelay = context.GetConfiguration(f => f.MaxDelay) ?? RetryOptions.Defaults.MaxDelay; - var backoffType = context.GetConfiguration(f => f.BackoffType) - ?? RetryOptions.Defaults.BackoffType; + var backoffType = context.GetConfiguration(f => f.BackoffType) ?? RetryOptions.Defaults.BackoffType; - var useJitter = context.GetConfiguration(f => f.UseJitter) - ?? RetryOptions.Defaults.UseJitter; + var useJitter = context.GetConfiguration(f => f.UseJitter) ?? RetryOptions.Defaults.UseJitter; // Resolve exception rules (atomically from first scope that has them). var exceptionRules = ResolveExceptionRules(context); @@ -116,9 +108,7 @@ public static ConsumerMiddlewareConfiguration Create() }; } - var pipeline = new ResiliencePipelineBuilder() - .AddRetry(strategyOptions) - .Build(); + var pipeline = new ResiliencePipelineBuilder().AddRetry(strategyOptions).Build(); var middleware = new ConsumerRetryMiddleware(pipeline); @@ -153,9 +143,7 @@ file static class Extensions /// /// Resolves configuration with the most specific scope taking precedence. /// - public static T? GetConfiguration( - this ConsumerMiddlewareFactoryContext context, - Func selector) + public static T? GetConfiguration(this ConsumerMiddlewareFactoryContext context, Func selector) { var busFeatures = context.Services.GetRequiredService(); diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryState.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs similarity index 86% rename from src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryState.cs rename to src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs index dad6f16da19..84f0fa5ee0c 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryState.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs @@ -5,7 +5,7 @@ namespace Mocha; /// context.Features.Get<RetryState>(). /// Null if AddRetry is not configured. /// -public sealed class RetryState +public sealed class RetryRuntimeFeature { /// /// Number of immediate retries already attempted for this delivery round. @@ -15,7 +15,7 @@ public sealed class RetryState /// /// Number of delayed redeliveries already attempted. - /// Read from the Mocha-Retry-DelayedRetryCount header. + /// Read from the delayed-retry-count header. /// public int DelayedRetryCount { get; internal set; } } diff --git a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs index c16aa1550e1..b08cac99b35 100644 --- a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs +++ b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs @@ -508,7 +508,7 @@ private sealed class RetryStateCapturingConsumer(RetryStateCapture capture) : IC { public ValueTask ConsumeAsync(IConsumeContext context) { - var retryState = context.Features.Get(); + var retryState = context.Features.Get(); capture.Record(retryState?.ImmediateRetryCount ?? -1); throw new InvalidOperationException("Fail to trigger retry"); } From c49da9cef0932e6aab81b9e43700ff55ef0a4f8d Mon Sep 17 00:00:00 2001 From: Pascal Senn Date: Mon, 6 Apr 2026 23:33:38 +0200 Subject: [PATCH 03/14] cleanup --- .../ExceptionPolicies.csproj | 13 + .../Exceptions/Exceptions.cs | 44 ++ .../ExceptionPolicies/Handlers/Handlers.cs | 148 ++++++ .../ExceptionPolicies/Messages/Messages.cs | 17 + .../examples/ExceptionPolicies/Program.cs | 125 +++++ src/Mocha/src/Demo/Demo.Billing/Program.cs | 3 +- src/Mocha/src/Demo/Demo.Catalog/Program.cs | 3 +- src/Mocha/src/Demo/Demo.Shipping/Program.cs | 3 +- .../Builder/MessageBusBuilderExtensions.cs | 2 + .../Consume/Retry/ConsumerRetryMiddleware.cs | 215 ++++---- .../Consume/Retry/ExceptionPolicyBuilder.cs | 195 ++++++- .../ExceptionPolicyConfigurationExtensions.cs | 122 +++++ .../Consume/Retry/ExceptionPolicyFeature.cs | 40 ++ ...leMatcher.cs => ExceptionPolicyMatcher.cs} | 18 +- .../Consume/Retry/ExceptionPolicyOptions.cs | 53 ++ .../Consume/Retry/ExceptionPolicyRule.cs | 48 ++ .../Consume/Retry/ExceptionRule.cs | 33 -- .../Consume/Retry/IAfterRedeliveryBuilder.cs | 13 + .../Consume/Retry/IAfterRetryBuilder.cs | 34 ++ .../Consume/Retry/IExceptionPolicyBuilder.cs | 73 +++ .../Consume/Retry/RedeliveryPolicyConfig.cs | 37 ++ .../Consume/Retry/RedeliveryPolicyDefaults.cs | 27 + .../Consume/Retry/RetryBackoffType.cs | 6 +- .../Retry/RetryConfigurationExtensions.cs | 90 ---- .../Middlewares/Consume/Retry/RetryFeature.cs | 75 --- .../Middlewares/Consume/Retry/RetryOptions.cs | 102 ---- .../Consume/Retry/RetryPolicyConfig.cs | 42 ++ .../Consume/Retry/RetryPolicyDefaults.cs | 32 ++ .../Consume/Retry/RetryRuntimeFeature.cs | 2 +- .../Redelivery/ReceiveRedeliveryMiddleware.cs | 165 +++--- .../RedeliveryConfigurationExtensions.cs | 93 ---- .../Receive/Redelivery/RedeliveryFeature.cs | 70 --- .../Receive/Redelivery/RedeliveryOptions.cs | 92 ---- .../Behaviors/RedeliveryTests.cs | 51 +- .../Behaviors/RetryTests.cs | 105 ++-- .../src/docs/mocha/v1/exception-policies.md | 481 ++++++++++++++++++ website/src/docs/mocha/v1/reliability.md | 301 ++++++----- 37 files changed, 1943 insertions(+), 1030 deletions(-) create mode 100644 src/Mocha/examples/ExceptionPolicies/ExceptionPolicies.csproj create mode 100644 src/Mocha/examples/ExceptionPolicies/Exceptions/Exceptions.cs create mode 100644 src/Mocha/examples/ExceptionPolicies/Handlers/Handlers.cs create mode 100644 src/Mocha/examples/ExceptionPolicies/Messages/Messages.cs create mode 100644 src/Mocha/examples/ExceptionPolicies/Program.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyConfigurationExtensions.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyFeature.cs rename src/Mocha/src/Mocha/Middlewares/Consume/Retry/{ExceptionRuleMatcher.cs => ExceptionPolicyMatcher.cs} (60%) create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyOptions.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyRule.cs delete mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRule.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/IAfterRedeliveryBuilder.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/IAfterRetryBuilder.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/IExceptionPolicyBuilder.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyConfig.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyDefaults.cs delete mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryConfigurationExtensions.cs delete mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryFeature.cs delete mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryOptions.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyConfig.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyDefaults.cs delete mode 100644 src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryConfigurationExtensions.cs delete mode 100644 src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryFeature.cs delete mode 100644 src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryOptions.cs create mode 100644 website/src/docs/mocha/v1/exception-policies.md diff --git a/src/Mocha/examples/ExceptionPolicies/ExceptionPolicies.csproj b/src/Mocha/examples/ExceptionPolicies/ExceptionPolicies.csproj new file mode 100644 index 00000000000..5719a995f08 --- /dev/null +++ b/src/Mocha/examples/ExceptionPolicies/ExceptionPolicies.csproj @@ -0,0 +1,13 @@ + + + Exe + net10.0 + enable + + + + + + + + diff --git a/src/Mocha/examples/ExceptionPolicies/Exceptions/Exceptions.cs b/src/Mocha/examples/ExceptionPolicies/Exceptions/Exceptions.cs new file mode 100644 index 00000000000..4154a60a0ee --- /dev/null +++ b/src/Mocha/examples/ExceptionPolicies/Exceptions/Exceptions.cs @@ -0,0 +1,44 @@ +namespace ExceptionPolicies.Exceptions; + +/// +/// Transient database failure — worth retrying because it usually resolves quickly. +/// +public class TransientDatabaseException(string message) : Exception(message); + +/// +/// The message payload is malformed — retrying will never help. +/// +public class MessageValidationException(string message) : Exception(message); + +/// +/// The message was already processed — expected in at-least-once delivery. +/// +public class DuplicateMessageException(string message) : Exception(message); + +/// +/// Payment gateway returned an error — flaky but usually recovers. +/// +public class PaymentGatewayException(string message) : Exception(message); + +/// +/// Auth token expired — immediate retry is pointless, need to wait for refresh. +/// +public class AuthTokenExpiredException(string message) : Exception(message); + +/// +/// External service is completely unavailable — needs time to recover. +/// +public class ExternalServiceUnavailableException(string message) : Exception(message); + +/// +/// HTTP-level failure with a status code for conditional policy matching. +/// +public class HttpServiceException(string message, int statusCode) : Exception(message) +{ + public int StatusCode { get; } = statusCode; +} + +/// +/// Corrupt or unparseable message payload — a poison message. +/// +public class PoisonMessageException(string message) : Exception(message); diff --git a/src/Mocha/examples/ExceptionPolicies/Handlers/Handlers.cs b/src/Mocha/examples/ExceptionPolicies/Handlers/Handlers.cs new file mode 100644 index 00000000000..7b48de3e008 --- /dev/null +++ b/src/Mocha/examples/ExceptionPolicies/Handlers/Handlers.cs @@ -0,0 +1,148 @@ +using ExceptionPolicies.Exceptions; +using ExceptionPolicies.Messages; +using Mocha; + +namespace ExceptionPolicies.Handlers; + +/// +/// Simulates a flaky payment gateway that succeeds after 3 failures. +/// Policy: Retry 5x with exponential backoff. +/// +public class ProcessPaymentHandler : IEventHandler +{ + private static int _attempts; + + public ValueTask HandleAsync(ProcessPayment message, CancellationToken cancellationToken) + { + var attempt = Interlocked.Increment(ref _attempts); + Console.WriteLine($"[Payment] Attempt {attempt} for order {message.OrderId}"); + + if (attempt <= 3) + { + throw new PaymentGatewayException("Gateway timeout"); + } + + Console.WriteLine($"[Payment] Successfully processed order {message.OrderId}"); + return ValueTask.CompletedTask; + } +} + +/// +/// Receives a message with an invalid payload. +/// Policy: DeadLetter immediately — the message is permanently bad. +/// +public class ValidateOrderHandler : IEventHandler +{ + public ValueTask HandleAsync(ValidateOrder message, CancellationToken cancellationToken) + { + Console.WriteLine($"[Validate] Validating order {message.OrderId}"); + throw new MessageValidationException($"Order {message.OrderId} has invalid schema"); + } +} + +/// +/// Detects a duplicate message that was already processed. +/// Policy: Discard silently — no retry, no dead-letter. +/// +public class DeduplicateMessageHandler : IEventHandler +{ + public ValueTask HandleAsync(DeduplicateMessage message, CancellationToken cancellationToken) + { + Console.WriteLine($"[Dedup] Message {message.MessageId} already processed"); + throw new DuplicateMessageException($"Message {message.MessageId} is a duplicate"); + } +} + +/// +/// Calls an external API that is completely down. +/// Policy: Retry 5x aggressively, then redeliver with increasing intervals, then dead-letter. +/// +public class CallExternalApiHandler : IEventHandler +{ + private static int _attempts; + + public ValueTask HandleAsync(CallExternalApi message, CancellationToken cancellationToken) + { + var attempt = Interlocked.Increment(ref _attempts); + Console.WriteLine($"[ExternalApi] Attempt {attempt} for {message.Url}"); + throw new ExternalServiceUnavailableException($"Service at {message.Url} is down"); + } +} + +/// +/// Service with an expired auth token. +/// Policy: Redeliver only (skip retry) — immediate retry won't help. +/// +public class RefreshAuthTokenHandler : IEventHandler +{ + private static int _attempts; + + public ValueTask HandleAsync(RefreshAuthToken message, CancellationToken cancellationToken) + { + var attempt = Interlocked.Increment(ref _attempts); + Console.WriteLine($"[Auth] Attempt {attempt} for service {message.Service}"); + throw new AuthTokenExpiredException($"Token for {message.Service} expired"); + } +} + +/// +/// Transient database failure during batch processing. +/// Policy: Retry 3x quickly, then escalate to redelivery. +/// +public class ProcessBatchHandler : IEventHandler +{ + private static int _attempts; + + public ValueTask HandleAsync(ProcessBatch message, CancellationToken cancellationToken) + { + var attempt = Interlocked.Increment(ref _attempts); + Console.WriteLine($"[Batch] Attempt {attempt} for batch {message.BatchId}"); + + if (attempt <= 4) + { + throw new TransientDatabaseException("Connection pool exhausted"); + } + + Console.WriteLine($"[Batch] Successfully processed batch {message.BatchId}"); + return ValueTask.CompletedTask; + } +} + +/// +/// Ingests telemetry data from a device, encountering various HTTP errors. +/// Policy: Conditional — different behavior for 404, 429, and 503 status codes. +/// +public class IngestTelemetryHandler : IEventHandler +{ + private static int _attempts; + + public ValueTask HandleAsync(IngestTelemetry message, CancellationToken cancellationToken) + { + var attempt = Interlocked.Increment(ref _attempts); + Console.WriteLine($"[Telemetry] Attempt {attempt} for device {message.DeviceId}"); + + // Rotate through different HTTP errors to demonstrate conditional policies + throw (attempt % 3) switch + { + 0 => new HttpServiceException("Not Found", 404), + 1 => new HttpServiceException("Too Many Requests", 429), + _ => new HttpServiceException("Service Unavailable", 503) + }; + } +} + +/// +/// Handles a corrupt/unparseable message. +/// Policy: Retry once (in case of transient parse issue), then dead-letter immediately. +/// +public class HandlePoisonMessageHandler : IEventHandler +{ + private static int _attempts; + + public ValueTask HandleAsync(HandlePoisonMessage message, CancellationToken cancellationToken) + { + var attempt = Interlocked.Increment(ref _attempts); + Console.WriteLine($"[Poison] Attempt {attempt} for data: {message.Data}"); + throw new PoisonMessageException("Payload cannot be deserialized"); + } +} diff --git a/src/Mocha/examples/ExceptionPolicies/Messages/Messages.cs b/src/Mocha/examples/ExceptionPolicies/Messages/Messages.cs new file mode 100644 index 00000000000..be0f9243ac0 --- /dev/null +++ b/src/Mocha/examples/ExceptionPolicies/Messages/Messages.cs @@ -0,0 +1,17 @@ +namespace ExceptionPolicies.Messages; + +public record ProcessPayment(string OrderId, decimal Amount); + +public record ValidateOrder(string OrderId); + +public record DeduplicateMessage(string MessageId); + +public record CallExternalApi(string Url); + +public record RefreshAuthToken(string Service); + +public record ProcessBatch(string BatchId); + +public record IngestTelemetry(string DeviceId); + +public record HandlePoisonMessage(string Data); diff --git a/src/Mocha/examples/ExceptionPolicies/Program.cs b/src/Mocha/examples/ExceptionPolicies/Program.cs new file mode 100644 index 00000000000..27892d97516 --- /dev/null +++ b/src/Mocha/examples/ExceptionPolicies/Program.cs @@ -0,0 +1,125 @@ +using ExceptionPolicies.Exceptions; +using ExceptionPolicies.Handlers; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Mocha; +using Mocha.Transport.InMemory; + +// --------------------------------------------------------------------------- +// Exception Policies Demo +// +// Demonstrates all per-exception policy configurations available in Mocha. +// Uses the InMemory transport for simplicity — no external dependencies. +// --------------------------------------------------------------------------- + +var builder = Host.CreateApplicationBuilder(args); + +builder.Services + .AddMessageBus() + + // ----------------------------------------------------------------------- + // Exception Policies — the main showcase + // + // Per-exception rules are configured in a single AddExceptionPolicy call. + // The On() catch-all provides global retry/redelivery defaults. + // ----------------------------------------------------------------------- + .AddExceptionPolicy(policy => + { + // --- Terminal: DeadLetter --- + // Validation errors are permanent — the message payload is bad. + // Skip retry and redelivery entirely; route straight to the error endpoint. + policy.On().DeadLetter(); + + // --- Terminal: Discard --- + // Duplicate messages are expected in at-least-once delivery systems. + // Silently drop them — no retry, no redelivery, no error endpoint. + policy.On().Discard(); + + // --- Retry only (skip redelivery) --- + // Payment gateway is flaky but usually recovers within a few attempts. + // Retry 5 times with exponential backoff, then dead-letter on exhaustion. + policy.On() + .Retry( + attempts: 5, + delay: TimeSpan.FromMilliseconds(200), + backoff: RetryBackoffType.Exponential); + + // --- Redeliver only (skip retry) --- + // Auth token expired — immediate retry is pointless because the token + // won't refresh in milliseconds. Wait for redelivery instead. + policy.On().Redeliver(); + + // --- Escalation: Retry then Redeliver --- + // Transient DB errors — try a few times quickly (connection hiccup), + // then back off with redelivery if the database is truly struggling. + policy.On() + .Retry(attempts: 3) + .ThenRedeliver(); + + // --- Escalation: Retry then DeadLetter (skip redelivery) --- + // Poison messages — try once in case it was a transient parse glitch, + // then give up immediately. Redelivery won't fix a corrupt payload. + policy.On() + .Retry(attempts: 1) + .ThenDeadLetter(); + + // --- Full chain: Retry -> Redeliver -> DeadLetter --- + // External service completely down — aggressive retry first, then + // patient redelivery with increasing intervals, then dead-letter + // as the last resort so operators can investigate. + policy.On() + .Retry(attempts: 5, delay: TimeSpan.FromMilliseconds(500)) + .ThenRedeliver( + [ + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(60) + ]) + .ThenDeadLetter(); + + // --- Conditional: Different policies for the same exception type --- + // HTTP 404 = resource is gone permanently, dead-letter it. + policy.On(ex => ex.StatusCode == 404) + .DeadLetter(); + + // HTTP 429 = rate limited, back off with redelivery. + policy.On(ex => ex.StatusCode == 429) + .Redeliver( + [ + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(15), + TimeSpan.FromSeconds(30) + ]); + + // HTTP 503 = service unavailable, retry quickly then redeliver. + policy.On(ex => ex.StatusCode == 503) + .Retry(attempts: 3) + .ThenRedeliver(); + + // --- Catch-all: Default for unmatched exceptions --- + // Most-specific-type-wins means this only fires for exceptions + // not matched by any of the rules above. + policy.On() + .Retry(attempts: 2) + .ThenRedeliver(); + }) + + // ----------------------------------------------------------------------- + // Register handlers + // ----------------------------------------------------------------------- + .AddEventHandler() + .AddEventHandler() + .AddEventHandler() + .AddEventHandler() + .AddEventHandler() + .AddEventHandler() + .AddEventHandler() + .AddEventHandler() + + // ----------------------------------------------------------------------- + // Transport — InMemory for this demo (no external dependencies) + // ----------------------------------------------------------------------- + .AddInMemory(); + +var app = builder.Build(); +await app.RunAsync(); diff --git a/src/Mocha/src/Demo/Demo.Billing/Program.cs b/src/Mocha/src/Demo/Demo.Billing/Program.cs index 9fd76fc4bb9..f9555aa69d1 100644 --- a/src/Mocha/src/Demo/Demo.Billing/Program.cs +++ b/src/Mocha/src/Demo/Demo.Billing/Program.cs @@ -31,8 +31,7 @@ builder .Services.AddMessageBus() .AddInstrumentation() - .AddRetry() - .AddRedelivery() + .AddExceptionPolicy() .AddBilling() .AddBatchHandler(opts => { diff --git a/src/Mocha/src/Demo/Demo.Catalog/Program.cs b/src/Mocha/src/Demo/Demo.Catalog/Program.cs index d81542bb287..f94211ca403 100644 --- a/src/Mocha/src/Demo/Demo.Catalog/Program.cs +++ b/src/Mocha/src/Demo/Demo.Catalog/Program.cs @@ -31,8 +31,7 @@ builder .Services.AddMessageBus() .AddInstrumentation() - .AddRetry() - .AddRedelivery() + .AddExceptionPolicy() .AddCatalog() .AddEntityFramework(p => { diff --git a/src/Mocha/src/Demo/Demo.Shipping/Program.cs b/src/Mocha/src/Demo/Demo.Shipping/Program.cs index a6f0553fb7d..2243c4aeae4 100644 --- a/src/Mocha/src/Demo/Demo.Shipping/Program.cs +++ b/src/Mocha/src/Demo/Demo.Shipping/Program.cs @@ -28,8 +28,7 @@ builder .Services.AddMessageBus() .AddInstrumentation() - .AddRetry() - .AddRedelivery() + .AddExceptionPolicy() .AddShipping() .AddEntityFramework(p => { diff --git a/src/Mocha/src/Mocha/Builder/MessageBusBuilderExtensions.cs b/src/Mocha/src/Mocha/Builder/MessageBusBuilderExtensions.cs index efb387b97db..3fe7171fa2d 100644 --- a/src/Mocha/src/Mocha/Builder/MessageBusBuilderExtensions.cs +++ b/src/Mocha/src/Mocha/Builder/MessageBusBuilderExtensions.cs @@ -30,6 +30,7 @@ public static IMessageBusBuilder ConfigureFeature( internal static void AddDefaults(this MessageBusBuilder builder) { + builder.UseConsume(ConsumerMiddlewares.Retry, before: "Instrumentation"); builder.UseConsume(ConsumerMiddlewares.Instrumentation); builder.UseReceive(ReceiveMiddlewares.TransportCircuitBreaker); @@ -37,6 +38,7 @@ internal static void AddDefaults(this MessageBusBuilder builder) builder.UseReceive(ReceiveMiddlewares.Instrumentation); builder.UseReceive(ReceiveMiddlewares.DeadLetter); builder.UseReceive(ReceiveMiddlewares.Fault); + builder.UseReceive(ReceiveMiddlewares.Redelivery, after: "Fault"); builder.UseReceive(ReceiveMiddlewares.CircuitBreaker); builder.UseReceive(ReceiveMiddlewares.Expiry); builder.UseReceive(ReceiveMiddlewares.MessageTypeSelection); diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs index eb37daa3278..5ad358a7ebd 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs @@ -1,14 +1,13 @@ using Microsoft.Extensions.DependencyInjection; -using Polly; -using Polly.Retry; namespace Mocha; /// -/// A consumer middleware that implements in-process retry using Polly, replaying the handler -/// with configurable backoff strategies when transient failures occur. +/// A consumer middleware that implements in-process retry with configurable backoff strategies +/// when transient failures occur. /// -internal sealed class ConsumerRetryMiddleware(ResiliencePipeline resiliencePipeline) +internal sealed class ConsumerRetryMiddleware( + IReadOnlyList exceptionPolicyRules) { public async ValueTask InvokeAsync(IConsumeContext context, ConsumerDelegate next) { @@ -22,152 +21,156 @@ public async ValueTask InvokeAsync(IConsumeContext context, ConsumerDelegate nex } // Expose retry state to handlers via features. - // ImmediateRetryCount starts at -1 so the first increment in the callback yields 0. - var retryState = new RetryRuntimeFeature { DelayedRetryCount = delayedRetryCount, ImmediateRetryCount = -1 }; + var retryState = new RetryRuntimeFeature { DelayedRetryCount = delayedRetryCount, ImmediateRetryCount = 0 }; context.Features.Set(retryState); - await resiliencePipeline.ExecuteAsync( - static (state, _) => - { - state.retryState.ImmediateRetryCount++; - return state.next(state.context); - }, - (context, next, retryState), - context.CancellationToken); - } + var attempts = 0; - public static ConsumerMiddlewareConfiguration Create() - => new( - static (context, next) => + while (true) + { + try { - // Feature values are resolved from consumer -> bus to support overrides. - var enabled = context.GetConfiguration(f => f.Enabled) ?? true; + await next(context); + return; + } + catch (Exception ex) + { + // Match exception against policy rules. + var rule = ExceptionPolicyMatcher.Match(exceptionPolicyRules, ex); - if (!enabled) + // No matching rule — no policy for this exception, let it propagate. + if (rule is null) { - return next; + throw; } - var intervals = context.GetConfiguration(f => f.Intervals); - var maxRetryAttempts = - intervals?.Length - ?? context.GetConfiguration(f => f.MaxRetryAttempts) - ?? RetryOptions.Defaults.MaxRetryAttempts; - - var delay = context.GetConfiguration(f => f.Delay) ?? RetryOptions.Defaults.Delay; - - var maxDelay = context.GetConfiguration(f => f.MaxDelay) ?? RetryOptions.Defaults.MaxDelay; - - var backoffType = context.GetConfiguration(f => f.BackoffType) ?? RetryOptions.Defaults.BackoffType; - - var useJitter = context.GetConfiguration(f => f.UseJitter) ?? RetryOptions.Defaults.UseJitter; - - // Resolve exception rules (atomically from first scope that has them). - var exceptionRules = ResolveExceptionRules(context); - - // Map RetryBackoffType to Polly's DelayBackoffType. - var pollyBackoffType = backoffType switch + // Discard: swallow at consumer level so other consumers can still run. + if (rule.Terminal == TerminalAction.Discard) { - RetryBackoffType.Constant => DelayBackoffType.Constant, - RetryBackoffType.Linear => DelayBackoffType.Linear, - RetryBackoffType.Exponential => DelayBackoffType.Exponential, - _ => DelayBackoffType.Exponential - }; + return; + } - var strategyOptions = new RetryStrategyOptions + // DeadLetter: don't retry, let it propagate to fault middleware. + if (rule.Terminal == TerminalAction.DeadLetter) { - MaxRetryAttempts = maxRetryAttempts, - Delay = delay, - MaxDelay = maxDelay, - BackoffType = pollyBackoffType, - UseJitter = useJitter - }; - - // Map Intervals to custom DelayGenerator. - if (intervals is { Length: > 0 }) + throw; + } + + // No retry configured for this rule, or retry explicitly disabled. + if (rule.Retry is null or { Enabled: false }) { - strategyOptions.DelayGenerator = args => - { - var index = Math.Min(args.AttemptNumber, intervals.Length - 1); - return new ValueTask(intervals[index]); - }; + throw; } - // Map On().Ignore() rules to ShouldHandle predicate. - if (exceptionRules.Count > 0) + attempts++; + + // Use the rule's retry config (fully populated by the builder). + var retryConfig = rule.Retry; + + if (attempts > (retryConfig.Attempts ?? RetryPolicyDefaults.Attempts)) { - strategyOptions.ShouldHandle = args => - { - if (args.Outcome.Exception is { } ex - && ExceptionRuleMatcher.ShouldIgnore(exceptionRules, ex)) - { - return new ValueTask(false); - } - - return new ValueTask(args.Outcome.Exception is not null); - }; + throw; } - var pipeline = new ResiliencePipelineBuilder().AddRetry(strategyOptions).Build(); + // Calculate delay. + var delay = CalculateDelay(attempts, retryConfig); - var middleware = new ConsumerRetryMiddleware(pipeline); + // Update runtime feature. + retryState.ImmediateRetryCount = attempts; - return ctx => middleware.InvokeAsync(ctx, next); - }, - "Retry"); + if (delay > TimeSpan.Zero) + { + await Task.Delay(delay, context.CancellationToken); + } + } + } + } - private static IReadOnlyList ResolveExceptionRules(ConsumerMiddlewareFactoryContext context) + private static TimeSpan CalculateDelay(int attempt, RetryPolicyConfig config) { - var busFeatures = context.Services.GetRequiredService(); + // Explicit intervals take precedence. + if (config.Intervals is { Length: > 0 } intervals) + { + var index = Math.Min(attempt - 1, intervals.Length - 1); + return intervals[index]; + } + + // Calculate based on backoff type. + var baseDelay = config.Delay ?? RetryPolicyDefaults.Delay; + var backoff = config.Backoff ?? RetryPolicyDefaults.Backoff; + var maxDelay = config.MaxDelay ?? RetryPolicyDefaults.MaxDelay; + var useJitter = config.UseJitter ?? RetryPolicyDefaults.UseJitter; - // Consumer rules take precedence if present. - if (context.Consumer.Configuration?.Features is { } consumerFeatures - && consumerFeatures.TryGet(out RetryFeature? consumerFeature) - && consumerFeature.ExceptionRules.Count > 0) + var delay = backoff switch { - return consumerFeature.ExceptionRules; + RetryBackoffType.Constant => baseDelay, + RetryBackoffType.Linear => baseDelay * attempt, + RetryBackoffType.Exponential => baseDelay * Math.Pow(2, attempt - 1), + _ => baseDelay * Math.Pow(2, attempt - 1) + }; + + // Cap at max delay. + if (delay > maxDelay) + { + delay = maxDelay; } - if (busFeatures.TryGet(out RetryFeature? busFeature) - && busFeature.ExceptionRules.Count > 0) + // Add jitter: +/- 25%. + if (useJitter) { - return busFeature.ExceptionRules; + var jitterRange = delay.TotalMilliseconds * 0.25; + var jitter = (Random.Shared.NextDouble() * 2 - 1) * jitterRange; + delay = TimeSpan.FromMilliseconds(Math.Max(0, delay.TotalMilliseconds + jitter)); } - return []; + return delay; } + + public static ConsumerMiddlewareConfiguration Create() + => new( + static (context, next) => + { + var feature = context.GetExceptionPolicyFeature(); + + if (feature is null) + { + // No exception policy configured — skip retry middleware entirely. + return next; + } + + var middleware = new ConsumerRetryMiddleware(feature.Rules); + + return ctx => middleware.InvokeAsync(ctx, next); + }, + "Retry"); } file static class Extensions { /// - /// Resolves configuration with the most specific scope taking precedence. + /// Resolves exception policy feature with the most specific scope taking precedence. + /// Consumer-level ExceptionPolicyFeature overrides bus-level. /// - public static T? GetConfiguration(this ConsumerMiddlewareFactoryContext context, Func selector) + public static ExceptionPolicyFeature? GetExceptionPolicyFeature(this ConsumerMiddlewareFactoryContext context) { var busFeatures = context.Services.GetRequiredService(); - // consumer -> bus (most specific first) - if (context.Consumer.Configuration?.Features is { } consumerFeatures) + // Consumer -> bus (most specific first). + var config = context.Consumer.Configuration; + if (config is not null) { - var value = consumerFeatures.GetFeatureValue(selector); - - if (value is not null) + var consumerFeatures = config.GetFeatures(); + if (consumerFeatures.TryGet(out ExceptionPolicyFeature? consumerFeature)) { - return value; + return consumerFeature; } } - return busFeatures.GetFeatureValue(selector); - } - - private static T? GetFeatureValue(this IFeatureCollection features, Func selector) - { - if (features.TryGet(out RetryFeature? feature)) + if (busFeatures.TryGet(out ExceptionPolicyFeature? busFeature)) { - return selector(feature); + return busFeature; } - return default; + return null; } } diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs index 42f03e6f37e..a7308f473cc 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs @@ -1,35 +1,186 @@ namespace Mocha; -/// -/// Fluent builder for configuring per-exception retry/redelivery behavior. -/// -/// The exception type to configure behavior for. -public sealed class ExceptionPolicyBuilder where TException : Exception +internal sealed class ExceptionPolicyBuilder + : IExceptionPolicyBuilder, IAfterRetryBuilder, IAfterRedeliveryBuilder + where TException : Exception { - private readonly List _rules; - private readonly Func? _predicate; + private readonly ExceptionPolicyRule _rule; + private readonly List _rules; + private bool _committed; - internal ExceptionPolicyBuilder(List rules, Func? predicate) + internal ExceptionPolicyBuilder(ExceptionPolicyRule rule, List rules) { + _rule = rule; _rules = rules; - _predicate = predicate; } - /// - /// Excludes this exception type from retry/redelivery. The exception propagates - /// immediately without being retried. - /// - public void Ignore() + private void EnsureCommitted() { - Func? wrappedPredicate = _predicate is not null - ? ex => ex is TException typed && _predicate(typed) - : null; + if (!_committed) + { + _rules.Add(_rule); + _committed = true; + } + } + + // IExceptionPolicyBuilder + + public void Discard() + { + EnsureCommitted(); + _rule.Terminal = TerminalAction.Discard; + } + + public void DeadLetter() + { + EnsureCommitted(); + _rule.Terminal = TerminalAction.DeadLetter; + } + + public IAfterRetryBuilder Retry() + { + EnsureCommitted(); + _rule.Retry = new RetryPolicyConfig + { + Attempts = RetryPolicyDefaults.Attempts, + Delay = RetryPolicyDefaults.Delay, + Backoff = RetryPolicyDefaults.Backoff, + UseJitter = RetryPolicyDefaults.UseJitter, + MaxDelay = RetryPolicyDefaults.MaxDelay + }; + _rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; + return this; + } - _rules.Add(new ExceptionRule + public IAfterRetryBuilder Retry(int attempts) + { + EnsureCommitted(); + _rule.Retry = new RetryPolicyConfig + { + Attempts = attempts, + Delay = RetryPolicyDefaults.Delay, + Backoff = RetryPolicyDefaults.Backoff, + UseJitter = RetryPolicyDefaults.UseJitter, + MaxDelay = RetryPolicyDefaults.MaxDelay + }; + _rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; + return this; + } + + public IAfterRetryBuilder Retry( + int attempts, + TimeSpan delay, + RetryBackoffType backoff = RetryBackoffType.Exponential) + { + EnsureCommitted(); + _rule.Retry = new RetryPolicyConfig + { + Attempts = attempts, + Delay = delay, + Backoff = backoff, + UseJitter = RetryPolicyDefaults.UseJitter, + MaxDelay = RetryPolicyDefaults.MaxDelay + }; + _rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; + return this; + } + + public IAfterRetryBuilder Retry(TimeSpan[] intervals) + { + EnsureCommitted(); + _rule.Retry = new RetryPolicyConfig + { + Intervals = intervals, + Attempts = intervals.Length + }; + _rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; + return this; + } + + public IAfterRedeliveryBuilder Redeliver() + { + EnsureCommitted(); + _rule.Retry = new RetryPolicyConfig { Enabled = false }; + _rule.Redelivery = new RedeliveryPolicyConfig { - ExceptionType = typeof(TException), - Predicate = wrappedPredicate, - Action = ExceptionAction.Ignore - }); + Intervals = RedeliveryPolicyDefaults.Intervals, + Attempts = RedeliveryPolicyDefaults.Intervals.Length, + UseJitter = RedeliveryPolicyDefaults.UseJitter, + MaxDelay = RedeliveryPolicyDefaults.MaxDelay + }; + return this; } + + public IAfterRedeliveryBuilder Redeliver(int attempts, TimeSpan baseDelay) + { + EnsureCommitted(); + _rule.Retry = new RetryPolicyConfig { Enabled = false }; + _rule.Redelivery = new RedeliveryPolicyConfig + { + Attempts = attempts, + BaseDelay = baseDelay, + UseJitter = RedeliveryPolicyDefaults.UseJitter, + MaxDelay = RedeliveryPolicyDefaults.MaxDelay + }; + return this; + } + + public IAfterRedeliveryBuilder Redeliver(TimeSpan[] intervals) + { + EnsureCommitted(); + _rule.Retry = new RetryPolicyConfig { Enabled = false }; + _rule.Redelivery = new RedeliveryPolicyConfig + { + Intervals = intervals, + Attempts = intervals.Length, + UseJitter = RedeliveryPolicyDefaults.UseJitter, + MaxDelay = RedeliveryPolicyDefaults.MaxDelay + }; + return this; + } + + // IAfterRetryBuilder + + public IAfterRedeliveryBuilder ThenRedeliver() + { + _rule.Redelivery = new RedeliveryPolicyConfig + { + Intervals = RedeliveryPolicyDefaults.Intervals, + Attempts = RedeliveryPolicyDefaults.Intervals.Length, + UseJitter = RedeliveryPolicyDefaults.UseJitter, + MaxDelay = RedeliveryPolicyDefaults.MaxDelay + }; + return this; + } + + public IAfterRedeliveryBuilder ThenRedeliver(int attempts, TimeSpan baseDelay) + { + _rule.Redelivery = new RedeliveryPolicyConfig + { + Attempts = attempts, + BaseDelay = baseDelay, + UseJitter = RedeliveryPolicyDefaults.UseJitter, + MaxDelay = RedeliveryPolicyDefaults.MaxDelay + }; + return this; + } + + public IAfterRedeliveryBuilder ThenRedeliver(TimeSpan[] intervals) + { + _rule.Redelivery = new RedeliveryPolicyConfig + { + Intervals = intervals, + Attempts = intervals.Length, + UseJitter = RedeliveryPolicyDefaults.UseJitter, + MaxDelay = RedeliveryPolicyDefaults.MaxDelay + }; + return this; + } + + public void ThenDeadLetter() + { + _rule.Terminal = TerminalAction.DeadLetter; + } + + // IAfterRedeliveryBuilder.ThenDeadLetter() is the same method } diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyConfigurationExtensions.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyConfigurationExtensions.cs new file mode 100644 index 00000000000..f04f5d4bfc4 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyConfigurationExtensions.cs @@ -0,0 +1,122 @@ +using Mocha.Features; + +namespace Mocha; + +/// +/// Provides extension methods for configuring exception policies including retry, redelivery, +/// and per-exception rules on message bus builders, host builders, descriptors, and consumers. +/// +public static class ExceptionPolicyConfigurationExtensions +{ + /// + /// Adds exception policy configuration to the message bus with default settings. + /// Registers a catch-all rule for with default retry and redelivery. + /// + /// The message bus builder. + /// The builder for method chaining. + public static IMessageBusBuilder AddExceptionPolicy(this IMessageBusBuilder builder) + { + builder.ConfigureFeature(f => f.GetOrSet() + .Configure(p => p.Default().Retry().ThenRedeliver())); + return builder; + } + + /// + /// Adds exception policy configuration to the message bus. + /// + /// The message bus builder. + /// The action to configure exception policy options. + /// The builder for method chaining. + public static IMessageBusBuilder AddExceptionPolicy( + this IMessageBusBuilder builder, + Action configure) + { + builder.ConfigureFeature(f => f.GetOrSet().Configure(configure)); + return builder; + } + + /// + /// Adds exception policy configuration to the host-level message bus with default settings. + /// + /// The host builder. + /// The builder for method chaining. + public static IMessageBusHostBuilder AddExceptionPolicy(this IMessageBusHostBuilder builder) + { + builder.ConfigureMessageBus(x => x.AddExceptionPolicy()); + return builder; + } + + /// + /// Adds exception policy configuration to the host-level message bus. + /// + /// The host builder. + /// The action to configure exception policy options. + /// The builder for method chaining. + public static IMessageBusHostBuilder AddExceptionPolicy( + this IMessageBusHostBuilder builder, + Action configure) + { + builder.ConfigureMessageBus(x => x.AddExceptionPolicy(configure)); + return builder; + } + + /// + /// Adds exception policy configuration to a specific descriptor (e.g., receive endpoint or transport) + /// with default settings. + /// Registers a catch-all rule for with default retry and redelivery. + /// + /// The descriptor type that supports receive middleware. + /// The descriptor to configure. + /// The descriptor for method chaining. + public static TDescriptor AddExceptionPolicy( + this TDescriptor descriptor) + where TDescriptor : IReceiveMiddlewareProvider + { + descriptor.Extend().Configuration.Features.GetOrSet() + .Configure(p => p.Default().Retry().ThenRedeliver()); + return descriptor; + } + + /// + /// Adds exception policy configuration to a specific descriptor (e.g., receive endpoint or transport). + /// + /// The descriptor type that supports receive middleware. + /// The descriptor to configure. + /// The action to configure exception policy options. + /// The descriptor for method chaining. + public static TDescriptor AddExceptionPolicy( + this TDescriptor descriptor, + Action configure) + where TDescriptor : IReceiveMiddlewareProvider + { + descriptor.Extend().Configuration.Features.GetOrSet().Configure(configure); + return descriptor; + } + + /// + /// Adds exception policy configuration to a specific consumer with default settings. + /// Registers a catch-all rule for with default retry and redelivery. + /// + /// The consumer descriptor to configure. + /// The descriptor for method chaining. + public static IConsumerDescriptor AddExceptionPolicy(this IConsumerDescriptor descriptor) + { + descriptor.Extend().Configuration.Features.GetOrSet() + .Configure(p => p.Default().Retry().ThenRedeliver()); + return descriptor; + } + + /// + /// Adds exception policy configuration to a specific consumer. + /// + /// The consumer descriptor to configure. + /// The action to configure exception policy options. + /// The descriptor for method chaining. + public static IConsumerDescriptor AddExceptionPolicy( + this IConsumerDescriptor descriptor, + Action configure) + { + descriptor.Extend().Configuration.Features.GetOrSet().Configure(configure); + return descriptor; + } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyFeature.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyFeature.cs new file mode 100644 index 00000000000..5c84cedd03e --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyFeature.cs @@ -0,0 +1,40 @@ +using Mocha.Features; + +namespace Mocha; + +/// +/// A feature that exposes the per-exception policy configuration. +/// +public sealed class ExceptionPolicyFeature : ISealable +{ + private readonly List _rules = []; + + /// + public bool IsReadOnly { get; private set; } + + /// + /// Gets the configured exception policy rules. + /// + public IReadOnlyList Rules => _rules; + + /// + public void Seal() + { + IsReadOnly = true; + } + + /// + /// Applies configuration to the exception policy options. + /// + /// An action that modifies the exception policy options. + /// Thrown if the feature has been sealed. + public void Configure(Action configure) + { + if (IsReadOnly) + { + throw ThrowHelper.FeatureIsReadOnly(); + } + + configure(new ExceptionPolicyOptions(_rules)); + } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRuleMatcher.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyMatcher.cs similarity index 60% rename from src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRuleMatcher.cs rename to src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyMatcher.cs index 955ed57b36e..22b7a4a4181 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRuleMatcher.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyMatcher.cs @@ -1,21 +1,21 @@ namespace Mocha; /// -/// Evaluates exception rules to determine if an exception should be ignored. -/// Most-specific-type-wins: if both DbException.Ignore() and NpgsqlException rules exist, +/// Evaluates exception policy rules to find the best matching rule for an exception. +/// Most-specific-type-wins: if both DbException and NpgsqlException rules exist, /// NpgsqlException rule takes priority for NpgsqlException instances. /// -internal static class ExceptionRuleMatcher +internal static class ExceptionPolicyMatcher { /// - /// Determines whether the given exception should be ignored based on the configured rules. + /// Finds the best matching exception policy rule for the given exception. /// - /// The list of exception rules to evaluate. + /// The list of exception policy rules to evaluate. /// The exception to match against. - /// true if the exception should be ignored; otherwise, false. - public static bool ShouldIgnore(IReadOnlyList rules, Exception exception) + /// The best matching rule, or null if no rule matches. + public static ExceptionPolicyRule? Match(IReadOnlyList rules, Exception exception) { - ExceptionRule? bestMatch = null; + ExceptionPolicyRule? bestMatch = null; var bestDepth = int.MaxValue; foreach (var rule in rules) @@ -47,6 +47,6 @@ public static bool ShouldIgnore(IReadOnlyList rules, Exception ex } } - return bestMatch?.Action == ExceptionAction.Ignore; + return bestMatch; } } diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyOptions.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyOptions.cs new file mode 100644 index 00000000000..2ded5c8d484 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyOptions.cs @@ -0,0 +1,53 @@ +namespace Mocha; + +/// +/// Options for configuring exception policies with per-exception rules. +/// +public class ExceptionPolicyOptions +{ + private readonly List _rules; + + /// + /// Initializes a new instance of the class. + /// + /// The shared list of exception policy rules to populate. + public ExceptionPolicyOptions(List rules) + { + _rules = rules; + } + + /// + /// Configures the default behavior for all exceptions that don't match a more specific rule. + /// Equivalent to On<Exception>(). + /// + /// A builder for configuring the default exception behavior. + public IExceptionPolicyBuilder Default() => On(null); + + /// + /// Configures behavior for a specific exception type. + /// + /// The exception type to configure. + /// A builder for configuring the exception behavior. + public IExceptionPolicyBuilder On() where TException : Exception + => On(null); + + /// + /// Configures behavior for a specific exception type matching a predicate. + /// + /// The exception type to configure. + /// An optional predicate to further filter the exception. + /// A builder for configuring the exception behavior. + public IExceptionPolicyBuilder On(Func? predicate) + where TException : Exception + { + Func? wrappedPredicate = predicate is not null + ? ex => ex is TException typed && predicate(typed) + : null; + var rule = new ExceptionPolicyRule + { + ExceptionType = typeof(TException), + Predicate = wrappedPredicate + }; + return new ExceptionPolicyBuilder(rule, _rules); + } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyRule.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyRule.cs new file mode 100644 index 00000000000..3e50fbff1ce --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyRule.cs @@ -0,0 +1,48 @@ +namespace Mocha; + +/// +/// Represents a single per-exception policy rule with optional retry, redelivery, and terminal actions. +/// +public sealed class ExceptionPolicyRule +{ + /// + /// Gets the exception type this rule applies to. + /// + public required Type ExceptionType { get; init; } + + /// + /// Gets the optional predicate to further filter the exception. + /// + public required Func? Predicate { get; init; } + + /// + /// Gets or sets the retry configuration for this exception. + /// + public RetryPolicyConfig? Retry { get; set; } + + /// + /// Gets or sets the redelivery configuration for this exception. + /// + public RedeliveryPolicyConfig? Redelivery { get; set; } + + /// + /// Gets or sets the terminal action for this exception. + /// + public TerminalAction? Terminal { get; set; } +} + +/// +/// Specifies the terminal action to take for an exception. +/// +public enum TerminalAction +{ + /// + /// Routes the message to the error endpoint. + /// + DeadLetter, + + /// + /// Swallows the exception; the message disappears. + /// + Discard +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRule.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRule.cs deleted file mode 100644 index 38587bf61b7..00000000000 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionRule.cs +++ /dev/null @@ -1,33 +0,0 @@ -namespace Mocha; - -/// -/// Internal representation of an exception filtering rule. -/// -internal sealed class ExceptionRule -{ - /// - /// Gets the exception type this rule applies to. - /// - public required Type ExceptionType { get; init; } - - /// - /// Gets the optional predicate that further filters the exception. - /// - public required Func? Predicate { get; init; } - - /// - /// Gets the action to take when this rule matches. - /// - public required ExceptionAction Action { get; init; } -} - -/// -/// Actions that can be taken for a matched exception rule. -/// -internal enum ExceptionAction -{ - /// - /// Don't retry/redeliver this exception. - /// - Ignore -} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/IAfterRedeliveryBuilder.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/IAfterRedeliveryBuilder.cs new file mode 100644 index 00000000000..ed037b58212 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/IAfterRedeliveryBuilder.cs @@ -0,0 +1,13 @@ +namespace Mocha; + +/// +/// Builder for chaining actions after redelivery configuration. +/// If nothing is chained, the default behavior (dead-letter on exhaustion) applies. +/// +public interface IAfterRedeliveryBuilder +{ + /// + /// Routes the message to the error endpoint after redelivery exhaustion. + /// + void ThenDeadLetter(); +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/IAfterRetryBuilder.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/IAfterRetryBuilder.cs new file mode 100644 index 00000000000..7fc4659c43a --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/IAfterRetryBuilder.cs @@ -0,0 +1,34 @@ +namespace Mocha; + +/// +/// Builder for chaining actions after retry configuration. +/// If nothing is chained, retry-only behavior applies and redelivery is skipped. +/// +public interface IAfterRetryBuilder +{ + /// + /// Chains redelivery after retry exhaustion with default settings. + /// + /// A builder for chaining additional actions after redelivery. + IAfterRedeliveryBuilder ThenRedeliver(); + + /// + /// Chains redelivery after retry exhaustion with the specified attempts and base delay. + /// + /// The number of redelivery attempts. + /// The base delay for redelivery. + /// A builder for chaining additional actions after redelivery. + IAfterRedeliveryBuilder ThenRedeliver(int attempts, TimeSpan baseDelay); + + /// + /// Chains redelivery after retry exhaustion with explicit intervals. + /// + /// The explicit intervals between redeliveries. + /// A builder for chaining additional actions after redelivery. + IAfterRedeliveryBuilder ThenRedeliver(TimeSpan[] intervals); + + /// + /// Routes the message to the error endpoint after retry exhaustion. + /// + void ThenDeadLetter(); +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/IExceptionPolicyBuilder.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/IExceptionPolicyBuilder.cs new file mode 100644 index 00000000000..d20a8996a96 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/IExceptionPolicyBuilder.cs @@ -0,0 +1,73 @@ +namespace Mocha; + +/// +/// Fluent builder for configuring per-exception retry/redelivery behavior. +/// +/// The exception type to configure behavior for. +public interface IExceptionPolicyBuilder where TException : Exception +{ + /// + /// Discards the message when this exception occurs. The exception is swallowed. + /// + void Discard(); + + /// + /// Routes the message to the error endpoint when this exception occurs. + /// Retry and redelivery are skipped. + /// + void DeadLetter(); + + /// + /// Retries the handler invocation with default settings. + /// Redelivery is disabled unless chained with . + /// + /// A builder for chaining additional actions after retry. + IAfterRetryBuilder Retry(); + + /// + /// Retries the handler invocation with the specified number of attempts. + /// Redelivery is disabled unless chained with . + /// + /// The number of retry attempts. + /// A builder for chaining additional actions after retry. + IAfterRetryBuilder Retry(int attempts); + + /// + /// Retries the handler invocation with full configuration. + /// Redelivery is disabled unless chained with . + /// + /// The number of retry attempts. + /// The base delay between retries. + /// The backoff strategy. + /// A builder for chaining additional actions after retry. + IAfterRetryBuilder Retry(int attempts, TimeSpan delay, RetryBackoffType backoff = RetryBackoffType.Exponential); + + /// + /// Retries the handler invocation with explicit intervals. + /// Redelivery is disabled unless chained with . + /// + /// The explicit intervals between retries. + /// A builder for chaining additional actions after retry. + IAfterRetryBuilder Retry(TimeSpan[] intervals); + + /// + /// Redelivers the message with default settings, skipping retry. + /// + /// A builder for chaining additional actions after redelivery. + IAfterRedeliveryBuilder Redeliver(); + + /// + /// Redelivers the message with the specified attempts and base delay, skipping retry. + /// + /// The number of redelivery attempts. + /// The base delay for redelivery. + /// A builder for chaining additional actions after redelivery. + IAfterRedeliveryBuilder Redeliver(int attempts, TimeSpan baseDelay); + + /// + /// Redelivers the message with explicit intervals, skipping retry. + /// + /// The explicit intervals between redeliveries. + /// A builder for chaining additional actions after redelivery. + IAfterRedeliveryBuilder Redeliver(TimeSpan[] intervals); +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyConfig.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyConfig.cs new file mode 100644 index 00000000000..746ee8e4ae3 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyConfig.cs @@ -0,0 +1,37 @@ +namespace Mocha; + +/// +/// Per-exception redelivery configuration overrides. +/// +public sealed class RedeliveryPolicyConfig +{ + /// + /// Gets whether redelivery is enabled for this exception. Defaults to true. + /// + public bool Enabled { get; init; } = true; + + /// + /// Gets the number of redelivery attempts, or null to use global defaults. + /// + public int? Attempts { get; init; } + + /// + /// Gets the base delay for redelivery, or null to use global defaults. + /// + public TimeSpan? BaseDelay { get; init; } + + /// + /// Gets the maximum delay cap, or null to use global defaults. + /// + public TimeSpan? MaxDelay { get; init; } + + /// + /// Gets whether jitter is enabled, or null to use global defaults. + /// + public bool? UseJitter { get; init; } + + /// + /// Gets the explicit redelivery intervals, or null to use global defaults. + /// + public TimeSpan[]? Intervals { get; init; } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyDefaults.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyDefaults.cs new file mode 100644 index 00000000000..3a7a943f979 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyDefaults.cs @@ -0,0 +1,27 @@ +namespace Mocha; + +/// +/// Provides the built-in default values for redelivery policy configuration. +/// +internal static class RedeliveryPolicyDefaults +{ + /// + /// The default redelivery intervals. Value: 5min, 15min, 30min. + /// + public static readonly TimeSpan[] Intervals = + [ + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(15), + TimeSpan.FromMinutes(30) + ]; + + /// + /// The default jitter setting. Value: true. + /// + public const bool UseJitter = true; + + /// + /// The default maximum delay cap. Value: 1 hour. + /// + public static readonly TimeSpan MaxDelay = TimeSpan.FromHours(1); +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryBackoffType.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryBackoffType.cs index 0fe169a38b5..d670e13b5cc 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryBackoffType.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryBackoffType.cs @@ -6,17 +6,17 @@ namespace Mocha; public enum RetryBackoffType { /// - /// Constant delay between retries. Every attempt waits the same . + /// Constant delay between retries. Every attempt waits the same base delay. /// Constant, /// - /// Linearly increasing delay. Delay = * (attempt + 1). + /// Linearly increasing delay. Delay = baseDelay * attempt. /// Linear, /// - /// Exponentially increasing delay. Delay = * 2^attempt. + /// Exponentially increasing delay. Delay = baseDelay * 2^(attempt-1). /// Exponential } diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryConfigurationExtensions.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryConfigurationExtensions.cs deleted file mode 100644 index 26c2e465252..00000000000 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryConfigurationExtensions.cs +++ /dev/null @@ -1,90 +0,0 @@ -using Mocha.Features; - -namespace Mocha; - -/// -/// Provides extension methods for configuring the retry middleware on message bus builders and consumer descriptors. -/// -public static class RetryConfigurationExtensions -{ - /// - /// Adds retry to the message bus consumer pipeline with default settings. - /// - /// The message bus builder. - /// The builder for method chaining. - public static IMessageBusBuilder AddRetry( - this IMessageBusBuilder builder) - { - builder.ConfigureFeature(f => f.GetOrSet()); - builder.UseConsume(ConsumerMiddlewares.Retry, before: "Instrumentation"); - return builder; - } - - /// - /// Adds retry to the message bus consumer pipeline. - /// - /// The message bus builder. - /// The action to configure retry options. - /// The builder for method chaining. - public static IMessageBusBuilder AddRetry( - this IMessageBusBuilder builder, - Action configure) - { - builder.ConfigureFeature(f => f.GetOrSet().Configure(configure)); - builder.UseConsume(ConsumerMiddlewares.Retry, before: "Instrumentation"); - return builder; - } - - /// - /// Adds retry to the host-level consumer pipeline with default settings. - /// - /// The host builder. - /// The builder for method chaining. - public static IMessageBusHostBuilder AddRetry( - this IMessageBusHostBuilder builder) - { - builder.ConfigureMessageBus(x => x.AddRetry()); - return builder; - } - - /// - /// Adds retry to the host-level consumer pipeline. - /// - /// The host builder. - /// The action to configure retry options. - /// The builder for method chaining. - public static IMessageBusHostBuilder AddRetry( - this IMessageBusHostBuilder builder, - Action configure) - { - builder.ConfigureMessageBus(x => x.AddRetry(configure)); - return builder; - } - - /// - /// Adds retry configuration to a specific consumer with default settings. - /// - /// The consumer descriptor to configure. - /// The descriptor for method chaining. - public static IConsumerDescriptor AddRetry( - this IConsumerDescriptor descriptor) - { - descriptor.Extend().Configuration.Features.GetOrSet(); - return descriptor; - } - - /// - /// Adds retry configuration to a specific consumer. - /// - /// The consumer descriptor to configure. - /// The action to configure retry options. - /// The descriptor for method chaining. - public static IConsumerDescriptor AddRetry( - this IConsumerDescriptor descriptor, - Action configure) - { - descriptor.Extend().Configuration.Features - .GetOrSet().Configure(configure); - return descriptor; - } -} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryFeature.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryFeature.cs deleted file mode 100644 index 690d675ab0d..00000000000 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryFeature.cs +++ /dev/null @@ -1,75 +0,0 @@ -using Mocha.Features; - -namespace Mocha; - -/// -/// A feature that exposes the retry configuration for a consumer. -/// -public sealed class RetryFeature : ISealable -{ - private readonly RetryOptions _options = new(); - - /// - public bool IsReadOnly { get; private set; } - - /// - /// Gets whether retry is enabled, or null if not configured. - /// - public bool? Enabled => _options.Enabled; - - /// - /// Gets the maximum retry attempts, or null if not configured. - /// - public int? MaxRetryAttempts => _options.MaxRetryAttempts; - - /// - /// Gets the base delay between retries, or null if not configured. - /// - public TimeSpan? Delay => _options.Delay; - - /// - /// Gets the maximum delay cap, or null if not configured. - /// - public TimeSpan? MaxDelay => _options.MaxDelay; - - /// - /// Gets the backoff strategy, or null if not configured. - /// - public RetryBackoffType? BackoffType => _options.BackoffType; - - /// - /// Gets whether jitter is enabled, or null if not configured. - /// - public bool? UseJitter => _options.UseJitter; - - /// - /// Gets the explicit retry intervals, or null if not configured. - /// - public TimeSpan[]? Intervals => _options.Intervals; - - /// - /// Gets the exception rules configured for this feature. - /// - internal IReadOnlyList ExceptionRules => _options.ExceptionRules; - - /// - public void Seal() - { - IsReadOnly = true; - } - - /// - /// Applies configuration to the retry options. - /// - /// An action that modifies the retry options. - /// Thrown if the feature has been sealed. - public void Configure(Action configure) - { - if (IsReadOnly) - { - throw ThrowHelper.FeatureIsReadOnly(); - } - - configure(_options); - } -} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryOptions.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryOptions.cs deleted file mode 100644 index 9dc179c58c1..00000000000 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryOptions.cs +++ /dev/null @@ -1,102 +0,0 @@ -namespace Mocha; - -/// -/// Options for configuring the retry middleware that retries failed message handler invocations -/// with configurable backoff strategies. -/// -public class RetryOptions -{ - /// - /// Gets or sets whether retry is enabled. Null inherits from parent scope; defaults to true. - /// - public bool? Enabled { get; set; } - - /// - /// Gets or sets the maximum retry attempts (not counting the initial attempt). - /// - public int? MaxRetryAttempts { get; set; } - - /// - /// Gets or sets the base delay between retries. Interpretation depends on . - /// - public TimeSpan? Delay { get; set; } - - /// - /// Gets or sets the maximum delay cap. Prevents exponential backoff from growing unbounded. - /// - public TimeSpan? MaxDelay { get; set; } - - /// - /// Gets or sets the backoff strategy: Constant, Linear, or Exponential. - /// - public RetryBackoffType? BackoffType { get; set; } - - /// - /// Gets or sets whether to add jitter to delay calculations. - /// - public bool? UseJitter { get; set; } - - /// - /// Gets or sets explicit retry intervals. When set, overrides , - /// , and . - /// The number of elements determines the number of retries. - /// - public TimeSpan[]? Intervals { get; set; } - - private List? _exceptionRules; - - internal IReadOnlyList ExceptionRules => _exceptionRules ?? (IReadOnlyList)[]; - - /// - /// Configures behavior for a specific exception type. - /// - /// The exception type to configure. - /// A builder for configuring the exception behavior. - public ExceptionPolicyBuilder On() where TException : Exception - => On(null); - - /// - /// Configures behavior for a specific exception type matching a predicate. - /// - /// The exception type to configure. - /// An optional predicate to further filter the exception. - /// A builder for configuring the exception behavior. - public ExceptionPolicyBuilder On(Func? predicate) - where TException : Exception - { - _exceptionRules ??= []; - var builder = new ExceptionPolicyBuilder(_exceptionRules, predicate); - return builder; - } - - /// - /// Provides the default values for retry options. - /// - public static class Defaults - { - /// - /// The default maximum retry attempts. - /// - public static int MaxRetryAttempts = 3; - - /// - /// The default base delay between retries. - /// - public static TimeSpan Delay = TimeSpan.FromMilliseconds(200); - - /// - /// The default maximum delay cap. - /// - public static TimeSpan MaxDelay = TimeSpan.FromSeconds(30); - - /// - /// The default backoff strategy. - /// - public static RetryBackoffType BackoffType = RetryBackoffType.Exponential; - - /// - /// The default jitter setting. - /// - public static bool UseJitter = true; - } -} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyConfig.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyConfig.cs new file mode 100644 index 00000000000..0414ec5b6bb --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyConfig.cs @@ -0,0 +1,42 @@ +namespace Mocha; + +/// +/// Per-exception retry configuration. +/// +public sealed class RetryPolicyConfig +{ + /// + /// Gets whether retry is enabled for this exception. Defaults to true. + /// + public bool Enabled { get; init; } = true; + + /// + /// Gets the number of retry attempts. + /// + public int? Attempts { get; init; } + + /// + /// Gets the base delay between retries. + /// + public TimeSpan? Delay { get; init; } + + /// + /// Gets the backoff strategy. + /// + public RetryBackoffType? Backoff { get; init; } + + /// + /// Gets the maximum delay cap. + /// + public TimeSpan? MaxDelay { get; init; } + + /// + /// Gets whether jitter is enabled. + /// + public bool? UseJitter { get; init; } + + /// + /// Gets the explicit retry intervals. + /// + public TimeSpan[]? Intervals { get; init; } +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyDefaults.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyDefaults.cs new file mode 100644 index 00000000000..91582f0da18 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyDefaults.cs @@ -0,0 +1,32 @@ +namespace Mocha; + +/// +/// Provides the built-in default values for retry policy configuration. +/// +internal static class RetryPolicyDefaults +{ + /// + /// The default number of retry attempts. Value: 3. + /// + public const int Attempts = 3; + + /// + /// The default base delay between retries. Value: 200ms. + /// + public static readonly TimeSpan Delay = TimeSpan.FromMilliseconds(200); + + /// + /// The default backoff strategy. Value: . + /// + public const RetryBackoffType Backoff = RetryBackoffType.Exponential; + + /// + /// The default jitter setting. Value: true. + /// + public const bool UseJitter = true; + + /// + /// The default maximum delay cap. Value: 30 seconds. + /// + public static readonly TimeSpan MaxDelay = TimeSpan.FromSeconds(30); +} diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs index 84f0fa5ee0c..fb60e9a78e5 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs @@ -3,7 +3,7 @@ namespace Mocha; /// /// Provides retry state information to message handlers via /// context.Features.Get<RetryState>(). -/// Null if AddRetry is not configured. +/// Null if retry is not configured via AddExceptionPolicy. /// public sealed class RetryRuntimeFeature { diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs index bc26d6f41f4..22d6074442f 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs @@ -15,12 +15,7 @@ namespace Mocha; /// because the caller would time out waiting for a response. /// internal sealed class ReceiveRedeliveryMiddleware( - int maxAttempts, - TimeSpan[]? intervals, - TimeSpan resolvedBaseDelay, - TimeSpan resolvedMaxDelay, - bool resolvedUseJitter, - IReadOnlyList exceptionRules, + IReadOnlyList exceptionPolicyRules, TimeProvider timeProvider, IMessagingPools pools) { @@ -51,20 +46,54 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next throw; } - // Check exception rules: if the exception is explicitly ignored, rethrow. - if (exceptionRules.Count > 0 && ExceptionRuleMatcher.ShouldIgnore(exceptionRules, ex)) + // Match exception against policy rules. + var rule = ExceptionPolicyMatcher.Match(exceptionPolicyRules, ex); + + // No matching rule — no policy for this exception, let it propagate. + if (rule is null) + { + throw; + } + + // Discard: swallow at receive level. + if (rule.Terminal == TerminalAction.Discard) + { + var consumerFeature = context.Features.Get(); + + if (consumerFeature is not null) + { + consumerFeature.MessageConsumed = true; + } + + return; + } + + // DeadLetter: skip redelivery, let fault middleware handle. + if (rule.Terminal == TerminalAction.DeadLetter) { throw; } + // No redelivery configured for this rule, or redelivery explicitly disabled. + if (rule.Redelivery is null or { Enabled: false }) + { + throw; + } + + var redeliveryConfig = rule.Redelivery; + // Check if redelivery attempts remain. + var maxAttempts = redeliveryConfig.Attempts + ?? redeliveryConfig.Intervals?.Length + ?? 0; + if (delayedRetryCount >= maxAttempts) { throw; } // Calculate the delay for this redelivery attempt. - var delay = CalculateDelay(delayedRetryCount); + var delay = CalculateDelay(delayedRetryCount, redeliveryConfig); var scheduledTime = timeProvider.GetUtcNow().Add(delay); // Update the header on the envelope so the next delivery round sees the incremented count. @@ -75,7 +104,13 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next throw; } - envelope.Headers?.Set(MessageHeaders.Retry.DelayedRetryCount.Key, delayedRetryCount + 1); + if (envelope.Headers is null) + { + throw new InvalidOperationException( + "Cannot increment delayed retry count because the envelope has no headers collection."); + } + + envelope.Headers.Set(MessageHeaders.Retry.DelayedRetryCount.Key, delayedRetryCount + 1); // Dispatch the envelope back to the same endpoint with the scheduled time. // Use the Source address (queue/topic) rather than the endpoint address, because @@ -107,11 +142,11 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next } } - private TimeSpan CalculateDelay(int attempt) + private static TimeSpan CalculateDelay(int attempt, RedeliveryPolicyConfig config) { TimeSpan baseDelay; - if (intervals is { Length: > 0 }) + if (config.Intervals is { Length: > 0 } intervals) { // Explicit intervals: use array index, clamp to last. baseDelay = intervals[Math.Min(attempt, intervals.Length - 1)]; @@ -119,17 +154,22 @@ private TimeSpan CalculateDelay(int attempt) else { // Calculated: BaseDelay * (attempt + 1). - baseDelay = resolvedBaseDelay * (attempt + 1); + var configuredBaseDelay = config.BaseDelay ?? RedeliveryPolicyDefaults.Intervals[0]; + baseDelay = configuredBaseDelay * (attempt + 1); } // Cap by MaxDelay. - if (baseDelay > resolvedMaxDelay) + var maxDelay = config.MaxDelay ?? RedeliveryPolicyDefaults.MaxDelay; + + if (baseDelay > maxDelay) { - baseDelay = resolvedMaxDelay; + baseDelay = maxDelay; } // Add jitter: +/- 25%. - if (resolvedUseJitter) + var useJitter = config.UseJitter ?? RedeliveryPolicyDefaults.UseJitter; + + if (useJitter) { var jitterRange = baseDelay.TotalMilliseconds * 0.25; var jitter = (Random.Shared.NextDouble() * 2 - 1) * jitterRange; @@ -143,105 +183,54 @@ public static ReceiveMiddlewareConfiguration Create() => new( static (context, next) => { - // Feature values are resolved from endpoint -> transport -> bus to support overrides. - var enabled = context.GetConfiguration(f => f.Enabled) ?? true; - - if (!enabled) - { - return next; - } + // Resolve exception policy feature from the most specific scope. + var feature = context.GetExceptionPolicyFeature(); - var intervals = context.GetConfiguration(f => f.Intervals) - ?? RedeliveryOptions.Defaults.Intervals; - - var maxAttempts = intervals is { Length: > 0 } - ? intervals.Length - : context.GetConfiguration(f => f.MaxAttempts) ?? 0; - - if (maxAttempts <= 0 && intervals is not { Length: > 0 }) + if (feature is null) { + // No exception policy configured — skip redelivery middleware entirely. return next; } - var baseDelay = context.GetConfiguration(f => f.BaseDelay) - ?? TimeSpan.FromMinutes(5); - - var maxDelay = context.GetConfiguration(f => f.MaxDelay) - ?? RedeliveryOptions.Defaults.MaxDelay; - - var useJitter = context.GetConfiguration(f => f.UseJitter) - ?? RedeliveryOptions.Defaults.UseJitter; - - // Resolve exception rules (atomically from first scope that has them). - var exceptionRules = ResolveExceptionRules(context); - var timeProvider = context.Services.GetRequiredService(); var pools = context.Services.GetRequiredService(); var middleware = new ReceiveRedeliveryMiddleware( - maxAttempts, - intervals, - baseDelay, - maxDelay, - useJitter, - exceptionRules, + feature.Rules, timeProvider, pools); return ctx => middleware.InvokeAsync(ctx, next); }, "Redelivery"); - - private static IReadOnlyList ResolveExceptionRules(ReceiveMiddlewareFactoryContext context) - { - var busFeatures = context.Services.GetRequiredService(); - - // Endpoint rules take precedence if present. - if (context.Endpoint.Features.TryGet(out RedeliveryFeature? endpointFeature) - && endpointFeature.ExceptionRules.Count > 0) - { - return endpointFeature.ExceptionRules; - } - - if (context.Transport.Features.TryGet(out RedeliveryFeature? transportFeature) - && transportFeature.ExceptionRules.Count > 0) - { - return transportFeature.ExceptionRules; - } - - if (busFeatures.TryGet(out RedeliveryFeature? busFeature) - && busFeature.ExceptionRules.Count > 0) - { - return busFeature.ExceptionRules; - } - - return []; - } } file static class Extensions { /// - /// Resolves configuration with the most specific scope taking precedence. + /// Resolves exception policy feature with the most specific scope taking precedence. + /// Endpoint -> Transport -> Bus. /// - public static T? GetConfiguration( - this ReceiveMiddlewareFactoryContext context, - Func selector) + public static ExceptionPolicyFeature? GetExceptionPolicyFeature(this ReceiveMiddlewareFactoryContext context) { var busFeatures = context.Services.GetRequiredService(); - return context.Endpoint.Features.GetFeatureValue(selector) - ?? context.Transport.Features.GetFeatureValue(selector) - ?? busFeatures.GetFeatureValue(selector); - } + // Endpoint -> Transport -> Bus (most specific first). + if (context.Endpoint.Features.TryGet(out ExceptionPolicyFeature? endpointFeature)) + { + return endpointFeature; + } - private static T? GetFeatureValue(this IFeatureCollection features, Func selector) - { - if (features.TryGet(out RedeliveryFeature? feature)) + if (context.Transport.Features.TryGet(out ExceptionPolicyFeature? transportFeature)) + { + return transportFeature; + } + + if (busFeatures.TryGet(out ExceptionPolicyFeature? busFeature)) { - return selector(feature); + return busFeature; } - return default; + return null; } } diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryConfigurationExtensions.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryConfigurationExtensions.cs deleted file mode 100644 index 897e914260a..00000000000 --- a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryConfigurationExtensions.cs +++ /dev/null @@ -1,93 +0,0 @@ -using Mocha.Features; - -namespace Mocha; - -/// -/// Provides extension methods for configuring the redelivery middleware on message bus builders and descriptors. -/// -public static class RedeliveryConfigurationExtensions -{ - /// - /// Adds redelivery to the message bus receive pipeline with default settings. - /// - /// The message bus builder. - /// The builder for method chaining. - public static IMessageBusBuilder AddRedelivery( - this IMessageBusBuilder builder) - { - builder.ConfigureFeature(f => f.GetOrSet()); - builder.UseReceive(ReceiveMiddlewares.Redelivery, after: "Fault"); - return builder; - } - - /// - /// Adds redelivery to the message bus receive pipeline. - /// - /// The message bus builder. - /// The action to configure redelivery options. - /// The builder for method chaining. - public static IMessageBusBuilder AddRedelivery( - this IMessageBusBuilder builder, - Action configure) - { - builder.ConfigureFeature(f => f.GetOrSet().Configure(configure)); - builder.UseReceive(ReceiveMiddlewares.Redelivery, after: "Fault"); - return builder; - } - - /// - /// Adds redelivery to the host-level receive pipeline with default settings. - /// - /// The host builder. - /// The builder for method chaining. - public static IMessageBusHostBuilder AddRedelivery( - this IMessageBusHostBuilder builder) - { - builder.ConfigureMessageBus(x => x.AddRedelivery()); - return builder; - } - - /// - /// Adds redelivery to the host-level receive pipeline. - /// - /// The host builder. - /// The action to configure redelivery options. - /// The builder for method chaining. - public static IMessageBusHostBuilder AddRedelivery( - this IMessageBusHostBuilder builder, - Action configure) - { - builder.ConfigureMessageBus(x => x.AddRedelivery(configure)); - return builder; - } - - /// - /// Adds redelivery configuration to a specific descriptor (e.g., receive endpoint or transport) with default settings. - /// - /// The descriptor type that supports receive middleware. - /// The descriptor to configure. - /// The descriptor for method chaining. - public static TDescriptor AddRedelivery( - this TDescriptor descriptor) - where TDescriptor : IReceiveMiddlewareProvider - { - descriptor.Extend().Configuration.Features.GetOrSet(); - return descriptor; - } - - /// - /// Adds redelivery configuration to a specific descriptor (e.g., receive endpoint or transport). - /// - /// The descriptor type that supports receive middleware. - /// The descriptor to configure. - /// The action to configure redelivery options. - /// The descriptor for method chaining. - public static TDescriptor AddRedelivery( - this TDescriptor descriptor, - Action configure) - where TDescriptor : IReceiveMiddlewareProvider - { - descriptor.Extend().Configuration.Features.GetOrSet().Configure(configure); - return descriptor; - } -} diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryFeature.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryFeature.cs deleted file mode 100644 index 96129f88db8..00000000000 --- a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryFeature.cs +++ /dev/null @@ -1,70 +0,0 @@ -using Mocha.Features; - -namespace Mocha; - -/// -/// A feature that exposes the redelivery configuration for a receive endpoint. -/// -public sealed class RedeliveryFeature : ISealable -{ - private readonly RedeliveryOptions _options = new(); - - /// - public bool IsReadOnly { get; private set; } - - /// - /// Gets whether redelivery is enabled, or null if not configured. - /// - public bool? Enabled => _options.Enabled; - - /// - /// Gets the maximum redelivery attempts, or null if not configured. - /// - public int? MaxAttempts => _options.MaxAttempts; - - /// - /// Gets the base delay for backoff calculation, or null if not configured. - /// - public TimeSpan? BaseDelay => _options.BaseDelay; - - /// - /// Gets the maximum delay cap, or null if not configured. - /// - public TimeSpan? MaxDelay => _options.MaxDelay; - - /// - /// Gets whether jitter is enabled, or null if not configured. - /// - public bool? UseJitter => _options.UseJitter; - - /// - /// Gets the explicit redelivery intervals, or null if not configured. - /// - public TimeSpan[]? Intervals => _options.Intervals; - - /// - /// Gets the exception rules configured for this feature. - /// - internal IReadOnlyList ExceptionRules => _options.ExceptionRules; - - /// - public void Seal() - { - IsReadOnly = true; - } - - /// - /// Applies configuration to the redelivery options. - /// - /// An action that modifies the redelivery options. - /// Thrown if the feature has been sealed. - public void Configure(Action configure) - { - if (IsReadOnly) - { - throw ThrowHelper.FeatureIsReadOnly(); - } - - configure(_options); - } -} diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryOptions.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryOptions.cs deleted file mode 100644 index f6a70102a88..00000000000 --- a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryOptions.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace Mocha; - -/// -/// Options for configuring the redelivery middleware that reschedules failed messages -/// for later delivery with configurable delay strategies. -/// -public class RedeliveryOptions -{ - /// - /// Gets or sets whether redelivery is enabled. Null inherits from parent scope; defaults to true. - /// - public bool? Enabled { get; set; } - - /// - /// Gets or sets the maximum number of redelivery attempts. - /// - public int? MaxAttempts { get; set; } - - /// - /// Gets or sets the base delay for backoff calculation. Actual delay = BaseDelay * (attempt + 1). - /// - public TimeSpan? BaseDelay { get; set; } - - /// - /// Gets or sets the maximum delay cap for any single redelivery delay. - /// - public TimeSpan? MaxDelay { get; set; } - - /// - /// Gets or sets whether to add jitter to delay calculations. - /// - public bool? UseJitter { get; set; } - - /// - /// Gets or sets explicit redelivery intervals. When set, overrides - /// and . The number of elements determines the number of - /// redelivery attempts. - /// - public TimeSpan[]? Intervals { get; set; } - - private List? _exceptionRules; - - internal IReadOnlyList ExceptionRules => _exceptionRules ?? (IReadOnlyList)[]; - - /// - /// Configures behavior for a specific exception type. - /// - /// The exception type to configure. - /// A builder for configuring the exception behavior. - public ExceptionPolicyBuilder On() where TException : Exception - => On(null); - - /// - /// Configures behavior for a specific exception type matching a predicate. - /// - /// The exception type to configure. - /// An optional predicate to further filter the exception. - /// A builder for configuring the exception behavior. - public ExceptionPolicyBuilder On(Func? predicate) - where TException : Exception - { - _exceptionRules ??= []; - var builder = new ExceptionPolicyBuilder(_exceptionRules, predicate); - return builder; - } - - /// - /// Provides the default values for redelivery options. - /// - public static class Defaults - { - /// - /// The default redelivery intervals: 5 minutes, 15 minutes, 30 minutes. - /// - public static TimeSpan[] Intervals = - [ - TimeSpan.FromMinutes(5), - TimeSpan.FromMinutes(15), - TimeSpan.FromMinutes(30) - ]; - - /// - /// The default maximum delay cap. - /// - public static TimeSpan MaxDelay = TimeSpan.FromHours(1); - - /// - /// The default jitter setting. - /// - public static bool UseJitter = true; - } -} diff --git a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs index 92cc7600382..efbb531042e 100644 --- a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs +++ b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs @@ -19,14 +19,14 @@ public async Task Redelivery_Should_ScheduleRedelivery_When_HandlerFails() .AddSingleton(counter) .AddSingleton(recorder) .AddMessageBus() - .AddRedelivery(redeliver => + .AddExceptionPolicy(p => { - redeliver.Intervals = + p.On().Redeliver( [ TimeSpan.FromMilliseconds(1), TimeSpan.FromMilliseconds(1), TimeSpan.FromMilliseconds(1) - ]; + ]); }) .AddEventHandler() .AddInMemory() @@ -55,10 +55,10 @@ public async Task Redelivery_Should_SkipRedelivery_When_ExceptionIsIgnored() await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddRedelivery(redeliver => + .AddExceptionPolicy(p => { - redeliver.Intervals = [TimeSpan.FromMilliseconds(1)]; - redeliver.On().Ignore(); + p.On().Redeliver([TimeSpan.FromMilliseconds(1)]); + p.On().DeadLetter(); }) .AddEventHandler() .AddInMemory() @@ -78,17 +78,12 @@ public async Task Redelivery_Should_SkipRedelivery_When_ExceptionIsIgnored() [Fact] public async Task Redelivery_Should_PassThrough_When_Disabled() { - // arrange + // arrange — no exception policy configured, so retry and redelivery are both no-ops var counter = new InvocationCounter(); await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddRedelivery(redeliver => - { - redeliver.Enabled = false; - redeliver.Intervals = [TimeSpan.FromMilliseconds(1)]; - }) .AddEventHandler() .AddInMemory() .BuildServiceProvider(); @@ -99,7 +94,7 @@ public async Task Redelivery_Should_PassThrough_When_Disabled() // act await bus.PublishAsync(new OrderCreated { OrderId = "ORD-DISABLED" }, CancellationToken.None); - // assert - only 1 invocation, redelivery disabled + // assert - no exception policy: only 1 invocation, no retry or redelivery await Task.Delay(500); Assert.Equal(1, counter.Count); } @@ -113,13 +108,13 @@ public async Task Redelivery_Should_PropagateToFault_When_AllAttemptsExhausted() await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddRedelivery(redeliver => + .AddExceptionPolicy(p => { - redeliver.Intervals = + p.On().Redeliver( [ TimeSpan.FromMilliseconds(1), TimeSpan.FromMilliseconds(1) - ]; + ]); }) .AddEventHandler() .AddInMemory() @@ -140,19 +135,21 @@ await counter.WaitForCountAsync(3, s_timeout), [Fact] public async Task Redelivery_Should_UseEndpointOverride_When_EndpointConfigured() { - // arrange - bus-level: 1 redelivery, but the endpoint overrides to disabled + // arrange - bus-level: redeliver once, but the transport overrides to discard var counter = new InvocationCounter(); var builder = new ServiceCollection() .AddSingleton(counter) .AddScoped() .AddMessageBus() - .AddRedelivery(redeliver => redeliver.Intervals = [TimeSpan.FromMilliseconds(1)]); + .AddExceptionPolicy(p => + p.On().Redeliver([TimeSpan.FromMilliseconds(1)])); - // Override at transport level to disable redelivery. + // Override at transport level to discard all exceptions. builder.ConfigureMessageBus(b => b.AddHandler()); await using var provider = await builder - .AddInMemory(t => t.AddRedelivery(redeliver => redeliver.Enabled = false)) + .AddInMemory(t => t.AddExceptionPolicy(p => + p.On().DeadLetter())) .BuildServiceProvider(); using var scope = provider.CreateScope(); @@ -161,21 +158,21 @@ public async Task Redelivery_Should_UseEndpointOverride_When_EndpointConfigured( // act await bus.PublishAsync(new OrderCreated { OrderId = "ORD-OVERRIDE" }, CancellationToken.None); - // assert - redelivery disabled at transport level: only 1 invocation + // assert - redelivery disabled at transport level via DeadLetter: only 1 invocation await Task.Delay(500); Assert.Equal(1, counter.Count); } [Fact] - public async Task Redelivery_Should_UseDefaults_When_ParameterlessAddRedelivery() + public async Task Redelivery_Should_UseDefaults_When_ParameterlessAddExceptionPolicy() { - // arrange - parameterless AddRedelivery uses defaults: 3 intervals + // arrange - defaults: 3 redelivery intervals from RedeliveryPolicyDefaults var counter = new InvocationCounter(); await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddRedelivery() + .AddExceptionPolicy() .AddEventHandler() .AddInMemory() .BuildServiceProvider(); @@ -186,10 +183,12 @@ public async Task Redelivery_Should_UseDefaults_When_ParameterlessAddRedelivery( // act await bus.PublishAsync(new OrderCreated { OrderId = "ORD-DEFAULT" }, CancellationToken.None); - // assert - default is 3 intervals: 1 original + 3 redeliveries = 4 total + // assert - default is 3 retries + 3 redeliveries: 1 original + 3 retries = 4 on first delivery, + // then 3 redeliveries each with 1 original + 3 retries = 4, total = 4 + 3*4 = 16 + // Wait for at least the first 4 (initial delivery with retries) Assert.True( await counter.WaitForCountAsync(4, s_timeout), - $"Expected 4 invocations (1 original + 3 default redeliveries), but got {counter.Count}"); + $"Expected at least 4 invocations (1 original + 3 default retries), but got {counter.Count}"); } // ============================================================ diff --git a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs index b08cac99b35..130e151cdf8 100644 --- a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs +++ b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs @@ -18,12 +18,10 @@ public async Task Retry_Should_RetryHandler_When_HandlerThrowsTransientException .AddSingleton(counter) .AddSingleton(recorder) .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(p => { - retry.MaxRetryAttempts = 3; - retry.Delay = TimeSpan.FromMilliseconds(1); - retry.BackoffType = RetryBackoffType.Constant; - retry.UseJitter = false; + p.On() + .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); }) .AddEventHandler() .AddInMemory() @@ -51,12 +49,10 @@ public async Task Retry_Should_PropagateToFault_When_AllRetriesExhausted() await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(p => { - retry.MaxRetryAttempts = 3; - retry.Delay = TimeSpan.FromMilliseconds(1); - retry.BackoffType = RetryBackoffType.Constant; - retry.UseJitter = false; + p.On() + .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); }) .AddEventHandler() .AddInMemory() @@ -82,13 +78,11 @@ public async Task Retry_Should_SkipRetry_When_ExceptionIsIgnored() await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(p => { - retry.MaxRetryAttempts = 3; - retry.Delay = TimeSpan.FromMilliseconds(1); - retry.BackoffType = RetryBackoffType.Constant; - retry.UseJitter = false; - retry.On().Ignore(); + p.On() + .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); + p.On().DeadLetter(); }) .AddEventHandler() .AddInMemory() @@ -116,13 +110,11 @@ public async Task Retry_Should_SkipRetry_When_PredicateMatchesIgnoredException() await using var matchingProvider = await new ServiceCollection() .AddSingleton(matchingCounter) .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(p => { - retry.MaxRetryAttempts = 3; - retry.Delay = TimeSpan.FromMilliseconds(1); - retry.BackoffType = RetryBackoffType.Constant; - retry.UseJitter = false; - retry.On(ex => ex.ParamName == "test").Ignore(); + p.On() + .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); + p.On(ex => ex.ParamName == "test").DeadLetter(); }) .AddEventHandler() .AddInMemory() @@ -141,13 +133,11 @@ public async Task Retry_Should_SkipRetry_When_PredicateMatchesIgnoredException() await using var nonMatchingProvider = await new ServiceCollection() .AddSingleton(nonMatchingCounter) .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(p => { - retry.MaxRetryAttempts = 3; - retry.Delay = TimeSpan.FromMilliseconds(1); - retry.BackoffType = RetryBackoffType.Constant; - retry.UseJitter = false; - retry.On(ex => ex.ParamName == "other").Ignore(); + p.On() + .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); + p.On(ex => ex.ParamName == "other").DeadLetter(); }) .AddEventHandler() .AddInMemory() @@ -173,24 +163,20 @@ public async Task Retry_Should_UseConsumerOverride_When_ConsumerHasDifferentConf .AddSingleton(counter) .AddScoped() .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(p => { - retry.MaxRetryAttempts = 2; - retry.Delay = TimeSpan.FromMilliseconds(1); - retry.BackoffType = RetryBackoffType.Constant; - retry.UseJitter = false; + p.On() + .Retry(2, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); }); builder.ConfigureMessageBus(b => { b.AddHandler(consumer => { - consumer.AddRetry(retry => + consumer.AddExceptionPolicy(p => { - retry.MaxRetryAttempts = 5; - retry.Delay = TimeSpan.FromMilliseconds(1); - retry.BackoffType = RetryBackoffType.Constant; - retry.UseJitter = false; + p.On() + .Retry(5, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); }); }); }); @@ -218,12 +204,10 @@ public async Task Retry_Should_ExposeRetryState_When_HandlerAccessesFeatures() .AddSingleton(stateCapture) .AddScoped() .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(p => { - retry.MaxRetryAttempts = 2; - retry.Delay = TimeSpan.FromMilliseconds(1); - retry.BackoffType = RetryBackoffType.Constant; - retry.UseJitter = false; + p.On() + .Retry(2, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); }); builder.ConfigureMessageBus(b => b.AddHandler()); @@ -256,17 +240,16 @@ public async Task Retry_Should_PassThrough_When_DisabledForConsumer() .AddSingleton(counter) .AddScoped() .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(p => { - retry.MaxRetryAttempts = 3; - retry.Delay = TimeSpan.FromMilliseconds(1); - retry.BackoffType = RetryBackoffType.Constant; - retry.UseJitter = false; + p.On() + .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); }); builder.ConfigureMessageBus(b => b.AddHandler(consumer => - consumer.AddRetry(retry => retry.Enabled = false))); + consumer.AddExceptionPolicy(p => + p.On().Redeliver([TimeSpan.FromHours(1)])))); await using var provider = await builder.AddInMemory().BuildServiceProvider(); @@ -276,7 +259,7 @@ public async Task Retry_Should_PassThrough_When_DisabledForConsumer() // act await bus.PublishAsync(new OrderCreated { OrderId = "ORD-DISABLED" }, CancellationToken.None); - // assert - retry disabled: only 1 invocation + // assert - retry disabled via consumer override: only 1 invocation (redeliver does not cause immediate retry) await Task.Delay(500); Assert.Equal(1, counter.Count); } @@ -289,14 +272,14 @@ public async Task Retry_Should_UseExplicitIntervals_When_IntervalsConfigured() await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(p => { - retry.Intervals = + p.On().Retry( [ TimeSpan.FromMilliseconds(10), TimeSpan.FromMilliseconds(20), TimeSpan.FromMilliseconds(30) - ]; + ]); }) .AddEventHandler() .AddInMemory() @@ -322,13 +305,11 @@ public async Task Retry_Should_RespectInheritance_When_BaseExceptionIgnored() await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(p => { - retry.MaxRetryAttempts = 3; - retry.Delay = TimeSpan.FromMilliseconds(1); - retry.BackoffType = RetryBackoffType.Constant; - retry.UseJitter = false; - retry.On().Ignore(); + p.On() + .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); + p.On().DeadLetter(); }) .AddEventHandler() .AddInMemory() @@ -346,14 +327,14 @@ public async Task Retry_Should_RespectInheritance_When_BaseExceptionIgnored() } [Fact] - public async Task Retry_Should_UseDefaults_When_ParameterlessAddRetry() + public async Task Retry_Should_UseDefaults_When_ParameterlessAddExceptionPolicy() { - // arrange - default: MaxRetryAttempts = 3 + // arrange - default: 3 retries (from RetryPolicyDefaults.Attempts) var counter = new RetryInvocationCounter(); await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddRetry() + .AddExceptionPolicy() .AddEventHandler() .AddInMemory() .BuildServiceProvider(); diff --git a/website/src/docs/mocha/v1/exception-policies.md b/website/src/docs/mocha/v1/exception-policies.md new file mode 100644 index 00000000000..b2760f87e7b --- /dev/null +++ b/website/src/docs/mocha/v1/exception-policies.md @@ -0,0 +1,481 @@ +--- +title: "Exception Policies" +description: "Configure per-exception handling with composable retry, redelivery, and terminal actions." +--- + +Not every exception deserves the same treatment. A database deadlock might resolve on immediate retry. A downstream service outage needs minutes to recover. A validation error will never succeed no matter how many times you retry. Exception policies let you define per-exception handling strategies — retry, redeliver, dead-letter, or discard — as composable escalation chains in a single `AddExceptionPolicy` call. + +```csharp +builder.Services + .AddMessageBus() + .AddExceptionPolicy(policy => + { + // Validation errors are permanent — route straight to the error endpoint + policy.On().DeadLetter(); + + // Duplicate messages are safe to drop + policy.On().Discard(); + + // Database deadlocks resolve quickly — retry then redeliver + policy.On(ex => ex.IsTransient) + .Retry(5, TimeSpan.FromMilliseconds(200)) + .ThenRedeliver(); + + // Everything else: retry 3 times, then redeliver on a schedule, then dead-letter + policy.Default() + .Retry() + .ThenRedeliver() + .ThenDeadLetter(); + }) + .AddEventHandler() + .AddRabbitMQ(); +``` + +# How exception handling works + +When a handler throws an exception, Mocha evaluates exception policies to determine what happens next. The decision flows through two pipeline stages — retry in the consumer pipeline and redelivery in the receive pipeline — before reaching the fault middleware as a last resort. + +```mermaid +flowchart TD + A[Handler throws exception] --> B{Exception policy\nmatches?} + B -->|Match found| D{Terminal action?} + B -->|No match| M[Route to error endpoint] + + D -->|Discard| E[Swallow exception\nMessage disappears] + D -->|DeadLetter| F[Skip retry and redelivery\nRoute to error endpoint] + D -->|No terminal| G{Retry configured?} + + G -->|Yes| H[Retry with configured settings] + G -->|Disabled| K{Redelivery configured?} + + H -->|Success| I[Done] + H -->|All retries exhausted| K + + K -->|Yes| L[Schedule redelivery\nMessage re-enters pipeline later] + K -->|Disabled| M + + L -->|Success on redelivery| I + L -->|All redeliveries exhausted| M +``` + +Each exception policy rule targets a specific exception type and defines an escalation chain. The chain controls which stages the message passes through and with what settings. + +Exception matching respects inheritance. A policy on `NpgsqlException` also matches any subclass. When multiple rules could match, the most specific type wins — the same precedence as C# `catch` blocks. + +# Configure exception policies + +`AddExceptionPolicy` is the single entry point for all exception handling configuration. There is no separate `AddRetry` or `AddRedelivery` call — retry and redelivery settings are configured per-exception within the policy. + +## Parameterless defaults + +The parameterless overload registers a catch-all `Default()` rule with both retry and redelivery enabled using built-in defaults: + +```csharp +builder.Services + .AddMessageBus() + .AddExceptionPolicy() + .AddEventHandler() + .AddRabbitMQ(); +``` + +This is equivalent to: + +```csharp +.AddExceptionPolicy(policy => +{ + policy.Default().Retry().ThenRedeliver(); +}) +``` + +## Default() and On<T>() + +`ExceptionPolicyOptions` exposes two methods for creating rules: + +- **`Default()`** — shorthand for `On()`. Configures the catch-all behavior for any exception that does not match a more specific rule. +- **`On()`** — configures behavior for a specific exception type. +- **`On(predicate)`** — configures behavior for a specific exception type when a predicate matches. + +```csharp +.AddExceptionPolicy(policy => +{ + policy.On().Retry(5).ThenRedeliver(); + policy.On().Retry(3); + policy.Default().Retry().ThenRedeliver(); +}) +``` + +## Bus-level policies + +Bus-level policies apply to all endpoints and all consumers across the entire message bus. + +```csharp +builder.Services + .AddMessageBus() + .AddExceptionPolicy(policy => + { + policy.On().DeadLetter(); + policy.On().Discard(); + policy.Default().Retry().ThenRedeliver(); + }) + .AddEventHandler() + .AddRabbitMQ(); +``` + +## Transport-level policies + +Override bus-level policies for a specific transport. Transport-level policies replace the bus-level policies entirely for all endpoints on that transport — they are not merged. + +```csharp +builder.Services + .AddMessageBus() + .AddExceptionPolicy(policy => + { + policy.Default().Retry().ThenRedeliver(); + }) + .AddRabbitMQ(transport => + { + transport.AddExceptionPolicy(policy => + { + policy.On().DeadLetter(); + }); + }); +``` + +## Consumer-level policies + +Override policies for a specific consumer. Consumer-level policies replace the bus-level and transport-level policies for that consumer. + +```csharp +builder.Services + .AddMessageBus() + .AddExceptionPolicy(policy => + { + policy.On() + .Retry(2, TimeSpan.FromMilliseconds(100), RetryBackoffType.Constant); + }); + +builder.Services.ConfigureMessageBus(bus => +{ + bus.AddHandler(consumer => + { + consumer.AddExceptionPolicy(policy => + { + policy.On() + .Retry(5, TimeSpan.FromMilliseconds(500), RetryBackoffType.Exponential); + }); + }); +}); +``` + +The `PaymentHandler` gets 5 retries with exponential backoff. All other consumers get the bus-level policy of 2 retries. + +## Scope hierarchy + +Exception policies resolve at four levels. The most specific scope wins, and replacement is atomic — the entire set of rules is replaced, not individual rules. + +| Scope | Applies to | Configured on | +| --------- | ------------------------------------- | ---------------------------- | +| Bus | All endpoints and consumers | `IMessageBusBuilder` | +| Host | All message buses on the host | `IMessageBusHostBuilder` | +| Transport | All endpoints on a specific transport | `IReceiveMiddlewareProvider` | +| Consumer | A single consumer | `IConsumerDescriptor` | + +``` +Consumer policies → Transport policies → Bus policies → Host policies + (highest priority) (lowest priority) +``` + +If a consumer defines exception policies, the bus-level and transport-level policies are ignored for that consumer. If you need a bus-level rule to also apply at the consumer level, include it in the consumer-level configuration. + +# Terminal actions + +Terminal actions end the message's lifecycle immediately. No retry, no redelivery — the message is either routed to the error endpoint or discarded. + +## DeadLetter + +`DeadLetter()` routes the message to the error endpoint, skipping both retry and redelivery. Use this for exceptions that are permanent — retrying will never succeed and you want the message preserved for inspection. + +```csharp +policy.On().DeadLetter(); +policy.On().DeadLetter(); +policy.On().DeadLetter(); +``` + +**When to use:** Malformed messages, authorization failures, schema violations, business rule violations that require manual intervention. + +## Discard + +`Discard()` swallows the exception and the message disappears. No error endpoint, no fault headers, no trace beyond logging. + +```csharp +policy.On().Discard(); +policy.On().Discard(); +``` + +**When to use:** Messages that are safe to lose — duplicates you have already processed, stale events that no longer matter. Use with caution: discarded messages leave no audit trail in the error endpoint. + +# Retry policies + +Retry re-runs the handler in-process using immediate retries. The message stays in memory, the concurrency slot is held, and the handler is invoked again after a short delay. Use retry for transient failures that resolve in milliseconds to seconds. + +When you call `.Retry()` without chaining `.ThenRedeliver()`, redelivery is disabled for that exception type. If all retries are exhausted, the message routes to the error endpoint. + +## Retry with defaults + +```csharp +policy.On().Retry(); +``` + +| Setting | Default value | +| --------- | ------------- | +| Attempts | 3 | +| Delay | 200 ms | +| Backoff | Exponential | +| Jitter | Enabled | +| Max delay | 30 seconds | + +## Retry with custom attempts + +```csharp +policy.On(ex => ex.IsTransient).Retry(5); +``` + +Overrides the number of retry attempts. Delay, backoff strategy, jitter, and max delay use the built-in defaults. + +## Retry with full configuration + +```csharp +policy.On() + .Retry(3, TimeSpan.FromMilliseconds(500), RetryBackoffType.Exponential); +``` + +Overrides attempts, base delay, and backoff strategy for this exception type. + +## Retry with explicit intervals + +```csharp +policy.On().Retry( +[ + TimeSpan.FromMilliseconds(100), + TimeSpan.FromMilliseconds(500), + TimeSpan.FromSeconds(2) +]); +``` + +Specifies the exact delay before each retry attempt. The array length determines the number of retries. + +## Backoff strategies + +| Strategy | Behavior | +| ------------- | ----------------------------------- | +| `Constant` | Same delay every attempt | +| `Linear` | `delay * attempt` | +| `Exponential` | `delay * 2^(attempt-1)` _(default)_ | + +All strategies apply jitter by default to prevent thundering herd effects. + +# Redelivery policies + +Redelivery schedules the message for later delivery through the transport. The concurrency slot is released, the message re-enters the full receive pipeline on each redelivery attempt, and fresh retry cycles run on each delivery. Use redelivery for failures that need minutes or hours to resolve — a downstream service recovering from an outage, a rate limit resetting, or a database completing a failover. + +When you call `.Redeliver()` directly (without `.Retry()` first), retry is disabled for that exception type. The handler failure goes straight to redelivery scheduling. + +## Redeliver with defaults + +```csharp +policy.On().Redeliver(); +``` + +| Setting | Default value | +| --------- | --------------------- | +| Intervals | 5 min, 15 min, 30 min | +| Jitter | Enabled | +| Max delay | 1 hour | + +## Redeliver with custom attempts and delay + +```csharp +policy.On().Redeliver(5, TimeSpan.FromMinutes(2)); +``` + +Overrides the number of attempts and base delay. + +## Redeliver with explicit intervals + +```csharp +policy.On().Redeliver( +[ + TimeSpan.FromSeconds(30), + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(30) +]); +``` + +Specifies the exact delay before each redelivery attempt. The array length determines the number of redelivery attempts. + +# Escalation chains + +The fluent API composes retry, redelivery, and terminal actions into escalation chains. The interface design enforces valid chains at compile time — you cannot chain `.ThenRedeliver()` after `.Redeliver()`, or `.Retry()` after `.ThenRedeliver()`. + +## Retry then redeliver + +```csharp +policy.On(ex => ex.IsTransient) + .Retry(3) + .ThenRedeliver(); +``` + +Try 3 immediate retries. If all fail, schedule redelivery with defaults (5, 15, 30 minutes). Each redelivery attempt runs a fresh cycle of 3 retries. + +## Retry then redeliver with custom settings + +```csharp +policy.On() + .Retry(5, TimeSpan.FromMilliseconds(500)) + .ThenRedeliver( + [ + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(15), + TimeSpan.FromMinutes(30) + ]) + .ThenDeadLetter(); +``` + +Try 5 immediate retries with 500 ms exponential backoff. If exhausted, schedule redeliveries at 5, 15, and 30 minutes. If all redeliveries are exhausted, route to the error endpoint. + +## Retry then dead-letter + +```csharp +policy.On() + .Retry(3) + .ThenDeadLetter(); +``` + +Try 3 immediate retries. If all fail, skip redelivery and route to the error endpoint immediately. + +## Chain behavior reference + +| Chain | Retry | Redelivery | Terminal | +| -------------------------------------------- | ---------- | ---------- | ---------- | +| `.Discard()` | - | - | Discard | +| `.DeadLetter()` | - | - | DeadLetter | +| `.Retry()` | Enabled | Disabled | Default | +| `.Retry(3)` | 3 attempts | Disabled | Default | +| `.Redeliver()` | Disabled | Enabled | Default | +| `.Retry(3).ThenRedeliver()` | 3 attempts | Enabled | Default | +| `.Retry(3).ThenDeadLetter()` | 3 attempts | Disabled | DeadLetter | +| `.Retry(3).ThenRedeliver().ThenDeadLetter()` | 3 attempts | Enabled | DeadLetter | + +**Default terminal behavior:** When no terminal action is specified, exhausted messages route to the error endpoint through the fault middleware. `.ThenDeadLetter()` makes this intent explicit but does not change the behavior. + +**Disabled vs. default:** "Disabled" means that tier is skipped for this exception type. A dash (-) means the tier is not configured and does not apply. + +# Conditional policies + +Use predicate overloads to apply policies only when an exception matches specific conditions. The predicate receives the typed exception instance. + +## Filter by exception property + +```csharp +// Only retry transient database errors +policy.On(ex => ex.IsTransient) + .Retry(5); + +// Dead-letter non-transient database errors +policy.On(ex => !ex.IsTransient) + .DeadLetter(); +``` + +## Filter by HTTP status code + +```csharp +policy.On(ex => + ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests) + .Redeliver( + [ + TimeSpan.FromSeconds(30), + TimeSpan.FromMinutes(2), + TimeSpan.FromMinutes(10) + ]); + +policy.On(ex => + ex.StatusCode == System.Net.HttpStatusCode.BadRequest) + .DeadLetter(); +``` + +## Filter by inner exception + +```csharp +policy.On(ex => + ex.InnerException is TimeoutException) + .Retry(3); +``` + +## Multiple rules for the same type + +You can define multiple rules for the same exception type with different predicates. When an exception is thrown, the first matching rule wins. + +```csharp +policy.On(ex => + ex.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable) + .Retry(3).ThenRedeliver(); + +policy.On(ex => + ex.StatusCode == System.Net.HttpStatusCode.BadRequest) + .DeadLetter(); + +// Catch-all for other HTTP errors +policy.On().Retry(3); +``` + +# API reference + +## ExceptionPolicyOptions + +| Method | Parameters | Returns | Description | +| --------------------------- | ------------------------- | ------------------------------------- | -------------------------------------------------------------------------- | +| `Default()` | - | `IExceptionPolicyBuilder` | Configure the catch-all default behavior. Equivalent to `On()`. | +| `On()` | - | `IExceptionPolicyBuilder` | Configure behavior for an exception type | +| `On(predicate)` | `Func?` | `IExceptionPolicyBuilder` | Configure behavior for an exception type matching a condition | + +## IExceptionPolicyBuilder<TException> + +| Method | Parameters | Returns | Description | +| --------------------------------- | ------------------------------------- | ------------------------- | ------------------------------------------------------------------------- | +| `Discard()` | - | `void` | Swallow the exception, discard the message | +| `DeadLetter()` | - | `void` | Route to error endpoint, skip retry and redelivery | +| `Retry()` | - | `IAfterRetryBuilder` | Retry with defaults (3 attempts, 200 ms, exponential), disable redelivery | +| `Retry(attempts)` | `int` | `IAfterRetryBuilder` | Retry with custom attempts, disable redelivery | +| `Retry(attempts, delay, backoff)` | `int`, `TimeSpan`, `RetryBackoffType` | `IAfterRetryBuilder` | Retry with full configuration, disable redelivery | +| `Retry(intervals)` | `TimeSpan[]` | `IAfterRetryBuilder` | Retry with explicit intervals, disable redelivery | +| `Redeliver()` | - | `IAfterRedeliveryBuilder` | Redeliver with defaults (5, 15, 30 min), disable retry | +| `Redeliver(attempts, baseDelay)` | `int`, `TimeSpan` | `IAfterRedeliveryBuilder` | Redeliver with custom attempts and delay, disable retry | +| `Redeliver(intervals)` | `TimeSpan[]` | `IAfterRedeliveryBuilder` | Redeliver with explicit intervals, disable retry | + +## IAfterRetryBuilder + +| Method | Parameters | Returns | Description | +| ------------------------------------ | ----------------- | ------------------------- | --------------------------------------------------------------- | +| `ThenRedeliver()` | - | `IAfterRedeliveryBuilder` | Chain redelivery with defaults after retry exhaustion | +| `ThenRedeliver(attempts, baseDelay)` | `int`, `TimeSpan` | `IAfterRedeliveryBuilder` | Chain redelivery with custom settings after retry exhaustion | +| `ThenRedeliver(intervals)` | `TimeSpan[]` | `IAfterRedeliveryBuilder` | Chain redelivery with explicit intervals after retry exhaustion | +| `ThenDeadLetter()` | - | `void` | Route to error endpoint after retry exhaustion | + +## IAfterRedeliveryBuilder + +| Method | Parameters | Returns | Description | +| ------------------ | ---------- | ------- | --------------------------------------------------- | +| `ThenDeadLetter()` | - | `void` | Route to error endpoint after redelivery exhaustion | + +## AddExceptionPolicy extensions + +| Target | Method | Description | +| ---------------------------- | ---------------------------------------------------- | ---------------------------------------------------------------------- | +| `IMessageBusBuilder` | `AddExceptionPolicy()` | Register default exception policies at the bus level | +| `IMessageBusBuilder` | `AddExceptionPolicy(Action)` | Configure exception policies at the bus level | +| `IMessageBusHostBuilder` | `AddExceptionPolicy()` | Register default exception policies at the host level | +| `IMessageBusHostBuilder` | `AddExceptionPolicy(Action)` | Configure exception policies at the host level | +| `IReceiveMiddlewareProvider` | `AddExceptionPolicy()` | Register default exception policies at the transport or endpoint level | +| `IReceiveMiddlewareProvider` | `AddExceptionPolicy(Action)` | Configure exception policies at the transport or endpoint level | +| `IConsumerDescriptor` | `AddExceptionPolicy()` | Register default exception policies at the consumer level | +| `IConsumerDescriptor` | `AddExceptionPolicy(Action)` | Configure exception policies at the consumer level | diff --git a/website/src/docs/mocha/v1/reliability.md b/website/src/docs/mocha/v1/reliability.md index cd131e23117..2f464af8cd3 100644 --- a/website/src/docs/mocha/v1/reliability.md +++ b/website/src/docs/mocha/v1/reliability.md @@ -1,6 +1,6 @@ --- title: "Reliability" -description: "Configure retry policies, delayed redelivery, fault handling, dead-letter routing, message expiry, concurrency limits, circuit breakers, the transactional outbox, and the idempotent inbox in Mocha to build resilient messaging pipelines." +description: "Configure exception policies, retry, delayed redelivery, fault handling, dead-letter routing, message expiry, concurrency limits, circuit breakers, the transactional outbox, and the idempotent inbox in Mocha to build resilient messaging pipelines." --- Messaging systems fail. Handlers throw exceptions, brokers go offline, databases lock up, and messages arrive faster than consumers can process them. Mocha's reliability features handle these failures at the infrastructure level so your handler code stays focused on business logic. @@ -8,8 +8,11 @@ Messaging systems fail. Handlers throw exceptions, brokers go offline, databases ```csharp builder.Services .AddMessageBus() - .AddRetry() - .AddRedelivery() + .AddExceptionPolicy(policy => + { + policy.On().DeadLetter(); + policy.Default().Retry().ThenRedeliver(); + }) .AddCircuitBreaker(opts => { opts.FailureRatio = 0.5; @@ -26,7 +29,7 @@ builder.Services .AddRabbitMQ(); ``` -That configuration adds immediate retry, delayed redelivery, circuit breaking, concurrency limiting, transactional outbox, idempotent inbox, and database transaction wrapping - all as middleware in the receive and dispatch pipelines. +That configuration adds per-exception retry and redelivery policies, circuit breaking, concurrency limiting, transactional outbox, idempotent inbox, and database transaction wrapping - all as middleware in the receive and dispatch pipelines. # Delivery guarantees @@ -50,13 +53,13 @@ TransportCircuitBreaker -> Instrumentation -> DeadLetter -> Fault - -> Redelivery ← schedules for later if retries exhausted + -> Redelivery <- schedules for later if retries exhausted -> CircuitBreaker -> Expiry -> MessageTypeSelection -> Routing -> Consumer pipeline - -> Retry ← immediate in-process retries (Polly) + -> Retry <- immediate in-process retries -> Transaction middleware (BEGIN) -> Inbox (claim inside transaction) -> Your handler @@ -152,7 +155,7 @@ For time-sensitive commands, a short expiry prevents stale operations from execu # Retry failed messages -When a handler throws a transient exception - a database timeout, an HTTP 503, temporary lock contention - retrying the same operation a few hundred milliseconds later often succeeds. The retry middleware re-runs the handler in-process using [Polly](https://github.com/App-vNext/Polly) without releasing the concurrency slot or leaving the consumer pipeline. +When a handler throws a transient exception - a database timeout, an HTTP 503, temporary lock contention - retrying the same operation a few hundred milliseconds later often succeeds. The retry middleware re-runs the handler in-process without releasing the concurrency slot or leaving the consumer pipeline. Retry lives in the **consumer pipeline**, not the receive pipeline. This matters for two reasons: @@ -164,25 +167,23 @@ Retry lives in the **consumer pipeline**, not the receive pipeline. This matters ```csharp builder.Services .AddMessageBus() - .AddRetry() + .AddExceptionPolicy() .AddEventHandler() .AddRabbitMQ(); ``` -With no configuration, retry uses exponential backoff: 3 attempts, 200 ms base delay, jitter enabled, 30-second maximum delay. These defaults handle the majority of transient failures without tuning. +The parameterless `AddExceptionPolicy()` registers a catch-all `Default()` rule with both retry and redelivery enabled. Retry defaults to 3 attempts, 200 ms base delay, exponential backoff, jitter enabled, and a 30-second maximum delay. These defaults handle the majority of transient failures without tuning. ## Customize retry behavior ```csharp builder.Services .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(policy => { - retry.MaxRetryAttempts = 5; - retry.Delay = TimeSpan.FromSeconds(1); - retry.BackoffType = RetryBackoffType.Exponential; - retry.UseJitter = true; - retry.MaxDelay = TimeSpan.FromSeconds(60); + policy.Default() + .Retry(5, TimeSpan.FromSeconds(1), RetryBackoffType.Exponential) + .ThenRedeliver(); }) .AddEventHandler() .AddRabbitMQ(); @@ -190,93 +191,103 @@ builder.Services ## Use explicit retry intervals -When you need precise control over each delay, set `Intervals` directly. This overrides `Delay`, `BackoffType`, and `MaxRetryAttempts` - the array length becomes the number of retries. +When you need precise control over each delay, pass an array of intervals. The array length determines the number of retries. ```csharp builder.Services .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(policy => { - retry.Intervals = + policy.Default().Retry( [ TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(500), TimeSpan.FromSeconds(2) - ]; + ]).ThenRedeliver(); }) .AddEventHandler() .AddRabbitMQ(); ``` -## Filter exceptions +## Handle different exceptions differently -Not every exception should be retried. Validation errors, authorization failures, and other permanent errors waste retry budget. Use `On().Ignore()` to exclude specific exception types from retry. The exception propagates immediately to the fault middleware. +Not every exception should be retried. Validation errors, authorization failures, and other permanent errors waste retry budget. Use `AddExceptionPolicy` to define per-exception handling strategies. ```csharp builder.Services .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(policy => { - retry.MaxRetryAttempts = 3; - - // Never retry validation failures - retry.On().Ignore(); + // Never retry validation failures - route to error endpoint + policy.On().DeadLetter(); // Never retry non-transient database errors - retry.On(ex => !ex.IsTransient).Ignore(); + policy.On(ex => !ex.IsTransient).DeadLetter(); + + // Transient database errors get more retries + policy.On(ex => ex.IsTransient) + .Retry(5, TimeSpan.FromMilliseconds(200)) + .ThenRedeliver(); + + // Everything else: retry then redeliver + policy.Default().Retry().ThenRedeliver(); }) .AddEventHandler() .AddRabbitMQ(); ``` -Exception filtering respects inheritance. `On().Ignore()` also ignores `NpgsqlException` and any other `DbException` subclass. When multiple rules match, the most specific type wins - the same precedence as C# `catch` blocks. +Exception matching respects inheritance. `On()` also matches `NpgsqlException` and any other `DbException` subclass. When multiple rules match, the most specific type wins - the same precedence as C# `catch` blocks. + +For the full API including predicates, conditional policies, and escalation chains, see [Exception Policies](/docs/mocha/v1/exception-policies). ## Override retry per consumer -The bus-level retry configuration applies to all consumers. Override it for specific consumers that need different behavior. See [Handlers and Consumers](/docs/mocha/v1/handlers-and-consumers) for the full consumer configuration API. +The bus-level exception policy applies to all consumers. Override it for specific consumers that need different behavior. See [Handlers and Consumers](/docs/mocha/v1/handlers-and-consumers) for the full consumer configuration API. ```csharp builder.Services .AddMessageBus() - .AddRetry() // bus-level: 3 retries for all consumers + .AddExceptionPolicy() // bus-level: default retry + redelivery for all consumers .AddEventHandler(consumer => { - // This consumer: 10 retries with longer delay - consumer.AddRetry(retry => + // This consumer: 10 retries with longer delay, then redeliver + consumer.AddExceptionPolicy(policy => { - retry.MaxRetryAttempts = 10; - retry.Delay = TimeSpan.FromSeconds(2); + policy.Default() + .Retry(10, TimeSpan.FromSeconds(2)) + .ThenRedeliver(); }); }) .AddEventHandler(consumer => { - // This consumer: no retries - consumer.AddRetry(retry => retry.Enabled = false); + // This consumer: skip retry, redeliver only + consumer.AddExceptionPolicy(policy => + { + policy.Default().Redeliver(); + }); }) .AddRabbitMQ(); ``` -Properties cascade: consumer overrides bus, bus overrides defaults. Properties not set at the consumer level (like `BackoffType`) fall through to the bus configuration, then to `RetryOptions.Defaults`. +Consumer-level policies replace the bus-level policies entirely for that consumer - they are not merged. If you need a bus-level rule to also apply at the consumer level, include it in the consumer-level configuration. -## Retry options reference +## Retry defaults reference -| Property | Type | Default | Description | -| ------------------ | ------------------- | ------------- | ------------------------------------------------------------------------------------------------ | -| `Enabled` | `bool?` | `true` | Set to `false` to disable retry at this scope. `null` inherits from parent. | -| `MaxRetryAttempts` | `int?` | `3` | Number of retries after the initial attempt. Total handler invocations = `MaxRetryAttempts + 1`. | -| `Delay` | `TimeSpan?` | 200 ms | Base delay between retries. Interpretation depends on `BackoffType`. | -| `MaxDelay` | `TimeSpan?` | 30 s | Upper bound on any single retry delay. Prevents exponential backoff from growing unbounded. | -| `BackoffType` | `RetryBackoffType?` | `Exponential` | Delay calculation strategy: `Constant`, `Linear`, or `Exponential`. | -| `UseJitter` | `bool?` | `true` | Adds random variance to delays, preventing synchronized retry storms across consumers. | -| `Intervals` | `TimeSpan[]?` | `null` | Explicit delay per retry. When set, overrides `Delay`, `BackoffType`, and `MaxRetryAttempts`. | +| Setting | Default value | +| --------- | ------------- | +| Attempts | 3 | +| Delay | 200 ms | +| Backoff | Exponential | +| Jitter | Enabled | +| Max delay | 30 seconds | ### RetryBackoffType values | Value | Formula | Example (Delay = 200 ms) | | ------------- | ----------------------- | ------------------------ | | `Constant` | `Delay` | 200 ms, 200 ms, 200 ms | -| `Linear` | `Delay × (attempt + 1)` | 400 ms, 600 ms, 800 ms | -| `Exponential` | `Delay × 2^attempt` | 400 ms, 800 ms, 1600 ms | +| `Linear` | `Delay * (attempt + 1)` | 400 ms, 600 ms, 800 ms | +| `Exponential` | `Delay * 2^attempt` | 400 ms, 800 ms, 1600 ms | # Redeliver failed messages @@ -285,7 +296,7 @@ Retry handles transient blips that resolve in milliseconds. Some failures take l Redelivery lives in the **receive pipeline**, not the consumer pipeline. This matters for two reasons: 1. **Single decision point.** When an endpoint has multiple consumers, you want one redelivery decision per message, not one per consumer. -2. **Releases the concurrency slot.** Retry holds the slot while Polly waits. Redelivery returns the message to the transport and frees the slot for other messages. +2. **Releases the concurrency slot.** Retry holds the slot while waiting. Redelivery returns the message to the transport and frees the slot for other messages. Redelivery uses Mocha's [scheduling infrastructure](/docs/mocha/v1/scheduling) to schedule the message for future delivery. The message re-enters the full receive pipeline when it arrives - fresh routing, fresh consumer invocations, fresh retry attempts. @@ -297,36 +308,38 @@ Receive pipeline: -> CircuitBreaker -> ... (rest of pipeline) -All retries exhausted → exception propagates to Redelivery - → redelivery attempts remaining → schedule for later delivery - → redelivery exhausted → exception propagates to Fault → error endpoint +All retries exhausted -> exception propagates to Redelivery + -> redelivery attempts remaining -> schedule for later delivery + -> redelivery exhausted -> exception propagates to Fault -> error endpoint ``` ## Add redelivery with defaults +Redelivery is included when you call `AddExceptionPolicy()` without arguments or when you chain `.ThenRedeliver()` in an escalation chain. + ```csharp builder.Services .AddMessageBus() - .AddRedelivery() + .AddExceptionPolicy() // Default() rule enables both retry and redelivery .AddEventHandler() .AddRabbitMQ(); ``` -The default redelivery schedule is three attempts at 5 minutes, 15 minutes, and 30 minutes with jitter enabled. +The default redelivery schedule is three attempts at 5 minutes, 15 minutes, and 30 minutes with jitter enabled and a 1-hour maximum delay. ## Use custom redelivery intervals ```csharp builder.Services .AddMessageBus() - .AddRedelivery(redeliver => + .AddExceptionPolicy(policy => { - redeliver.Intervals = + policy.Default().Retry().ThenRedeliver( [ TimeSpan.FromSeconds(30), TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(30) - ]; + ]); }) .AddEventHandler() .AddRabbitMQ(); @@ -336,152 +349,136 @@ The array length determines the number of redelivery attempts. Each element is t ## Use calculated redelivery delays -Instead of explicit intervals, configure a base delay and maximum. The actual delay for each attempt is `BaseDelay × (attempt + 1)`, capped at `MaxDelay`. +Instead of explicit intervals, configure a number of attempts and a base delay: ```csharp builder.Services .AddMessageBus() - .AddRedelivery(redeliver => + .AddExceptionPolicy(policy => { - redeliver.MaxAttempts = 5; - redeliver.BaseDelay = TimeSpan.FromMinutes(2); - redeliver.MaxDelay = TimeSpan.FromHours(1); - redeliver.UseJitter = true; + policy.Default() + .Retry() + .ThenRedeliver(5, TimeSpan.FromMinutes(2)); }) .AddEventHandler() .AddRabbitMQ(); ``` -## Filter exceptions for redelivery +## Redeliver without retry -The same `On().Ignore()` pattern applies to redelivery. Exceptions that match an ignore rule propagate immediately to the fault middleware without scheduling a redelivery. +To skip retry entirely and go straight to redelivery, use `.Redeliver()` directly: ```csharp builder.Services .AddMessageBus() - .AddRedelivery(redeliver => + .AddExceptionPolicy(policy => { - redeliver.Intervals = - [ - TimeSpan.FromMinutes(5), - TimeSpan.FromMinutes(15), - TimeSpan.FromMinutes(30) - ]; - - // Validation failures are permanent - don't redeliver - redeliver.On().Ignore(); - - // Missing configuration won't fix itself - redeliver.On().Ignore(); + policy.Default().Redeliver(); }) .AddEventHandler() .AddRabbitMQ(); ``` +When `.Redeliver()` is called without `.Retry()` first, retry is disabled for that rule. The handler failure goes straight to redelivery scheduling. + ## Override redelivery per endpoint -The bus-level redelivery configuration applies to all endpoints. Override it at the transport or endpoint level for more granular control. +Override exception policies at the transport level for more granular control. Transport-level policies replace the bus-level policies entirely for all endpoints on that transport. ```csharp builder.Services .AddMessageBus() - .AddRedelivery() // bus-level default + .AddExceptionPolicy() // bus-level default .AddRabbitMQ(transport => { - transport.AddRedelivery(redeliver => + transport.AddExceptionPolicy(policy => { - // Override for this transport: longer intervals - redeliver.Intervals = + // Override for this transport: longer redelivery intervals + policy.Default().Retry().ThenRedeliver( [ TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(30), TimeSpan.FromHours(1) - ]; + ]); }); }); ``` -Disable redelivery for a specific transport or endpoint by setting `Enabled` to `false`: +Disable redelivery for a specific transport by configuring retry-only: ```csharp -transport.AddRedelivery(redeliver => redeliver.Enabled = false); +transport.AddExceptionPolicy(policy => +{ + policy.Default().Retry(); +}); ``` -Properties cascade: endpoint overrides transport, transport overrides bus, bus overrides defaults. - -## Redelivery options reference +## Redelivery defaults reference -| Property | Type | Default | Description | -| ------------- | ------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Enabled` | `bool?` | `true` | Set to `false` to disable redelivery at this scope. `null` inherits from parent. | -| `MaxAttempts` | `int?` | - | Maximum redelivery attempts. Inferred from `Intervals.Length` when `Intervals` is set. Required when using calculated delays without `Intervals`. | -| `BaseDelay` | `TimeSpan?` | 5 min | Base delay for calculated backoff: `BaseDelay × (attempt + 1)`. | -| `MaxDelay` | `TimeSpan?` | 1 h | Upper bound on any single redelivery delay. | -| `UseJitter` | `bool?` | `true` | Adds random variance to delays. | -| `Intervals` | `TimeSpan[]?` | [5 min, 15 min, 30 min] | Explicit delay per redelivery attempt. When set, overrides `BaseDelay` and `MaxAttempts`. | +| Setting | Default value | +| --------- | --------------------- | +| Intervals | 5 min, 15 min, 30 min | +| Jitter | Enabled | +| Max delay | 1 hour | # Combine retry and redelivery -The two tiers compose naturally. When both are configured, a message goes through immediate retry first. If all retries are exhausted, the exception propagates up from the consumer pipeline to the receive pipeline, where redelivery catches it and schedules the message for later delivery. On the next delivery round, the full retry cycle runs again. +The two tiers compose naturally through escalation chains. When both are configured, a message goes through immediate retry first. If all retries are exhausted, the exception propagates up from the consumer pipeline to the receive pipeline, where redelivery catches it and schedules the message for later delivery. On the next delivery round, the full retry cycle runs again. ```text Message arrives - │ - ├─ Consumer pipeline: Retry (Polly) - │ ├─ Attempt 1 → handler throws → retry - │ ├─ Attempt 2 → handler throws → retry - │ ├─ Attempt 3 → handler throws → retry - │ └─ Attempt 4 → handler throws → retries exhausted, exception propagates - │ - ├─ Receive pipeline: Redelivery - │ └─ Schedule message for delivery in 5 minutes - │ - ├─ ... 5 minutes later, message re-enters pipeline ... - │ - ├─ Consumer pipeline: Retry (Polly) - │ ├─ Attempt 1 → handler throws → retry - │ ├─ ... - │ └─ Attempt 4 → retries exhausted, exception propagates - │ - ├─ Receive pipeline: Redelivery - │ └─ Schedule message for delivery in 15 minutes - │ - ├─ ... continues until redelivery attempts exhausted ... - │ - └─ Fault middleware → error endpoint (dead letter) + | + +- Consumer pipeline: Retry + | +- Attempt 1 -> handler throws -> retry + | +- Attempt 2 -> handler throws -> retry + | +- Attempt 3 -> handler throws -> retry + | +- Attempt 4 -> handler throws -> retries exhausted, exception propagates + | + +- Receive pipeline: Redelivery + | +- Schedule message for delivery in 5 minutes + | + +- ... 5 minutes later, message re-enters pipeline ... + | + +- Consumer pipeline: Retry + | +- Attempt 1 -> handler throws -> retry + | +- ... + | +- Attempt 4 -> retries exhausted, exception propagates + | + +- Receive pipeline: Redelivery + | +- Schedule message for delivery in 15 minutes + | + +- ... continues until redelivery attempts exhausted ... + | + +- Fault middleware -> error endpoint (dead letter) ``` The total number of handler invocations before a message reaches the error endpoint: ``` -Total attempts = (MaxRetryAttempts + 1) × (redelivery attempts + 1) +Total attempts = (retry attempts + 1) x (redelivery attempts + 1) ``` -With the defaults (3 retries, 3 redeliveries): `(3 + 1) × (3 + 1) = 16` total handler invocations. +With the defaults (3 retries, 3 redeliveries): `(3 + 1) x (3 + 1) = 16` total handler invocations. ```csharp builder.Services .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(policy => { - retry.MaxRetryAttempts = 5; - retry.Delay = TimeSpan.FromSeconds(1); - retry.BackoffType = RetryBackoffType.Exponential; - }) - .AddRedelivery(redeliver => - { - redeliver.Intervals = - [ - TimeSpan.FromMinutes(5), - TimeSpan.FromMinutes(15), - TimeSpan.FromMinutes(30) - ]; + policy.Default() + .Retry(5, TimeSpan.FromSeconds(1), RetryBackoffType.Exponential) + .ThenRedeliver( + [ + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(15), + TimeSpan.FromMinutes(30) + ]); }) .AddEventHandler() .AddRabbitMQ(); ``` -With this configuration: `(5 + 1) × (3 + 1) = 24` total handler invocations before dead-letter. +With this configuration: `(5 + 1) x (3 + 1) = 24` total handler invocations before dead-letter. :::note Request/reply messages skip redelivery Redelivery does not apply to request/reply messages. The caller is waiting synchronously for a response - scheduling the message for delivery minutes later would cause a timeout. When a request/reply handler fails after retry exhaustion, the exception propagates directly to the fault middleware, which sends a `NotAcknowledgedEvent` back to the caller. Immediate retry still applies to request/reply handlers. @@ -496,7 +493,7 @@ public class PaymentConsumer(ILogger logger) : IConsumer context) { - var state = context.Features.Get(); + var state = context.Features.Get(); if (state is { ImmediateRetryCount: > 2 }) { @@ -519,7 +516,7 @@ public class PaymentConsumer(ILogger logger) : IConsumer logger) : IConsumer().Ignore()` to exclude permanent failures from retry and redelivery: +Use `AddExceptionPolicy` to route permanent failures directly to the error endpoint: ```csharp builder.Services .AddMessageBus() - .AddRetry(retry => + .AddExceptionPolicy(policy => { - retry.On().Ignore(); - }) - .AddRedelivery(redeliver => - { - redeliver.On().Ignore(); + policy.On().DeadLetter(); + policy.Default().Retry().ThenRedeliver(); }) .AddEventHandler() .AddRabbitMQ(); ``` +See [Exception Policies](/docs/mocha/v1/exception-policies) for the full per-exception configuration API, including predicates, terminal actions, and escalation chains. + ## "Request/reply messages are not redelivered" This is by design. Redelivery skips request/reply messages because the caller is waiting for a synchronous response. Immediate retry still applies. See the [note above](#combine-retry-and-redelivery) for details. @@ -976,9 +972,10 @@ The inbox cleanup worker is a background hosted service (`IHostedService`). It r # Next steps -Your messaging pipeline now retries transient failures, redelivers after sustained errors, limits concurrency, breaks circuits on repeated failures, guarantees delivery through the outbox, and deduplicates messages through the inbox. To monitor your messaging system, see [Observability](/docs/mocha/v1/observability). +Your messaging pipeline now handles exceptions per-type with retry and redelivery policies, limits concurrency, breaks circuits on repeated failures, guarantees delivery through the outbox, and deduplicates messages through the inbox. To monitor your messaging system, see [Observability](/docs/mocha/v1/observability). -- [**Handlers and Consumers**](/docs/mocha/v1/handlers-and-consumers) - Configure per-consumer retry overrides and understand handler exception behavior. +- [**Exception Policies**](/docs/mocha/v1/exception-policies) - Configure per-exception handling with composable retry, redelivery, and terminal actions. +- [**Handlers and Consumers**](/docs/mocha/v1/handlers-and-consumers) - Configure per-consumer exception policy overrides and understand handler exception behavior. - [**Scheduling**](/docs/mocha/v1/scheduling) - Configure the scheduling infrastructure that redelivery uses for delayed message delivery. - [**Middleware and Pipelines**](/docs/mocha/v1/middleware-and-pipelines) - Write custom middleware, control pipeline ordering, and understand the three pipeline stages. - [**Sagas**](/docs/mocha/v1/sagas) - Coordinate multi-step workflows with state machine sagas that use compensation when steps fail. From c576cafba7a896003a3840dc1a09802ea3297d9d Mon Sep 17 00:00:00 2001 From: Pascal Senn Date: Mon, 6 Apr 2026 22:36:35 +0000 Subject: [PATCH 04/14] cleanup --- .../examples/ExceptionPolicies/Program.cs | 4 +- src/Mocha/src/Demo/Demo.Billing/Program.cs | 2 +- src/Mocha/src/Demo/Demo.Catalog/Program.cs | 2 +- src/Mocha/src/Demo/Demo.Shipping/Program.cs | 2 +- .../Consume/Retry/ExceptionPolicyBuilder.cs | 10 ++++ ...s => ResilienceConfigurationExtensions.cs} | 22 ++++---- .../Consume/Retry/RetryRuntimeFeature.cs | 2 +- .../Behaviors/RedeliveryTests.cs | 14 +++--- .../Behaviors/RetryTests.cs | 28 +++++------ .../src/docs/mocha/v1/exception-policies.md | 50 ++++++++++--------- website/src/docs/mocha/v1/reliability.md | 46 ++++++++--------- 11 files changed, 98 insertions(+), 84 deletions(-) rename src/Mocha/src/Mocha/Middlewares/Consume/Retry/{ExceptionPolicyConfigurationExtensions.cs => ResilienceConfigurationExtensions.cs} (85%) diff --git a/src/Mocha/examples/ExceptionPolicies/Program.cs b/src/Mocha/examples/ExceptionPolicies/Program.cs index 27892d97516..597d209fde5 100644 --- a/src/Mocha/examples/ExceptionPolicies/Program.cs +++ b/src/Mocha/examples/ExceptionPolicies/Program.cs @@ -20,10 +20,10 @@ // ----------------------------------------------------------------------- // Exception Policies — the main showcase // - // Per-exception rules are configured in a single AddExceptionPolicy call. + // Per-exception rules are configured in a single AddResilience call. // The On() catch-all provides global retry/redelivery defaults. // ----------------------------------------------------------------------- - .AddExceptionPolicy(policy => + .AddResilience(policy => { // --- Terminal: DeadLetter --- // Validation errors are permanent — the message payload is bad. diff --git a/src/Mocha/src/Demo/Demo.Billing/Program.cs b/src/Mocha/src/Demo/Demo.Billing/Program.cs index f9555aa69d1..50959616b33 100644 --- a/src/Mocha/src/Demo/Demo.Billing/Program.cs +++ b/src/Mocha/src/Demo/Demo.Billing/Program.cs @@ -31,7 +31,7 @@ builder .Services.AddMessageBus() .AddInstrumentation() - .AddExceptionPolicy() + .AddResilience() .AddBilling() .AddBatchHandler(opts => { diff --git a/src/Mocha/src/Demo/Demo.Catalog/Program.cs b/src/Mocha/src/Demo/Demo.Catalog/Program.cs index f94211ca403..ed8e6181c62 100644 --- a/src/Mocha/src/Demo/Demo.Catalog/Program.cs +++ b/src/Mocha/src/Demo/Demo.Catalog/Program.cs @@ -31,7 +31,7 @@ builder .Services.AddMessageBus() .AddInstrumentation() - .AddExceptionPolicy() + .AddResilience() .AddCatalog() .AddEntityFramework(p => { diff --git a/src/Mocha/src/Demo/Demo.Shipping/Program.cs b/src/Mocha/src/Demo/Demo.Shipping/Program.cs index 2243c4aeae4..0d136e3976d 100644 --- a/src/Mocha/src/Demo/Demo.Shipping/Program.cs +++ b/src/Mocha/src/Demo/Demo.Shipping/Program.cs @@ -28,7 +28,7 @@ builder .Services.AddMessageBus() .AddInstrumentation() - .AddExceptionPolicy() + .AddResilience() .AddShipping() .AddEntityFramework(p => { diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs index a7308f473cc..7605dc115c5 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs @@ -18,6 +18,16 @@ private void EnsureCommitted() { if (!_committed) { + // Replace any existing rule for the same exception type and predicate. + for (var i = _rules.Count - 1; i >= 0; i--) + { + if (_rules[i].ExceptionType == _rule.ExceptionType + && _rules[i].Predicate == _rule.Predicate) + { + _rules.RemoveAt(i); + } + } + _rules.Add(_rule); _committed = true; } diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyConfigurationExtensions.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ResilienceConfigurationExtensions.cs similarity index 85% rename from src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyConfigurationExtensions.cs rename to src/Mocha/src/Mocha/Middlewares/Consume/Retry/ResilienceConfigurationExtensions.cs index f04f5d4bfc4..e1a0176d43e 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyConfigurationExtensions.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ResilienceConfigurationExtensions.cs @@ -6,7 +6,7 @@ namespace Mocha; /// Provides extension methods for configuring exception policies including retry, redelivery, /// and per-exception rules on message bus builders, host builders, descriptors, and consumers. /// -public static class ExceptionPolicyConfigurationExtensions +public static class ResilienceConfigurationExtensions { /// /// Adds exception policy configuration to the message bus with default settings. @@ -14,7 +14,7 @@ public static class ExceptionPolicyConfigurationExtensions /// /// The message bus builder. /// The builder for method chaining. - public static IMessageBusBuilder AddExceptionPolicy(this IMessageBusBuilder builder) + public static IMessageBusBuilder AddResilience(this IMessageBusBuilder builder) { builder.ConfigureFeature(f => f.GetOrSet() .Configure(p => p.Default().Retry().ThenRedeliver())); @@ -27,7 +27,7 @@ public static IMessageBusBuilder AddExceptionPolicy(this IMessageBusBuilder buil /// The message bus builder. /// The action to configure exception policy options. /// The builder for method chaining. - public static IMessageBusBuilder AddExceptionPolicy( + public static IMessageBusBuilder AddResilience( this IMessageBusBuilder builder, Action configure) { @@ -40,9 +40,9 @@ public static IMessageBusBuilder AddExceptionPolicy( /// /// The host builder. /// The builder for method chaining. - public static IMessageBusHostBuilder AddExceptionPolicy(this IMessageBusHostBuilder builder) + public static IMessageBusHostBuilder AddResilience(this IMessageBusHostBuilder builder) { - builder.ConfigureMessageBus(x => x.AddExceptionPolicy()); + builder.ConfigureMessageBus(x => x.AddResilience()); return builder; } @@ -52,11 +52,11 @@ public static IMessageBusHostBuilder AddExceptionPolicy(this IMessageBusHostBuil /// The host builder. /// The action to configure exception policy options. /// The builder for method chaining. - public static IMessageBusHostBuilder AddExceptionPolicy( + public static IMessageBusHostBuilder AddResilience( this IMessageBusHostBuilder builder, Action configure) { - builder.ConfigureMessageBus(x => x.AddExceptionPolicy(configure)); + builder.ConfigureMessageBus(x => x.AddResilience(configure)); return builder; } @@ -68,7 +68,7 @@ public static IMessageBusHostBuilder AddExceptionPolicy( /// The descriptor type that supports receive middleware. /// The descriptor to configure. /// The descriptor for method chaining. - public static TDescriptor AddExceptionPolicy( + public static TDescriptor AddResilience( this TDescriptor descriptor) where TDescriptor : IReceiveMiddlewareProvider { @@ -84,7 +84,7 @@ public static TDescriptor AddExceptionPolicy( /// The descriptor to configure. /// The action to configure exception policy options. /// The descriptor for method chaining. - public static TDescriptor AddExceptionPolicy( + public static TDescriptor AddResilience( this TDescriptor descriptor, Action configure) where TDescriptor : IReceiveMiddlewareProvider @@ -99,7 +99,7 @@ public static TDescriptor AddExceptionPolicy( /// /// The consumer descriptor to configure. /// The descriptor for method chaining. - public static IConsumerDescriptor AddExceptionPolicy(this IConsumerDescriptor descriptor) + public static IConsumerDescriptor AddResilience(this IConsumerDescriptor descriptor) { descriptor.Extend().Configuration.Features.GetOrSet() .Configure(p => p.Default().Retry().ThenRedeliver()); @@ -112,7 +112,7 @@ public static IConsumerDescriptor AddExceptionPolicy(this IConsumerDescriptor de /// The consumer descriptor to configure. /// The action to configure exception policy options. /// The descriptor for method chaining. - public static IConsumerDescriptor AddExceptionPolicy( + public static IConsumerDescriptor AddResilience( this IConsumerDescriptor descriptor, Action configure) { diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs index fb60e9a78e5..f1732962d68 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs @@ -3,7 +3,7 @@ namespace Mocha; /// /// Provides retry state information to message handlers via /// context.Features.Get<RetryState>(). -/// Null if retry is not configured via AddExceptionPolicy. +/// Null if retry is not configured via AddResilience. /// public sealed class RetryRuntimeFeature { diff --git a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs index efbb531042e..482d9f6bd4b 100644 --- a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs +++ b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs @@ -19,7 +19,7 @@ public async Task Redelivery_Should_ScheduleRedelivery_When_HandlerFails() .AddSingleton(counter) .AddSingleton(recorder) .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => { p.On().Redeliver( [ @@ -55,7 +55,7 @@ public async Task Redelivery_Should_SkipRedelivery_When_ExceptionIsIgnored() await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => { p.On().Redeliver([TimeSpan.FromMilliseconds(1)]); p.On().DeadLetter(); @@ -108,7 +108,7 @@ public async Task Redelivery_Should_PropagateToFault_When_AllAttemptsExhausted() await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => { p.On().Redeliver( [ @@ -141,14 +141,14 @@ public async Task Redelivery_Should_UseEndpointOverride_When_EndpointConfigured( .AddSingleton(counter) .AddScoped() .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => p.On().Redeliver([TimeSpan.FromMilliseconds(1)])); // Override at transport level to discard all exceptions. builder.ConfigureMessageBus(b => b.AddHandler()); await using var provider = await builder - .AddInMemory(t => t.AddExceptionPolicy(p => + .AddInMemory(t => t.AddResilience(p => p.On().DeadLetter())) .BuildServiceProvider(); @@ -164,7 +164,7 @@ public async Task Redelivery_Should_UseEndpointOverride_When_EndpointConfigured( } [Fact] - public async Task Redelivery_Should_UseDefaults_When_ParameterlessAddExceptionPolicy() + public async Task Redelivery_Should_UseDefaults_When_ParameterlessAddResilience() { // arrange - defaults: 3 redelivery intervals from RedeliveryPolicyDefaults var counter = new InvocationCounter(); @@ -172,7 +172,7 @@ public async Task Redelivery_Should_UseDefaults_When_ParameterlessAddExceptionPo await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddExceptionPolicy() + .AddResilience() .AddEventHandler() .AddInMemory() .BuildServiceProvider(); diff --git a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs index 130e151cdf8..5acf11ee721 100644 --- a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs +++ b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs @@ -18,7 +18,7 @@ public async Task Retry_Should_RetryHandler_When_HandlerThrowsTransientException .AddSingleton(counter) .AddSingleton(recorder) .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => { p.On() .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); @@ -49,7 +49,7 @@ public async Task Retry_Should_PropagateToFault_When_AllRetriesExhausted() await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => { p.On() .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); @@ -78,7 +78,7 @@ public async Task Retry_Should_SkipRetry_When_ExceptionIsIgnored() await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => { p.On() .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); @@ -110,7 +110,7 @@ public async Task Retry_Should_SkipRetry_When_PredicateMatchesIgnoredException() await using var matchingProvider = await new ServiceCollection() .AddSingleton(matchingCounter) .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => { p.On() .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); @@ -133,7 +133,7 @@ public async Task Retry_Should_SkipRetry_When_PredicateMatchesIgnoredException() await using var nonMatchingProvider = await new ServiceCollection() .AddSingleton(nonMatchingCounter) .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => { p.On() .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); @@ -163,7 +163,7 @@ public async Task Retry_Should_UseConsumerOverride_When_ConsumerHasDifferentConf .AddSingleton(counter) .AddScoped() .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => { p.On() .Retry(2, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); @@ -173,7 +173,7 @@ public async Task Retry_Should_UseConsumerOverride_When_ConsumerHasDifferentConf { b.AddHandler(consumer => { - consumer.AddExceptionPolicy(p => + consumer.AddResilience(p => { p.On() .Retry(5, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); @@ -204,7 +204,7 @@ public async Task Retry_Should_ExposeRetryState_When_HandlerAccessesFeatures() .AddSingleton(stateCapture) .AddScoped() .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => { p.On() .Retry(2, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); @@ -240,7 +240,7 @@ public async Task Retry_Should_PassThrough_When_DisabledForConsumer() .AddSingleton(counter) .AddScoped() .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => { p.On() .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); @@ -248,7 +248,7 @@ public async Task Retry_Should_PassThrough_When_DisabledForConsumer() builder.ConfigureMessageBus(b => b.AddHandler(consumer => - consumer.AddExceptionPolicy(p => + consumer.AddResilience(p => p.On().Redeliver([TimeSpan.FromHours(1)])))); await using var provider = await builder.AddInMemory().BuildServiceProvider(); @@ -272,7 +272,7 @@ public async Task Retry_Should_UseExplicitIntervals_When_IntervalsConfigured() await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => { p.On().Retry( [ @@ -305,7 +305,7 @@ public async Task Retry_Should_RespectInheritance_When_BaseExceptionIgnored() await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddExceptionPolicy(p => + .AddResilience(p => { p.On() .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); @@ -327,14 +327,14 @@ public async Task Retry_Should_RespectInheritance_When_BaseExceptionIgnored() } [Fact] - public async Task Retry_Should_UseDefaults_When_ParameterlessAddExceptionPolicy() + public async Task Retry_Should_UseDefaults_When_ParameterlessAddResilience() { // arrange - default: 3 retries (from RetryPolicyDefaults.Attempts) var counter = new RetryInvocationCounter(); await using var provider = await new ServiceCollection() .AddSingleton(counter) .AddMessageBus() - .AddExceptionPolicy() + .AddResilience() .AddEventHandler() .AddInMemory() .BuildServiceProvider(); diff --git a/website/src/docs/mocha/v1/exception-policies.md b/website/src/docs/mocha/v1/exception-policies.md index b2760f87e7b..850a1f1a7cc 100644 --- a/website/src/docs/mocha/v1/exception-policies.md +++ b/website/src/docs/mocha/v1/exception-policies.md @@ -3,12 +3,12 @@ title: "Exception Policies" description: "Configure per-exception handling with composable retry, redelivery, and terminal actions." --- -Not every exception deserves the same treatment. A database deadlock might resolve on immediate retry. A downstream service outage needs minutes to recover. A validation error will never succeed no matter how many times you retry. Exception policies let you define per-exception handling strategies — retry, redeliver, dead-letter, or discard — as composable escalation chains in a single `AddExceptionPolicy` call. +Not every exception deserves the same treatment. A database deadlock might resolve on immediate retry. A downstream service outage needs minutes to recover. A validation error will never succeed no matter how many times you retry. Exception policies let you define per-exception handling strategies — retry, redeliver, dead-letter, or discard — as composable escalation chains in a single `AddResilience` call. ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy(policy => + .AddResilience(policy => { // Validation errors are permanent — route straight to the error endpoint policy.On().DeadLetter(); @@ -64,7 +64,11 @@ Exception matching respects inheritance. A policy on `NpgsqlException` also matc # Configure exception policies -`AddExceptionPolicy` is the single entry point for all exception handling configuration. There is no separate `AddRetry` or `AddRedelivery` call — retry and redelivery settings are configured per-exception within the policy. +`AddResilience` is the single entry point for all exception handling configuration. There is no separate `AddRetry` or `AddRedelivery` call — retry and redelivery settings are configured per-exception within the policy. + +:::note Replacement semantics +Calling `On()` for the same exception type replaces the previous rule for that type — last write wins. If you call `On()` twice without a predicate, the second call overwrites the first. The same applies to `Default()`: calling it again replaces the previous default rule. For example, the parameterless `AddResilience()` registers `Default().Retry().ThenRedeliver()`. If you later call `AddResilience(p => p.Default().Retry(5))`, the new default replaces the one registered by the parameterless overload. +::: ## Parameterless defaults @@ -73,7 +77,7 @@ The parameterless overload registers a catch-all `Default()` rule with both retr ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy() + .AddResilience() .AddEventHandler() .AddRabbitMQ(); ``` @@ -81,7 +85,7 @@ builder.Services This is equivalent to: ```csharp -.AddExceptionPolicy(policy => +.AddResilience(policy => { policy.Default().Retry().ThenRedeliver(); }) @@ -96,7 +100,7 @@ This is equivalent to: - **`On(predicate)`** — configures behavior for a specific exception type when a predicate matches. ```csharp -.AddExceptionPolicy(policy => +.AddResilience(policy => { policy.On().Retry(5).ThenRedeliver(); policy.On().Retry(3); @@ -111,7 +115,7 @@ Bus-level policies apply to all endpoints and all consumers across the entire me ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy(policy => + .AddResilience(policy => { policy.On().DeadLetter(); policy.On().Discard(); @@ -128,13 +132,13 @@ Override bus-level policies for a specific transport. Transport-level policies r ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy(policy => + .AddResilience(policy => { policy.Default().Retry().ThenRedeliver(); }) .AddRabbitMQ(transport => { - transport.AddExceptionPolicy(policy => + transport.AddResilience(policy => { policy.On().DeadLetter(); }); @@ -148,7 +152,7 @@ Override policies for a specific consumer. Consumer-level policies replace the b ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy(policy => + .AddResilience(policy => { policy.On() .Retry(2, TimeSpan.FromMilliseconds(100), RetryBackoffType.Constant); @@ -158,7 +162,7 @@ builder.Services.ConfigureMessageBus(bus => { bus.AddHandler(consumer => { - consumer.AddExceptionPolicy(policy => + consumer.AddResilience(policy => { policy.On() .Retry(5, TimeSpan.FromMilliseconds(500), RetryBackoffType.Exponential); @@ -467,15 +471,15 @@ policy.On().Retry(3); | ------------------ | ---------- | ------- | --------------------------------------------------- | | `ThenDeadLetter()` | - | `void` | Route to error endpoint after redelivery exhaustion | -## AddExceptionPolicy extensions - -| Target | Method | Description | -| ---------------------------- | ---------------------------------------------------- | ---------------------------------------------------------------------- | -| `IMessageBusBuilder` | `AddExceptionPolicy()` | Register default exception policies at the bus level | -| `IMessageBusBuilder` | `AddExceptionPolicy(Action)` | Configure exception policies at the bus level | -| `IMessageBusHostBuilder` | `AddExceptionPolicy()` | Register default exception policies at the host level | -| `IMessageBusHostBuilder` | `AddExceptionPolicy(Action)` | Configure exception policies at the host level | -| `IReceiveMiddlewareProvider` | `AddExceptionPolicy()` | Register default exception policies at the transport or endpoint level | -| `IReceiveMiddlewareProvider` | `AddExceptionPolicy(Action)` | Configure exception policies at the transport or endpoint level | -| `IConsumerDescriptor` | `AddExceptionPolicy()` | Register default exception policies at the consumer level | -| `IConsumerDescriptor` | `AddExceptionPolicy(Action)` | Configure exception policies at the consumer level | +## AddResilience extensions + +| Target | Method | Description | +| ---------------------------- | ----------------------------------------------- | ---------------------------------------------------------------------- | +| `IMessageBusBuilder` | `AddResilience()` | Register default exception policies at the bus level | +| `IMessageBusBuilder` | `AddResilience(Action)` | Configure exception policies at the bus level | +| `IMessageBusHostBuilder` | `AddResilience()` | Register default exception policies at the host level | +| `IMessageBusHostBuilder` | `AddResilience(Action)` | Configure exception policies at the host level | +| `IReceiveMiddlewareProvider` | `AddResilience()` | Register default exception policies at the transport or endpoint level | +| `IReceiveMiddlewareProvider` | `AddResilience(Action)` | Configure exception policies at the transport or endpoint level | +| `IConsumerDescriptor` | `AddResilience()` | Register default exception policies at the consumer level | +| `IConsumerDescriptor` | `AddResilience(Action)` | Configure exception policies at the consumer level | diff --git a/website/src/docs/mocha/v1/reliability.md b/website/src/docs/mocha/v1/reliability.md index 2f464af8cd3..387c537bacd 100644 --- a/website/src/docs/mocha/v1/reliability.md +++ b/website/src/docs/mocha/v1/reliability.md @@ -8,7 +8,7 @@ Messaging systems fail. Handlers throw exceptions, brokers go offline, databases ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy(policy => + .AddResilience(policy => { policy.On().DeadLetter(); policy.Default().Retry().ThenRedeliver(); @@ -167,19 +167,19 @@ Retry lives in the **consumer pipeline**, not the receive pipeline. This matters ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy() + .AddResilience() .AddEventHandler() .AddRabbitMQ(); ``` -The parameterless `AddExceptionPolicy()` registers a catch-all `Default()` rule with both retry and redelivery enabled. Retry defaults to 3 attempts, 200 ms base delay, exponential backoff, jitter enabled, and a 30-second maximum delay. These defaults handle the majority of transient failures without tuning. +The parameterless `AddResilience()` registers a catch-all `Default()` rule with both retry and redelivery enabled. Retry defaults to 3 attempts, 200 ms base delay, exponential backoff, jitter enabled, and a 30-second maximum delay. These defaults handle the majority of transient failures without tuning. ## Customize retry behavior ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy(policy => + .AddResilience(policy => { policy.Default() .Retry(5, TimeSpan.FromSeconds(1), RetryBackoffType.Exponential) @@ -196,7 +196,7 @@ When you need precise control over each delay, pass an array of intervals. The a ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy(policy => + .AddResilience(policy => { policy.Default().Retry( [ @@ -211,12 +211,12 @@ builder.Services ## Handle different exceptions differently -Not every exception should be retried. Validation errors, authorization failures, and other permanent errors waste retry budget. Use `AddExceptionPolicy` to define per-exception handling strategies. +Not every exception should be retried. Validation errors, authorization failures, and other permanent errors waste retry budget. Use `AddResilience` to define per-exception handling strategies. ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy(policy => + .AddResilience(policy => { // Never retry validation failures - route to error endpoint policy.On().DeadLetter(); @@ -247,11 +247,11 @@ The bus-level exception policy applies to all consumers. Override it for specifi ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy() // bus-level: default retry + redelivery for all consumers + .AddResilience() // bus-level: default retry + redelivery for all consumers .AddEventHandler(consumer => { // This consumer: 10 retries with longer delay, then redeliver - consumer.AddExceptionPolicy(policy => + consumer.AddResilience(policy => { policy.Default() .Retry(10, TimeSpan.FromSeconds(2)) @@ -261,7 +261,7 @@ builder.Services .AddEventHandler(consumer => { // This consumer: skip retry, redeliver only - consumer.AddExceptionPolicy(policy => + consumer.AddResilience(policy => { policy.Default().Redeliver(); }); @@ -315,12 +315,12 @@ All retries exhausted -> exception propagates to Redelivery ## Add redelivery with defaults -Redelivery is included when you call `AddExceptionPolicy()` without arguments or when you chain `.ThenRedeliver()` in an escalation chain. +Redelivery is included when you call `AddResilience()` without arguments or when you chain `.ThenRedeliver()` in an escalation chain. ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy() // Default() rule enables both retry and redelivery + .AddResilience() // Default() rule enables both retry and redelivery .AddEventHandler() .AddRabbitMQ(); ``` @@ -332,7 +332,7 @@ The default redelivery schedule is three attempts at 5 minutes, 15 minutes, and ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy(policy => + .AddResilience(policy => { policy.Default().Retry().ThenRedeliver( [ @@ -354,7 +354,7 @@ Instead of explicit intervals, configure a number of attempts and a base delay: ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy(policy => + .AddResilience(policy => { policy.Default() .Retry() @@ -371,7 +371,7 @@ To skip retry entirely and go straight to redelivery, use `.Redeliver()` directl ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy(policy => + .AddResilience(policy => { policy.Default().Redeliver(); }) @@ -388,10 +388,10 @@ Override exception policies at the transport level for more granular control. Tr ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy() // bus-level default + .AddResilience() // bus-level default .AddRabbitMQ(transport => { - transport.AddExceptionPolicy(policy => + transport.AddResilience(policy => { // Override for this transport: longer redelivery intervals policy.Default().Retry().ThenRedeliver( @@ -407,7 +407,7 @@ builder.Services Disable redelivery for a specific transport by configuring retry-only: ```csharp -transport.AddExceptionPolicy(policy => +transport.AddResilience(policy => { policy.Default().Retry(); }); @@ -463,7 +463,7 @@ With the defaults (3 retries, 3 redeliveries): `(3 + 1) x (3 + 1) = 16` total ha ```csharp builder.Services .AddMessageBus() - .AddExceptionPolicy(policy => + .AddResilience(policy => { policy.Default() .Retry(5, TimeSpan.FromSeconds(1), RetryBackoffType.Exponential) @@ -516,7 +516,7 @@ public class PaymentConsumer(ILogger logger) : IConsumer + .AddResilience(policy => { policy.On().DeadLetter(); policy.Default().Retry().ThenRedeliver(); From a699ee63ba9d255aa8a954402da5090e22e5b79e Mon Sep 17 00:00:00 2001 From: Pascal Senn Date: Tue, 7 Apr 2026 23:25:53 +0200 Subject: [PATCH 05/14] Implement retry logic and exception policies in ConsumerRetryMiddleware and add tests for RetryExecutor --- .../Consume/Retry/ConsumerRetryMiddleware.cs | 119 +-- .../Consume/Retry/ExceptionPolicyBuilder.cs | 51 +- .../Consume/Retry/RetryExecutor.cs | 137 +++ .../Retry/ExceptionPolicyMatcherTests.cs | 154 +++ .../Consume/Retry/RetryExecutorTests.cs | 909 ++++++++++++++++++ 5 files changed, 1235 insertions(+), 135 deletions(-) create mode 100644 src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs create mode 100644 src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/ExceptionPolicyMatcherTests.cs create mode 100644 src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs index 5ad358a7ebd..98b38a0cc85 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Mocha.Features; namespace Mocha; @@ -6,8 +7,7 @@ namespace Mocha; /// A consumer middleware that implements in-process retry with configurable backoff strategies /// when transient failures occur. /// -internal sealed class ConsumerRetryMiddleware( - IReadOnlyList exceptionPolicyRules) +internal sealed class ConsumerRetryMiddleware(IReadOnlyList exceptionPolicyRules) { public async ValueTask InvokeAsync(IConsumeContext context, ConsumerDelegate next) { @@ -21,109 +21,16 @@ public async ValueTask InvokeAsync(IConsumeContext context, ConsumerDelegate nex } // Expose retry state to handlers via features. - var retryState = new RetryRuntimeFeature { DelayedRetryCount = delayedRetryCount, ImmediateRetryCount = 0 }; - context.Features.Set(retryState); - - var attempts = 0; - - while (true) - { - try - { - await next(context); - return; - } - catch (Exception ex) - { - // Match exception against policy rules. - var rule = ExceptionPolicyMatcher.Match(exceptionPolicyRules, ex); - - // No matching rule — no policy for this exception, let it propagate. - if (rule is null) - { - throw; - } - - // Discard: swallow at consumer level so other consumers can still run. - if (rule.Terminal == TerminalAction.Discard) - { - return; - } - - // DeadLetter: don't retry, let it propagate to fault middleware. - if (rule.Terminal == TerminalAction.DeadLetter) - { - throw; - } - - // No retry configured for this rule, or retry explicitly disabled. - if (rule.Retry is null or { Enabled: false }) - { - throw; - } - - attempts++; - - // Use the rule's retry config (fully populated by the builder). - var retryConfig = rule.Retry; - - if (attempts > (retryConfig.Attempts ?? RetryPolicyDefaults.Attempts)) - { - throw; - } - - // Calculate delay. - var delay = CalculateDelay(attempts, retryConfig); - - // Update runtime feature. - retryState.ImmediateRetryCount = attempts; - - if (delay > TimeSpan.Zero) - { - await Task.Delay(delay, context.CancellationToken); - } - } - } - } - - private static TimeSpan CalculateDelay(int attempt, RetryPolicyConfig config) - { - // Explicit intervals take precedence. - if (config.Intervals is { Length: > 0 } intervals) - { - var index = Math.Min(attempt - 1, intervals.Length - 1); - return intervals[index]; - } - - // Calculate based on backoff type. - var baseDelay = config.Delay ?? RetryPolicyDefaults.Delay; - var backoff = config.Backoff ?? RetryPolicyDefaults.Backoff; - var maxDelay = config.MaxDelay ?? RetryPolicyDefaults.MaxDelay; - var useJitter = config.UseJitter ?? RetryPolicyDefaults.UseJitter; - - var delay = backoff switch - { - RetryBackoffType.Constant => baseDelay, - RetryBackoffType.Linear => baseDelay * attempt, - RetryBackoffType.Exponential => baseDelay * Math.Pow(2, attempt - 1), - _ => baseDelay * Math.Pow(2, attempt - 1) - }; - - // Cap at max delay. - if (delay > maxDelay) - { - delay = maxDelay; - } - - // Add jitter: +/- 25%. - if (useJitter) - { - var jitterRange = delay.TotalMilliseconds * 0.25; - var jitter = (Random.Shared.NextDouble() * 2 - 1) * jitterRange; - delay = TimeSpan.FromMilliseconds(Math.Max(0, delay.TotalMilliseconds + jitter)); - } - - return delay; + var retryState = context.Features.GetOrSet(); + retryState.DelayedRetryCount = delayedRetryCount; + retryState.ImmediateRetryCount = 0; + + await RetryExecutor.ExecuteAsync( + exceptionPolicyRules, + (next, context, retryState), + static (s) => s.next(s.context), + static (s, attempts) => s.retryState.ImmediateRetryCount = attempts, + context.CancellationToken); } public static ConsumerMiddlewareConfiguration Create() @@ -134,7 +41,7 @@ public static ConsumerMiddlewareConfiguration Create() if (feature is null) { - // No exception policy configured — skip retry middleware entirely. + // No exception policy configured - skip retry middleware entirely. return next; } diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs index 7605dc115c5..970dbd5db63 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs @@ -1,8 +1,9 @@ namespace Mocha; internal sealed class ExceptionPolicyBuilder - : IExceptionPolicyBuilder, IAfterRetryBuilder, IAfterRedeliveryBuilder - where TException : Exception + : IExceptionPolicyBuilder + , IAfterRetryBuilder + , IAfterRedeliveryBuilder where TException : Exception { private readonly ExceptionPolicyRule _rule; private readonly List _rules; @@ -14,27 +15,6 @@ internal ExceptionPolicyBuilder(ExceptionPolicyRule rule, List= 0; i--) - { - if (_rules[i].ExceptionType == _rule.ExceptionType - && _rules[i].Predicate == _rule.Predicate) - { - _rules.RemoveAt(i); - } - } - - _rules.Add(_rule); - _committed = true; - } - } - - // IExceptionPolicyBuilder - public void Discard() { EnsureCommitted(); @@ -98,11 +78,7 @@ public IAfterRetryBuilder Retry( public IAfterRetryBuilder Retry(TimeSpan[] intervals) { EnsureCommitted(); - _rule.Retry = new RetryPolicyConfig - { - Intervals = intervals, - Attempts = intervals.Length - }; + _rule.Retry = new RetryPolicyConfig { Intervals = intervals, Attempts = intervals.Length }; _rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; return this; } @@ -192,5 +168,22 @@ public void ThenDeadLetter() _rule.Terminal = TerminalAction.DeadLetter; } - // IAfterRedeliveryBuilder.ThenDeadLetter() is the same method + private void EnsureCommitted() + { + if (!_committed) + { + // Replace any existing rule for the same exception type and predicate. + for (var i = _rules.Count - 1; i >= 0; i--) + { + if (_rules[i].ExceptionType == _rule.ExceptionType + && _rules[i].Predicate == _rule.Predicate) + { + _rules.RemoveAt(i); + } + } + + _rules.Add(_rule); + _committed = true; + } + } } diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs new file mode 100644 index 00000000000..87ea7d6845a --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs @@ -0,0 +1,137 @@ +namespace Mocha; + +/// +/// Executes an action with retry logic based on exception policy rules. +/// Returns normally on success or discard. Throws on dead-letter, no match, or exhausted retries. +/// +internal static class RetryExecutor +{ + public static ValueTask ExecuteAsync( + IReadOnlyList rules, + TState state, + Func action, + Action? onRetry, + CancellationToken cancellationToken) + { + return ExecuteAsync(rules, state, action, onRetry, TimeProvider.System, cancellationToken); + } + + public static async ValueTask ExecuteAsync( + IReadOnlyList rules, + TState state, + Func action, + Action? onRetry, + TimeProvider timeProvider, + CancellationToken cancellationToken) + { + var attempts = 0; + + while (true) + { + try + { + await action(state); + return; + } + catch (Exception ex) + { + // Match exception against policy rules. + var rule = ExceptionPolicyMatcher.Match(rules, ex); + + // No matching rule - no policy for this exception, let it propagate. + if (rule is null) + { + throw; + } + + // Discard: swallow the exception (always immediate, no retry chaining exists). + if (rule.Terminal == TerminalAction.Discard) + { + return; + } + + // No retry configured for this rule, or retry explicitly disabled. + // Terminal (e.g. DeadLetter) is metadata for the fault middleware downstream. + if (rule.Retry is null or { Enabled: false }) + { + throw; + } + + attempts++; + + // Use the rule's retry config (fully populated by the builder). + var retryConfig = rule.Retry; + + if (attempts > (retryConfig.Attempts ?? RetryPolicyDefaults.Attempts)) + { + throw; + } + + // Calculate delay. + var delay = CalculateDelay(attempts, retryConfig); + + // Notify caller of retry attempt. + onRetry?.Invoke(state, attempts); + + if (delay > TimeSpan.Zero) + { + await DelayAsync(timeProvider, delay, cancellationToken); + } + } + } + } + + private static async Task DelayAsync(TimeProvider timeProvider, TimeSpan delay, CancellationToken cancellationToken) + { + var tcs = new TaskCompletionSource(); + await using var timer = timeProvider.CreateTimer( + static s => ((TaskCompletionSource)s!).TrySetResult(), + tcs, + delay, + Timeout.InfiniteTimeSpan); + await using var registration = cancellationToken.Register( + static s => ((TaskCompletionSource)s!).TrySetCanceled(), + tcs); + await tcs.Task.ConfigureAwait(false); + } + + internal static TimeSpan CalculateDelay(int attempt, RetryPolicyConfig config) + { + // Explicit intervals take precedence. + if (config.Intervals is { Length: > 0 } intervals) + { + var index = Math.Min(attempt - 1, intervals.Length - 1); + return intervals[index]; + } + + // Calculate based on backoff type. + var baseDelay = config.Delay ?? RetryPolicyDefaults.Delay; + var backoff = config.Backoff ?? RetryPolicyDefaults.Backoff; + var maxDelay = config.MaxDelay ?? RetryPolicyDefaults.MaxDelay; + var useJitter = config.UseJitter ?? RetryPolicyDefaults.UseJitter; + + var delay = backoff switch + { + RetryBackoffType.Constant => baseDelay, + RetryBackoffType.Linear => baseDelay * attempt, + RetryBackoffType.Exponential => baseDelay * Math.Pow(2, attempt - 1), + _ => baseDelay * Math.Pow(2, attempt - 1) + }; + + // Cap at max delay. + if (delay > maxDelay) + { + delay = maxDelay; + } + + // Add jitter: +/- 25%. + if (useJitter) + { + var jitterRange = delay.TotalMilliseconds * 0.25; + var jitter = ((Random.Shared.NextDouble() * 2) - 1) * jitterRange; + delay = TimeSpan.FromMilliseconds(Math.Max(0, delay.TotalMilliseconds + jitter)); + } + + return delay; + } +} diff --git a/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/ExceptionPolicyMatcherTests.cs b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/ExceptionPolicyMatcherTests.cs new file mode 100644 index 00000000000..4e707f0d2c4 --- /dev/null +++ b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/ExceptionPolicyMatcherTests.cs @@ -0,0 +1,154 @@ +namespace Mocha.Tests.Middlewares.Consume.Retry; + +public sealed class ExceptionPolicyMatcherTests +{ + [Fact] + public void Match_Should_ReturnNull_When_RulesListIsEmpty() + { + // arrange + var rules = Array.Empty(); + var exception = new InvalidOperationException("test"); + + // act + var result = ExceptionPolicyMatcher.Match(rules, exception); + + // assert + Assert.Null(result); + } + + [Fact] + public void Match_Should_ReturnNull_When_NoRuleMatchesExceptionType() + { + // arrange + var rules = new[] + { + CreateRule() + }; + var exception = new InvalidOperationException("test"); + + // act + var result = ExceptionPolicyMatcher.Match(rules, exception); + + // assert + Assert.Null(result); + } + + [Fact] + public void Match_Should_ReturnRule_When_ExceptionTypeMatchesExactly() + { + // arrange + var rule = CreateRule(); + var rules = new[] { rule }; + var exception = new InvalidOperationException("test"); + + // act + var result = ExceptionPolicyMatcher.Match(rules, exception); + + // assert + Assert.Same(rule, result); + } + + [Fact] + public void Match_Should_ReturnRule_When_ExceptionIsDerivedType() + { + // arrange + var rule = CreateRule(); + var rules = new[] { rule }; + var exception = new InvalidOperationException("test"); + + // act + var result = ExceptionPolicyMatcher.Match(rules, exception); + + // assert + Assert.Same(rule, result); + } + + [Fact] + public void Match_Should_ReturnMostSpecificRule_When_MultipleRulesMatchAtDifferentDepths() + { + // arrange + var baseRule = CreateRule(); + var specificRule = CreateRule(); + var rules = new[] { baseRule, specificRule }; + var exception = new InvalidOperationException("test"); + + // act + var result = ExceptionPolicyMatcher.Match(rules, exception); + + // assert + Assert.Same(specificRule, result); + } + + [Fact] + public void Match_Should_ReturnNull_When_TypeMatchesButPredicateReturnsFalse() + { + // arrange + var rule = CreateRule(predicate: _ => false); + var rules = new[] { rule }; + var exception = new InvalidOperationException("test"); + + // act + var result = ExceptionPolicyMatcher.Match(rules, exception); + + // assert + Assert.Null(result); + } + + [Fact] + public void Match_Should_ReturnRule_When_TypeMatchesAndPredicateReturnsTrue() + { + // arrange + var rule = CreateRule(predicate: _ => true); + var rules = new[] { rule }; + var exception = new InvalidOperationException("test"); + + // act + var result = ExceptionPolicyMatcher.Match(rules, exception); + + // assert + Assert.Same(rule, result); + } + + [Fact] + public void Match_Should_ReturnMoreSpecificRule_When_LessSpecificRuleHasPredicate() + { + // arrange + var baseRuleWithPredicate = CreateRule(predicate: _ => true); + var specificRule = CreateRule(); + var rules = new[] { baseRuleWithPredicate, specificRule }; + var exception = new InvalidOperationException("test"); + + // act + var result = ExceptionPolicyMatcher.Match(rules, exception); + + // assert + Assert.Same(specificRule, result); + } + + [Fact] + public void Match_Should_ReturnPassingRule_When_TwoRulesForSameTypeAndOnePredicateFails() + { + // arrange + var failingRule = CreateRule(predicate: _ => false); + var passingRule = CreateRule(predicate: _ => true); + var rules = new[] { failingRule, passingRule }; + var exception = new ArgumentNullException("param"); + + // act + var result = ExceptionPolicyMatcher.Match(rules, exception); + + // assert + Assert.Same(passingRule, result); + } + + private static ExceptionPolicyRule CreateRule( + Func? predicate = null) + where TException : Exception + { + return new ExceptionPolicyRule + { + ExceptionType = typeof(TException), + Predicate = predicate + }; + } +} diff --git a/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs new file mode 100644 index 00000000000..348651cf4be --- /dev/null +++ b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs @@ -0,0 +1,909 @@ +using Microsoft.Extensions.Time.Testing; + +namespace Mocha.Tests.Middlewares.Consume.Retry; + +public sealed class RetryExecutorTests +{ + [Fact] + public async Task ExecuteAsync_Should_Succeed_When_ActionDoesNotThrow() + { + // arrange + var rules = BuildRules(p => p.On().Retry()); + var counter = new Counter(); + + // act + await RetryExecutor.ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + return ValueTask.CompletedTask; + }, + onRetry: null, + cancellationToken: default); + + // assert + Assert.Equal(1, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_Throw_When_NoRuleMatchesException() + { + // arrange + var rules = BuildRules(p => p.On().Retry()); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + 0, + static (_) => throw new InvalidOperationException("no match"), + onRetry: null, + cancellationToken: default) + .AsTask() + ); + } + + [Fact] + public async Task ExecuteAsync_Should_Return_When_TerminalIsDiscard() + { + // arrange + var rules = BuildRules(p => p.On().Discard()); + + // act - should not throw + await RetryExecutor.ExecuteAsync( + rules, + 0, + static (_) => throw new InvalidOperationException("discard me"), + onRetry: null, + cancellationToken: default); + } + + [Fact] + public async Task ExecuteAsync_Should_Throw_When_TerminalIsDeadLetter() + { + // arrange + var rules = BuildRules(p => p.On().DeadLetter()); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + 0, + static (_) => throw new InvalidOperationException("dead letter"), + onRetry: null, + cancellationToken: default) + .AsTask() + ); + } + + [Fact] + public async Task ExecuteAsync_Should_Throw_When_RetryIsDisabled() + { + // arrange - Redeliver() sets Retry.Enabled = false + var rules = BuildRules(p => p.On().Redeliver()); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + 0, + static (_) => throw new InvalidOperationException("no retry"), + onRetry: null, + cancellationToken: default) + .AsTask() + ); + } + + [Fact] + public async Task ExecuteAsync_Should_RetryAndSucceed_When_ActionFailsThenSucceeds() + { + // arrange + var rules = BuildRules(p => + p.On().Retry(3, TimeSpan.Zero, RetryBackoffType.Constant) + ); + var counter = new Counter(); + + // act + await RetryExecutor.ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + + if (s.Count == 1) + { + throw new InvalidOperationException("transient"); + } + + return ValueTask.CompletedTask; + }, + onRetry: null, + cancellationToken: default); + + // assert + Assert.Equal(2, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_Throw_When_AllRetriesExhausted() + { + // arrange + var rules = BuildRules(p => + p.On().Retry(2, TimeSpan.Zero, RetryBackoffType.Constant) + ); + var counter = new Counter(); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new InvalidOperationException("always fails"); + }, + onRetry: null, + cancellationToken: default) + .AsTask() + ); + + Assert.Equal(3, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_InvokeOnRetry_When_Retrying() + { + // arrange + var rules = BuildRules(p => + p.On().Retry(3, TimeSpan.Zero, RetryBackoffType.Constant) + ); + var counter = new Counter(); + var retryAttempts = new List(); + + // act + await RetryExecutor.ExecuteAsync( + rules, + (Counter: counter, Attempts: retryAttempts), + static (s) => + { + s.Counter.Increment(); + + if (s.Counter.Count <= 2) + { + throw new InvalidOperationException("transient"); + } + + return ValueTask.CompletedTask; + }, + onRetry: static (s, attempt) => s.Attempts.Add(attempt), + cancellationToken: default); + + // assert + Assert.Equal(3, counter.Count); + Assert.Equal(2, retryAttempts.Count); + Assert.Equal(1, retryAttempts[0]); + Assert.Equal(2, retryAttempts[1]); + } + + [Fact] + public async Task ExecuteAsync_Should_NotInvokeOnRetry_When_ActionSucceeds() + { + // arrange + var rules = BuildRules(p => + p.On().Retry(3, TimeSpan.Zero, RetryBackoffType.Constant) + ); + + // act + await RetryExecutor.ExecuteAsync( + rules, + 0, + static (_) => ValueTask.CompletedTask, + onRetry: static (_, _) => throw new InvalidOperationException("should not be called"), + cancellationToken: default); + } + + // -- CalculateDelay tests -- + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(5)] + public void CalculateDelay_Should_ReturnConstantDelay_When_BackoffIsConstant(int attempt) + { + // arrange + var config = new RetryPolicyConfig + { + Backoff = RetryBackoffType.Constant, + Delay = TimeSpan.FromMilliseconds(100), + UseJitter = false, + MaxDelay = TimeSpan.FromSeconds(30) + }; + + // act + var delay = RetryExecutor.CalculateDelay(attempt, config); + + // assert + Assert.Equal(TimeSpan.FromMilliseconds(100), delay); + } + + [Fact] + public void CalculateDelay_Should_ReturnLinearDelay_When_BackoffIsLinear() + { + // arrange + var config = new RetryPolicyConfig + { + Backoff = RetryBackoffType.Linear, + Delay = TimeSpan.FromMilliseconds(100), + UseJitter = false, + MaxDelay = TimeSpan.FromSeconds(30) + }; + + // act & assert + Assert.Equal(TimeSpan.FromMilliseconds(100), RetryExecutor.CalculateDelay(1, config)); + Assert.Equal(TimeSpan.FromMilliseconds(200), RetryExecutor.CalculateDelay(2, config)); + Assert.Equal(TimeSpan.FromMilliseconds(300), RetryExecutor.CalculateDelay(3, config)); + } + + [Fact] + public void CalculateDelay_Should_ReturnExponentialDelay_When_BackoffIsExponential() + { + // arrange + var config = new RetryPolicyConfig + { + Backoff = RetryBackoffType.Exponential, + Delay = TimeSpan.FromMilliseconds(100), + UseJitter = false, + MaxDelay = TimeSpan.FromSeconds(30) + }; + + // act & assert + Assert.Equal(TimeSpan.FromMilliseconds(100), RetryExecutor.CalculateDelay(1, config)); + Assert.Equal(TimeSpan.FromMilliseconds(200), RetryExecutor.CalculateDelay(2, config)); + Assert.Equal(TimeSpan.FromMilliseconds(400), RetryExecutor.CalculateDelay(3, config)); + } + + [Fact] + public void CalculateDelay_Should_CapAtMaxDelay_When_DelayExceedsMax() + { + // arrange + var config = new RetryPolicyConfig + { + Backoff = RetryBackoffType.Exponential, + Delay = TimeSpan.FromSeconds(1), + MaxDelay = TimeSpan.FromSeconds(2), + UseJitter = false + }; + + // act + var delay = RetryExecutor.CalculateDelay(10, config); + + // assert + Assert.Equal(TimeSpan.FromSeconds(2), delay); + } + + [Fact] + public void CalculateDelay_Should_UseExplicitIntervals_When_IntervalsProvided() + { + // arrange + var config = new RetryPolicyConfig + { + Intervals = [TimeSpan.FromMilliseconds(10), TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(100)], + UseJitter = false + }; + + // act & assert + Assert.Equal(TimeSpan.FromMilliseconds(10), RetryExecutor.CalculateDelay(1, config)); + Assert.Equal(TimeSpan.FromMilliseconds(50), RetryExecutor.CalculateDelay(2, config)); + Assert.Equal(TimeSpan.FromMilliseconds(100), RetryExecutor.CalculateDelay(3, config)); + } + + [Fact] + public void CalculateDelay_Should_ClampToLastInterval_When_AttemptExceedsIntervalCount() + { + // arrange + var config = new RetryPolicyConfig + { + Intervals = [TimeSpan.FromMilliseconds(10), TimeSpan.FromMilliseconds(50)], + UseJitter = false + }; + + // act + var delay = RetryExecutor.CalculateDelay(3, config); + + // assert + Assert.Equal(TimeSpan.FromMilliseconds(50), delay); + } + + // -- Multi-rule policy scenarios -- + + [Fact] + public async Task ExecuteAsync_Should_Discard_When_SpecificExceptionMatchesDiscardRule() + { + // arrange + var rules = BuildRules(p => + { + p.On().Retry(3, TimeSpan.Zero, RetryBackoffType.Constant); + p.On().Discard(); + }); + var counter = new Counter(); + + // act + await RetryExecutor.ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new ArgumentNullException("param"); + }, + onRetry: null, + cancellationToken: default); + + // assert + Assert.Equal(1, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_DeadLetter_When_SpecificExceptionMatchesDeadLetterRule() + { + // arrange + var rules = BuildRules(p => + { + p.On().Retry(3, TimeSpan.Zero, RetryBackoffType.Constant); + p.On().DeadLetter(); + }); + var counter = new Counter(); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new InvalidOperationException("dead letter"); + }, + onRetry: null, + cancellationToken: default) + .AsTask() + ); + + Assert.Equal(1, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_RetryWithBaseRule_When_DerivedExceptionHasNoSpecificRule() + { + // arrange + var rules = BuildRules(p => p.On().Retry(2, TimeSpan.Zero, RetryBackoffType.Constant)); + var counter = new Counter(); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new ArgumentException("derived"); + }, + onRetry: null, + cancellationToken: default) + .AsTask() + ); + + Assert.Equal(3, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_UseMostSpecificRule_When_MultipleRulesMatch() + { + // arrange + var rules = BuildRules(p => + { + p.On().Retry(5, TimeSpan.Zero, RetryBackoffType.Constant); + p.On().Retry(1, TimeSpan.Zero, RetryBackoffType.Constant); + }); + var counter = new Counter(); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new InvalidOperationException("specific"); + }, + onRetry: null, + cancellationToken: default) + .AsTask() + ); + + Assert.Equal(2, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_Discard_When_PredicateMatches() + { + // arrange + var rules = BuildRules(p => + p.On(static ex => ex.Message.Contains("transient")).Discard() + ); + + // act - should not throw + await RetryExecutor.ExecuteAsync( + rules, + 0, + static (_) => throw new InvalidOperationException("transient failure"), + onRetry: null, + cancellationToken: default); + } + + [Fact] + public async Task ExecuteAsync_Should_Throw_When_PredicateDoesNotMatch() + { + // arrange + var rules = BuildRules(p => + p.On(static ex => ex.Message.Contains("transient")).Discard() + ); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + 0, + static (_) => throw new InvalidOperationException("permanent failure"), + onRetry: null, + cancellationToken: default) + .AsTask() + ); + } + + [Fact] + public async Task ExecuteAsync_Should_RetryThenDiscard_When_DifferentExceptionsThrown() + { + // arrange + var rules = BuildRules(p => + { + p.On().Retry(3, TimeSpan.Zero, RetryBackoffType.Constant); + p.On().Discard(); + }); + var counter = new Counter(); + + // act + await RetryExecutor.ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + + if (s.Count == 1) + { + throw new InvalidOperationException("retry this"); + } + + throw new ArgumentException("discard this"); + }, + onRetry: null, + cancellationToken: default); + + // assert + Assert.Equal(2, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_RetryThenSucceed_When_ExceptionChangesOnRetry() + { + // arrange + var rules = BuildRules(p => + p.On().Retry(3, TimeSpan.Zero, RetryBackoffType.Constant) + ); + var counter = new Counter(); + + // act + await RetryExecutor.ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + + if (s.Count <= 2) + { + throw new InvalidOperationException("transient"); + } + + return ValueTask.CompletedTask; + }, + onRetry: null, + cancellationToken: default); + + // assert + Assert.Equal(3, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_Throw_When_RulesListIsEmpty() + { + // arrange + var rules = BuildRules(_ => { }); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + 0, + static (_) => throw new InvalidOperationException("no rules"), + onRetry: null, + cancellationToken: default) + .AsTask() + ); + } + + [Fact] + public async Task ExecuteAsync_Should_RetryExactlyConfiguredAttempts_When_AlwaysFailing() + { + // arrange + var rules = BuildRules(p => p.On().Retry(5, TimeSpan.Zero, RetryBackoffType.Constant)); + var counter = new Counter(); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new InvalidOperationException("always fails"); + }, + onRetry: null, + cancellationToken: default) + .AsTask() + ); + + Assert.Equal(6, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_TrackCorrectAttemptNumbers_When_Retrying() + { + // arrange + var rules = BuildRules(p => p.On().Retry(4, TimeSpan.Zero, RetryBackoffType.Constant)); + var counter = new Counter(); + var retryAttempts = new List(); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + (Counter: counter, Attempts: retryAttempts), + static (s) => + { + s.Counter.Increment(); + throw new InvalidOperationException("always fails"); + }, + onRetry: static (s, attempt) => s.Attempts.Add(attempt), + cancellationToken: default) + .AsTask() + ); + + Assert.Equal([1, 2, 3, 4], retryAttempts); + } + + [Fact] + public async Task ExecuteAsync_Should_UseDefaultAttempts_When_ParameterlessRetry() + { + // arrange - parameterless Retry() uses RetryPolicyDefaults.Attempts (3) + var rules = BuildRules(p => p.On().Retry()); + var counter = new Counter(); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new InvalidOperationException("always fails"); + }, + onRetry: null, + cancellationToken: default) + .AsTask() + ); + + // RetryPolicyDefaults.Attempts is 3, so 1 original + 3 retries = 4 + Assert.Equal(4, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_ThrowOperationCanceled_When_CancellationRequestedDuringDelay() + { + // arrange + var timeProvider = new FakeTimeProvider(); + var rules = BuildRules(p => p.On().Retry(3, TimeSpan.FromSeconds(10), RetryBackoffType.Constant)); + using var cts = new CancellationTokenSource(); + var counter = new Counter(); + + // act - start the executor, it will block on the first retry delay + var task = RetryExecutor.ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new InvalidOperationException("fail"); + }, + onRetry: null, + timeProvider, + cts.Token); + + // Cancel while waiting for the delay + cts.Cancel(); + + // assert + await Assert.ThrowsAnyAsync(() => task.AsTask()); + Assert.Equal(1, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_NotComplete_When_DelayHasNotElapsed() + { + // arrange + var timeProvider = new FakeTimeProvider(); + var rules = BuildRules(p => + p.On().Retry(1, TimeSpan.FromSeconds(5), RetryBackoffType.Constant) + ); + + // act - start the executor; it will block on the retry delay + var task = RetryExecutor.ExecuteAsync( + rules, + 0, + static (_) => throw new InvalidOperationException("fail"), + onRetry: null, + timeProvider, + cancellationToken: default); + + // assert - task should not complete until time advances + Assert.False(task.IsCompleted); + + // Advance past the delay to unblock + timeProvider.Advance(TimeSpan.FromSeconds(5)); + // Second attempt also fails, retries exhausted - should throw + await Assert.ThrowsAsync(() => task.AsTask()); + } + + [Fact] + public async Task ExecuteAsync_Should_RetryWithExplicitIntervals_When_IntervalsConfigured() + { + // arrange + var rules = BuildRules(p => p.On().Retry([TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero])); + var counter = new Counter(); + + // act + await RetryExecutor.ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + + if (s.Count <= 2) + { + throw new InvalidOperationException("transient"); + } + + return ValueTask.CompletedTask; + }, + onRetry: null, + cancellationToken: default); + + // assert + Assert.Equal(3, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_PropagateOriginalException_When_RetriesExhausted() + { + // arrange + var rules = BuildRules(p => p.On().Retry(1, TimeSpan.Zero, RetryBackoffType.Constant)); + + // act & assert + var ex = await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + 0, + static (_) => throw new InvalidOperationException("specific message"), + onRetry: null, + cancellationToken: default) + .AsTask() + ); + + Assert.Equal("specific message", ex.Message); + } + + [Fact] + public async Task ExecuteAsync_Should_UseDefaultPolicy_When_DefaultConfigured() + { + // arrange - Default() is equivalent to On() + var rules = BuildRules(p => p.Default().Retry(2, TimeSpan.Zero, RetryBackoffType.Constant)); + var counter = new Counter(); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new ArgumentException("caught by default"); + }, + onRetry: null, + cancellationToken: default) + .AsTask() + ); + + Assert.Equal(3, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_OverrideDefaultWithSpecific_When_BothConfigured() + { + // arrange + var rules = BuildRules(p => + { + p.Default().Retry(5, TimeSpan.Zero, RetryBackoffType.Constant); + p.On().Discard(); + }); + var counter = new Counter(); + + // act - InvalidOperationException should be discarded, not retried + await RetryExecutor.ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new InvalidOperationException("discarded despite default retry"); + }, + onRetry: null, + cancellationToken: default); + + Assert.Equal(1, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_FallThroughToDefault_When_NoSpecificRuleMatches() + { + // arrange + var rules = BuildRules(p => + { + p.On().Discard(); + p.Default().Retry(2, TimeSpan.Zero, RetryBackoffType.Constant); + }); + var counter = new Counter(); + + // act & assert - ArgumentException falls through to Default (Exception) rule + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new ArgumentException("falls to default"); + }, + onRetry: null, + cancellationToken: default) + .AsTask() + ); + + Assert.Equal(3, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_RetryThenThrow_When_RetryWithThenDeadLetter() + { + // arrange - ThenDeadLetter() is metadata for fault middleware, + // the executor should still retry before propagating. + var rules = BuildRules(p => + p.On().Retry(2, TimeSpan.Zero, RetryBackoffType.Constant).ThenDeadLetter() + ); + var counter = new Counter(); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new InvalidOperationException("exhaust then dead letter"); + }, + onRetry: null, + cancellationToken: default) + .AsTask() + ); + + // 1 original + 2 retries = 3 + Assert.Equal(3, counter.Count); + } + + [Fact] + public async Task ExecuteAsync_Should_RetryThenThrow_When_FullChainConfigured() + { + // arrange - Retry(2).ThenRedeliver().ThenDeadLetter() + // The executor only handles retry; redelivery and terminal are for other layers. + var rules = BuildRules(p => + p.On() + .Retry(2, TimeSpan.Zero, RetryBackoffType.Constant) + .ThenRedeliver() + .ThenDeadLetter() + ); + var counter = new Counter(); + + // act & assert + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new InvalidOperationException("full chain"); + }, + onRetry: null, + cancellationToken: default) + .AsTask() + ); + + // 1 original + 2 retries = 3 + Assert.Equal(3, counter.Count); + } + + // -- Helpers -- + + private static IReadOnlyList BuildRules(Action configure) + { + var feature = new ExceptionPolicyFeature(); + feature.Configure(configure); + return feature.Rules; + } + + private sealed class Counter + { + public int Count { get; private set; } + + public void Increment() => Count++; + } +} From 4a2b57ec9d60f29df87ab6f76561f8aa8d3300f6 Mon Sep 17 00:00:00 2001 From: Pascal Senn Date: Wed, 8 Apr 2026 00:29:13 +0200 Subject: [PATCH 06/14] Refactor exception policy feature resolution and clean up unused consumer resilience methods in middleware --- .../Consume/Retry/ConsumerRetryMiddleware.cs | 14 +--- .../ResilienceConfigurationExtensions.cs | 29 +------- .../Behaviors/RetryTests.cs | 74 ------------------- 3 files changed, 2 insertions(+), 115 deletions(-) diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs index 98b38a0cc85..a60a36f1e23 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs @@ -55,24 +55,12 @@ public static ConsumerMiddlewareConfiguration Create() file static class Extensions { /// - /// Resolves exception policy feature with the most specific scope taking precedence. - /// Consumer-level ExceptionPolicyFeature overrides bus-level. + /// Resolves the bus-level exception policy feature, if configured. /// public static ExceptionPolicyFeature? GetExceptionPolicyFeature(this ConsumerMiddlewareFactoryContext context) { var busFeatures = context.Services.GetRequiredService(); - // Consumer -> bus (most specific first). - var config = context.Consumer.Configuration; - if (config is not null) - { - var consumerFeatures = config.GetFeatures(); - if (consumerFeatures.TryGet(out ExceptionPolicyFeature? consumerFeature)) - { - return consumerFeature; - } - } - if (busFeatures.TryGet(out ExceptionPolicyFeature? busFeature)) { return busFeature; diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ResilienceConfigurationExtensions.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ResilienceConfigurationExtensions.cs index e1a0176d43e..b7af55546cd 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ResilienceConfigurationExtensions.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ResilienceConfigurationExtensions.cs @@ -4,7 +4,7 @@ namespace Mocha; /// /// Provides extension methods for configuring exception policies including retry, redelivery, -/// and per-exception rules on message bus builders, host builders, descriptors, and consumers. +/// and per-exception rules on message bus builders, host builders, and descriptors. /// public static class ResilienceConfigurationExtensions { @@ -92,31 +92,4 @@ public static TDescriptor AddResilience( descriptor.Extend().Configuration.Features.GetOrSet().Configure(configure); return descriptor; } - - /// - /// Adds exception policy configuration to a specific consumer with default settings. - /// Registers a catch-all rule for with default retry and redelivery. - /// - /// The consumer descriptor to configure. - /// The descriptor for method chaining. - public static IConsumerDescriptor AddResilience(this IConsumerDescriptor descriptor) - { - descriptor.Extend().Configuration.Features.GetOrSet() - .Configure(p => p.Default().Retry().ThenRedeliver()); - return descriptor; - } - - /// - /// Adds exception policy configuration to a specific consumer. - /// - /// The consumer descriptor to configure. - /// The action to configure exception policy options. - /// The descriptor for method chaining. - public static IConsumerDescriptor AddResilience( - this IConsumerDescriptor descriptor, - Action configure) - { - descriptor.Extend().Configuration.Features.GetOrSet().Configure(configure); - return descriptor; - } } diff --git a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs index 5acf11ee721..7155eaba98b 100644 --- a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs +++ b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs @@ -154,47 +154,6 @@ await nonMatchingCounter.WaitForCountAsync(4, s_timeout), $"Expected 4 invocations for non-matching predicate, but got {nonMatchingCounter.Count}"); } - [Fact] - public async Task Retry_Should_UseConsumerOverride_When_ConsumerHasDifferentConfig() - { - // arrange - bus-level: 2 retries, consumer-level: 5 retries - var counter = new RetryInvocationCounter(); - var builder = new ServiceCollection() - .AddSingleton(counter) - .AddScoped() - .AddMessageBus() - .AddResilience(p => - { - p.On() - .Retry(2, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); - }); - - builder.ConfigureMessageBus(b => - { - b.AddHandler(consumer => - { - consumer.AddResilience(p => - { - p.On() - .Retry(5, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); - }); - }); - }); - - await using var provider = await builder.AddInMemory().BuildServiceProvider(); - - using var scope = provider.CreateScope(); - var bus = scope.ServiceProvider.GetRequiredService(); - - // act - await bus.PublishAsync(new OrderCreated { OrderId = "ORD-OVERRIDE" }, CancellationToken.None); - - // assert - consumer override: 1 original + 5 retries = 6 total invocations - Assert.True( - await counter.WaitForCountAsync(6, s_timeout), - $"Expected 6 invocations (1 original + 5 consumer-level retries), but got {counter.Count}"); - } - [Fact] public async Task Retry_Should_ExposeRetryState_When_HandlerAccessesFeatures() { @@ -231,39 +190,6 @@ await stateCapture.WaitForCountAsync(3, s_timeout), Assert.Equal(2, states[2]); // second retry } - [Fact] - public async Task Retry_Should_PassThrough_When_DisabledForConsumer() - { - // arrange - var counter = new RetryInvocationCounter(); - var builder = new ServiceCollection() - .AddSingleton(counter) - .AddScoped() - .AddMessageBus() - .AddResilience(p => - { - p.On() - .Retry(3, TimeSpan.FromMilliseconds(1), RetryBackoffType.Constant); - }); - - builder.ConfigureMessageBus(b => - b.AddHandler(consumer => - consumer.AddResilience(p => - p.On().Redeliver([TimeSpan.FromHours(1)])))); - - await using var provider = await builder.AddInMemory().BuildServiceProvider(); - - using var scope = provider.CreateScope(); - var bus = scope.ServiceProvider.GetRequiredService(); - - // act - await bus.PublishAsync(new OrderCreated { OrderId = "ORD-DISABLED" }, CancellationToken.None); - - // assert - retry disabled via consumer override: only 1 invocation (redeliver does not cause immediate retry) - await Task.Delay(500); - Assert.Equal(1, counter.Count); - } - [Fact] public async Task Retry_Should_UseExplicitIntervals_When_IntervalsConfigured() { From 60aa6ca933b472b5114b1e90315f743e02a58b47 Mon Sep 17 00:00:00 2001 From: Pascal Senn Date: Wed, 8 Apr 2026 00:31:20 +0200 Subject: [PATCH 07/14] Refactor ExceptionPolicyBuilder to use parameters directly instead of private fields --- .../Consume/Retry/ExceptionPolicyBuilder.cs | 77 ++++++++++--------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs index 970dbd5db63..88ca2dd900c 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs @@ -1,36 +1,31 @@ namespace Mocha; -internal sealed class ExceptionPolicyBuilder +internal sealed class ExceptionPolicyBuilder(ExceptionPolicyRule rule, List rules) : IExceptionPolicyBuilder , IAfterRetryBuilder , IAfterRedeliveryBuilder where TException : Exception { - private readonly ExceptionPolicyRule _rule; - private readonly List _rules; private bool _committed; - internal ExceptionPolicyBuilder(ExceptionPolicyRule rule, List rules) - { - _rule = rule; - _rules = rules; - } - public void Discard() { EnsureCommitted(); - _rule.Terminal = TerminalAction.Discard; + + rule.Terminal = TerminalAction.Discard; } public void DeadLetter() { EnsureCommitted(); - _rule.Terminal = TerminalAction.DeadLetter; + + rule.Terminal = TerminalAction.DeadLetter; } public IAfterRetryBuilder Retry() { EnsureCommitted(); - _rule.Retry = new RetryPolicyConfig + + rule.Retry = new RetryPolicyConfig { Attempts = RetryPolicyDefaults.Attempts, Delay = RetryPolicyDefaults.Delay, @@ -38,14 +33,15 @@ public IAfterRetryBuilder Retry() UseJitter = RetryPolicyDefaults.UseJitter, MaxDelay = RetryPolicyDefaults.MaxDelay }; - _rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; + rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; return this; } public IAfterRetryBuilder Retry(int attempts) { EnsureCommitted(); - _rule.Retry = new RetryPolicyConfig + + rule.Retry = new RetryPolicyConfig { Attempts = attempts, Delay = RetryPolicyDefaults.Delay, @@ -53,7 +49,8 @@ public IAfterRetryBuilder Retry(int attempts) UseJitter = RetryPolicyDefaults.UseJitter, MaxDelay = RetryPolicyDefaults.MaxDelay }; - _rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; + rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; + return this; } @@ -63,7 +60,8 @@ public IAfterRetryBuilder Retry( RetryBackoffType backoff = RetryBackoffType.Exponential) { EnsureCommitted(); - _rule.Retry = new RetryPolicyConfig + + rule.Retry = new RetryPolicyConfig { Attempts = attempts, Delay = delay, @@ -71,51 +69,60 @@ public IAfterRetryBuilder Retry( UseJitter = RetryPolicyDefaults.UseJitter, MaxDelay = RetryPolicyDefaults.MaxDelay }; - _rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; + + rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; + return this; } public IAfterRetryBuilder Retry(TimeSpan[] intervals) { EnsureCommitted(); - _rule.Retry = new RetryPolicyConfig { Intervals = intervals, Attempts = intervals.Length }; - _rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; + + rule.Retry = new RetryPolicyConfig { Intervals = intervals, Attempts = intervals.Length }; + rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; + return this; } public IAfterRedeliveryBuilder Redeliver() { EnsureCommitted(); - _rule.Retry = new RetryPolicyConfig { Enabled = false }; - _rule.Redelivery = new RedeliveryPolicyConfig + + rule.Retry = new RetryPolicyConfig { Enabled = false }; + rule.Redelivery = new RedeliveryPolicyConfig { Intervals = RedeliveryPolicyDefaults.Intervals, Attempts = RedeliveryPolicyDefaults.Intervals.Length, UseJitter = RedeliveryPolicyDefaults.UseJitter, MaxDelay = RedeliveryPolicyDefaults.MaxDelay }; + return this; } public IAfterRedeliveryBuilder Redeliver(int attempts, TimeSpan baseDelay) { EnsureCommitted(); - _rule.Retry = new RetryPolicyConfig { Enabled = false }; - _rule.Redelivery = new RedeliveryPolicyConfig + + rule.Retry = new RetryPolicyConfig { Enabled = false }; + rule.Redelivery = new RedeliveryPolicyConfig { Attempts = attempts, BaseDelay = baseDelay, UseJitter = RedeliveryPolicyDefaults.UseJitter, MaxDelay = RedeliveryPolicyDefaults.MaxDelay }; + return this; } public IAfterRedeliveryBuilder Redeliver(TimeSpan[] intervals) { EnsureCommitted(); - _rule.Retry = new RetryPolicyConfig { Enabled = false }; - _rule.Redelivery = new RedeliveryPolicyConfig + + rule.Retry = new RetryPolicyConfig { Enabled = false }; + rule.Redelivery = new RedeliveryPolicyConfig { Intervals = intervals, Attempts = intervals.Length, @@ -125,11 +132,9 @@ public IAfterRedeliveryBuilder Redeliver(TimeSpan[] intervals) return this; } - // IAfterRetryBuilder - public IAfterRedeliveryBuilder ThenRedeliver() { - _rule.Redelivery = new RedeliveryPolicyConfig + rule.Redelivery = new RedeliveryPolicyConfig { Intervals = RedeliveryPolicyDefaults.Intervals, Attempts = RedeliveryPolicyDefaults.Intervals.Length, @@ -141,7 +146,7 @@ public IAfterRedeliveryBuilder ThenRedeliver() public IAfterRedeliveryBuilder ThenRedeliver(int attempts, TimeSpan baseDelay) { - _rule.Redelivery = new RedeliveryPolicyConfig + rule.Redelivery = new RedeliveryPolicyConfig { Attempts = attempts, BaseDelay = baseDelay, @@ -153,7 +158,7 @@ public IAfterRedeliveryBuilder ThenRedeliver(int attempts, TimeSpan baseDelay) public IAfterRedeliveryBuilder ThenRedeliver(TimeSpan[] intervals) { - _rule.Redelivery = new RedeliveryPolicyConfig + rule.Redelivery = new RedeliveryPolicyConfig { Intervals = intervals, Attempts = intervals.Length, @@ -165,7 +170,7 @@ public IAfterRedeliveryBuilder ThenRedeliver(TimeSpan[] intervals) public void ThenDeadLetter() { - _rule.Terminal = TerminalAction.DeadLetter; + rule.Terminal = TerminalAction.DeadLetter; } private void EnsureCommitted() @@ -173,16 +178,16 @@ private void EnsureCommitted() if (!_committed) { // Replace any existing rule for the same exception type and predicate. - for (var i = _rules.Count - 1; i >= 0; i--) + for (var i = rules.Count - 1; i >= 0; i--) { - if (_rules[i].ExceptionType == _rule.ExceptionType - && _rules[i].Predicate == _rule.Predicate) + if (rules[i].ExceptionType == rule.ExceptionType + && rules[i].Predicate == rule.Predicate) { - _rules.RemoveAt(i); + rules.RemoveAt(i); } } - _rules.Add(_rule); + rules.Add(rule); _committed = true; } } From 6f8112e057ce4953ae8fb5b834a9416a432b4c02 Mon Sep 17 00:00:00 2001 From: Pascal Senn Date: Wed, 8 Apr 2026 00:39:09 +0200 Subject: [PATCH 08/14] Refactor exception policy rules to use ImmutableArray instead of IReadOnlyList for consistency and improved performance --- .../Consume/Retry/ConsumerRetryMiddleware.cs | 5 ++-- .../Consume/Retry/ExceptionPolicyMatcher.cs | 4 +++- .../Consume/Retry/RetryExecutor.cs | 6 +++-- .../Redelivery/ReceiveRedeliveryMiddleware.cs | 5 ++-- .../Retry/ExceptionPolicyMatcherTests.cs | 23 +++++++++---------- .../Consume/Retry/RetryExecutorTests.cs | 5 ++-- 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs index a60a36f1e23..93c377112d6 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs @@ -1,3 +1,4 @@ +using System.Collections.Immutable; using Microsoft.Extensions.DependencyInjection; using Mocha.Features; @@ -7,7 +8,7 @@ namespace Mocha; /// A consumer middleware that implements in-process retry with configurable backoff strategies /// when transient failures occur. /// -internal sealed class ConsumerRetryMiddleware(IReadOnlyList exceptionPolicyRules) +internal sealed class ConsumerRetryMiddleware(ImmutableArray exceptionPolicyRules) { public async ValueTask InvokeAsync(IConsumeContext context, ConsumerDelegate next) { @@ -45,7 +46,7 @@ public static ConsumerMiddlewareConfiguration Create() return next; } - var middleware = new ConsumerRetryMiddleware(feature.Rules); + var middleware = new ConsumerRetryMiddleware(feature.Rules.ToImmutableArray()); return ctx => middleware.InvokeAsync(ctx, next); }, diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyMatcher.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyMatcher.cs index 22b7a4a4181..38ad7b6f963 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyMatcher.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyMatcher.cs @@ -1,3 +1,5 @@ +using System.Collections.Immutable; + namespace Mocha; /// @@ -13,7 +15,7 @@ internal static class ExceptionPolicyMatcher /// The list of exception policy rules to evaluate. /// The exception to match against. /// The best matching rule, or null if no rule matches. - public static ExceptionPolicyRule? Match(IReadOnlyList rules, Exception exception) + public static ExceptionPolicyRule? Match(ImmutableArray rules, Exception exception) { ExceptionPolicyRule? bestMatch = null; var bestDepth = int.MaxValue; diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs index 87ea7d6845a..a13201f7bdc 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs @@ -1,3 +1,5 @@ +using System.Collections.Immutable; + namespace Mocha; /// @@ -7,7 +9,7 @@ namespace Mocha; internal static class RetryExecutor { public static ValueTask ExecuteAsync( - IReadOnlyList rules, + ImmutableArray rules, TState state, Func action, Action? onRetry, @@ -17,7 +19,7 @@ public static ValueTask ExecuteAsync( } public static async ValueTask ExecuteAsync( - IReadOnlyList rules, + ImmutableArray rules, TState state, Func action, Action? onRetry, diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs index 22d6074442f..24d89f8f155 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs @@ -1,3 +1,4 @@ +using System.Collections.Immutable; using Microsoft.Extensions.DependencyInjection; using Mocha.Features; using Mocha.Middlewares; @@ -15,7 +16,7 @@ namespace Mocha; /// because the caller would time out waiting for a response. /// internal sealed class ReceiveRedeliveryMiddleware( - IReadOnlyList exceptionPolicyRules, + ImmutableArray exceptionPolicyRules, TimeProvider timeProvider, IMessagingPools pools) { @@ -196,7 +197,7 @@ public static ReceiveMiddlewareConfiguration Create() var pools = context.Services.GetRequiredService(); var middleware = new ReceiveRedeliveryMiddleware( - feature.Rules, + feature.Rules.ToImmutableArray(), timeProvider, pools); diff --git a/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/ExceptionPolicyMatcherTests.cs b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/ExceptionPolicyMatcherTests.cs index 4e707f0d2c4..aa5c0dfa7ae 100644 --- a/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/ExceptionPolicyMatcherTests.cs +++ b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/ExceptionPolicyMatcherTests.cs @@ -1,3 +1,5 @@ +using System.Collections.Immutable; + namespace Mocha.Tests.Middlewares.Consume.Retry; public sealed class ExceptionPolicyMatcherTests @@ -6,7 +8,7 @@ public sealed class ExceptionPolicyMatcherTests public void Match_Should_ReturnNull_When_RulesListIsEmpty() { // arrange - var rules = Array.Empty(); + var rules = ImmutableArray.Empty; var exception = new InvalidOperationException("test"); // act @@ -20,10 +22,7 @@ public void Match_Should_ReturnNull_When_RulesListIsEmpty() public void Match_Should_ReturnNull_When_NoRuleMatchesExceptionType() { // arrange - var rules = new[] - { - CreateRule() - }; + var rules = ImmutableArray.Create(CreateRule()); var exception = new InvalidOperationException("test"); // act @@ -38,7 +37,7 @@ public void Match_Should_ReturnRule_When_ExceptionTypeMatchesExactly() { // arrange var rule = CreateRule(); - var rules = new[] { rule }; + var rules = ImmutableArray.Create(rule); var exception = new InvalidOperationException("test"); // act @@ -53,7 +52,7 @@ public void Match_Should_ReturnRule_When_ExceptionIsDerivedType() { // arrange var rule = CreateRule(); - var rules = new[] { rule }; + var rules = ImmutableArray.Create(rule); var exception = new InvalidOperationException("test"); // act @@ -69,7 +68,7 @@ public void Match_Should_ReturnMostSpecificRule_When_MultipleRulesMatchAtDiffere // arrange var baseRule = CreateRule(); var specificRule = CreateRule(); - var rules = new[] { baseRule, specificRule }; + var rules = ImmutableArray.Create(baseRule, specificRule); var exception = new InvalidOperationException("test"); // act @@ -84,7 +83,7 @@ public void Match_Should_ReturnNull_When_TypeMatchesButPredicateReturnsFalse() { // arrange var rule = CreateRule(predicate: _ => false); - var rules = new[] { rule }; + var rules = ImmutableArray.Create(rule); var exception = new InvalidOperationException("test"); // act @@ -99,7 +98,7 @@ public void Match_Should_ReturnRule_When_TypeMatchesAndPredicateReturnsTrue() { // arrange var rule = CreateRule(predicate: _ => true); - var rules = new[] { rule }; + var rules = ImmutableArray.Create(rule); var exception = new InvalidOperationException("test"); // act @@ -115,7 +114,7 @@ public void Match_Should_ReturnMoreSpecificRule_When_LessSpecificRuleHasPredicat // arrange var baseRuleWithPredicate = CreateRule(predicate: _ => true); var specificRule = CreateRule(); - var rules = new[] { baseRuleWithPredicate, specificRule }; + var rules = ImmutableArray.Create(baseRuleWithPredicate, specificRule); var exception = new InvalidOperationException("test"); // act @@ -131,7 +130,7 @@ public void Match_Should_ReturnPassingRule_When_TwoRulesForSameTypeAndOnePredica // arrange var failingRule = CreateRule(predicate: _ => false); var passingRule = CreateRule(predicate: _ => true); - var rules = new[] { failingRule, passingRule }; + var rules = ImmutableArray.Create(failingRule, passingRule); var exception = new ArgumentNullException("param"); // act diff --git a/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs index 348651cf4be..9c0bee8f74f 100644 --- a/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs +++ b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs @@ -1,3 +1,4 @@ +using System.Collections.Immutable; using Microsoft.Extensions.Time.Testing; namespace Mocha.Tests.Middlewares.Consume.Retry; @@ -893,11 +894,11 @@ await Assert.ThrowsAsync(() => // -- Helpers -- - private static IReadOnlyList BuildRules(Action configure) + private static ImmutableArray BuildRules(Action configure) { var feature = new ExceptionPolicyFeature(); feature.Configure(configure); - return feature.Rules; + return [.. feature.Rules]; } private sealed class Counter From 88fa392b1eda74e542c987721bce124fa616918e Mon Sep 17 00:00:00 2001 From: Pascal Senn Date: Wed, 8 Apr 2026 00:48:53 +0200 Subject: [PATCH 09/14] Refactor retry feature naming and update related documentation for consistency --- .../Consume/Retry/ConsumerRetryMiddleware.cs | 2 +- .../Consume/Retry/ExceptionPolicyOptions.cs | 28 ++++++---------- .../Consume/Retry/RedeliveryPolicyConfig.cs | 4 ++- .../Consume/Retry/RedeliveryPolicyDefaults.cs | 4 ++- .../ResilienceConfigurationExtensions.cs | 33 ++++++++++++++----- ...RetryRuntimeFeature.cs => RetryFeature.cs} | 2 +- .../Consume/Retry/RetryPolicyConfig.cs | 4 ++- .../Behaviors/RetryTests.cs | 2 +- website/src/docs/mocha/v1/reliability.md | 4 +-- 9 files changed, 49 insertions(+), 34 deletions(-) rename src/Mocha/src/Mocha/Middlewares/Consume/Retry/{RetryRuntimeFeature.cs => RetryFeature.cs} (94%) diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs index 93c377112d6..dd0c0b815f5 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs @@ -22,7 +22,7 @@ public async ValueTask InvokeAsync(IConsumeContext context, ConsumerDelegate nex } // Expose retry state to handlers via features. - var retryState = context.Features.GetOrSet(); + var retryState = context.Features.GetOrSet(); retryState.DelayedRetryCount = delayedRetryCount; retryState.ImmediateRetryCount = 0; diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyOptions.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyOptions.cs index 2ded5c8d484..3674db06f89 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyOptions.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyOptions.cs @@ -3,18 +3,13 @@ namespace Mocha; /// /// Options for configuring exception policies with per-exception rules. /// -public class ExceptionPolicyOptions +/// +/// Initializes a new instance of the class. +/// +/// The shared list of exception policy rules to populate. +public class ExceptionPolicyOptions(List rules) { - private readonly List _rules; - - /// - /// Initializes a new instance of the class. - /// - /// The shared list of exception policy rules to populate. - public ExceptionPolicyOptions(List rules) - { - _rules = rules; - } + private readonly List _rules = rules; /// /// Configures the default behavior for all exceptions that don't match a more specific rule. @@ -28,8 +23,7 @@ public ExceptionPolicyOptions(List rules) /// /// The exception type to configure. /// A builder for configuring the exception behavior. - public IExceptionPolicyBuilder On() where TException : Exception - => On(null); + public IExceptionPolicyBuilder On() where TException : Exception => On(null); /// /// Configures behavior for a specific exception type matching a predicate. @@ -43,11 +37,9 @@ public IExceptionPolicyBuilder On(Func Func? wrappedPredicate = predicate is not null ? ex => ex is TException typed && predicate(typed) : null; - var rule = new ExceptionPolicyRule - { - ExceptionType = typeof(TException), - Predicate = wrappedPredicate - }; + + var rule = new ExceptionPolicyRule { ExceptionType = typeof(TException), Predicate = wrappedPredicate }; + return new ExceptionPolicyBuilder(rule, _rules); } } diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyConfig.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyConfig.cs index 746ee8e4ae3..407f985bb82 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyConfig.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyConfig.cs @@ -1,3 +1,5 @@ +using System.Collections.Immutable; + namespace Mocha; /// @@ -33,5 +35,5 @@ public sealed class RedeliveryPolicyConfig /// /// Gets the explicit redelivery intervals, or null to use global defaults. /// - public TimeSpan[]? Intervals { get; init; } + public ImmutableArray? Intervals { get; init; } } diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyDefaults.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyDefaults.cs index 3a7a943f979..bb4df697960 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyDefaults.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RedeliveryPolicyDefaults.cs @@ -1,3 +1,5 @@ +using System.Collections.Immutable; + namespace Mocha; /// @@ -8,7 +10,7 @@ internal static class RedeliveryPolicyDefaults /// /// The default redelivery intervals. Value: 5min, 15min, 30min. /// - public static readonly TimeSpan[] Intervals = + public static readonly ImmutableArray Intervals = [ TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(15), diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ResilienceConfigurationExtensions.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ResilienceConfigurationExtensions.cs index b7af55546cd..a37f1bf3632 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ResilienceConfigurationExtensions.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ResilienceConfigurationExtensions.cs @@ -16,8 +16,8 @@ public static class ResilienceConfigurationExtensions /// The builder for method chaining. public static IMessageBusBuilder AddResilience(this IMessageBusBuilder builder) { - builder.ConfigureFeature(f => f.GetOrSet() - .Configure(p => p.Default().Retry().ThenRedeliver())); + builder.ConfigureFeature(f => f.GetOrSet().Configure(p => p.AddDefaultPolicy())); + return builder; } @@ -31,7 +31,13 @@ public static IMessageBusBuilder AddResilience( this IMessageBusBuilder builder, Action configure) { - builder.ConfigureFeature(f => f.GetOrSet().Configure(configure)); + builder.ConfigureFeature(f => + { + var feature = f.GetOrSet(); + feature.Configure(p => p.AddDefaultPolicy()); + feature.Configure(configure); + }); + return builder; } @@ -68,12 +74,14 @@ public static IMessageBusHostBuilder AddResilience( /// The descriptor type that supports receive middleware. /// The descriptor to configure. /// The descriptor for method chaining. - public static TDescriptor AddResilience( - this TDescriptor descriptor) + public static TDescriptor AddResilience(this TDescriptor descriptor) where TDescriptor : IReceiveMiddlewareProvider { - descriptor.Extend().Configuration.Features.GetOrSet() - .Configure(p => p.Default().Retry().ThenRedeliver()); + descriptor + .Extend() + .Configuration.Features.GetOrSet() + .Configure(p => p.AddDefaultPolicy()); + return descriptor; } @@ -89,7 +97,16 @@ public static TDescriptor AddResilience( Action configure) where TDescriptor : IReceiveMiddlewareProvider { - descriptor.Extend().Configuration.Features.GetOrSet().Configure(configure); + var feature = descriptor.Extend().Configuration.Features.GetOrSet(); + feature.Configure(p => p.AddDefaultPolicy()); + feature.Configure(configure); + return descriptor; } + + private static ExceptionPolicyOptions AddDefaultPolicy(this ExceptionPolicyOptions options) + { + options.Default().Retry().ThenRedeliver(); + return options; + } } diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryFeature.cs similarity index 94% rename from src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs rename to src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryFeature.cs index f1732962d68..03af7aa0955 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryRuntimeFeature.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryFeature.cs @@ -5,7 +5,7 @@ namespace Mocha; /// context.Features.Get<RetryState>(). /// Null if retry is not configured via AddResilience. /// -public sealed class RetryRuntimeFeature +public sealed class RetryFeature { /// /// Number of immediate retries already attempted for this delivery round. diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyConfig.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyConfig.cs index 0414ec5b6bb..f8ad6f3f47a 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyConfig.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyConfig.cs @@ -1,3 +1,5 @@ +using System.Collections.Immutable; + namespace Mocha; /// @@ -38,5 +40,5 @@ public sealed class RetryPolicyConfig /// /// Gets the explicit retry intervals. /// - public TimeSpan[]? Intervals { get; init; } + public ImmutableArray[]? Intervals { get; init; } } diff --git a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs index 7155eaba98b..369e6e71ceb 100644 --- a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs +++ b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RetryTests.cs @@ -415,7 +415,7 @@ private sealed class RetryStateCapturingConsumer(RetryStateCapture capture) : IC { public ValueTask ConsumeAsync(IConsumeContext context) { - var retryState = context.Features.Get(); + var retryState = context.Features.Get(); capture.Record(retryState?.ImmediateRetryCount ?? -1); throw new InvalidOperationException("Fail to trigger retry"); } diff --git a/website/src/docs/mocha/v1/reliability.md b/website/src/docs/mocha/v1/reliability.md index 387c537bacd..da44c413666 100644 --- a/website/src/docs/mocha/v1/reliability.md +++ b/website/src/docs/mocha/v1/reliability.md @@ -493,7 +493,7 @@ public class PaymentConsumer(ILogger logger) : IConsumer context) { - var state = context.Features.Get(); + var state = context.Features.Get(); if (state is { ImmediateRetryCount: > 2 }) { @@ -516,7 +516,7 @@ public class PaymentConsumer(ILogger logger) : IConsumer Date: Wed, 8 Apr 2026 01:17:03 +0200 Subject: [PATCH 10/14] Refactor redelivery and retry logic, enhance exception handling, and improve documentation for clarity --- .../Consume/Retry/ConsumerRetryMiddleware.cs | 11 +- .../Consume/Retry/ExceptionPolicyBuilder.cs | 6 +- .../Consume/Retry/RetryPolicyConfig.cs | 2 +- .../MiddlewareConfigurationExtensions.cs | 6 - .../Redelivery/ReceiveRedeliveryMiddleware.cs | 64 +---- .../Receive/Redelivery/RedeliveryExecutor.cs | 44 ++++ .../Retry/ExceptionPolicyMatcherTests.cs | 9 +- .../Redelivery/RedeliveryExecutorTests.cs | 237 ++++++++++++++++++ .../src/docs/mocha/v1/exception-policies.md | 34 +-- 9 files changed, 322 insertions(+), 91 deletions(-) create mode 100644 src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs create mode 100644 src/Mocha/test/Mocha.Tests/Middlewares/Receive/Redelivery/RedeliveryExecutorTests.cs diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs index dd0c0b815f5..d0c333fb6a0 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs @@ -15,10 +15,15 @@ public async ValueTask InvokeAsync(IConsumeContext context, ConsumerDelegate nex // Read delayed retry count from headers (set by redelivery middleware). var delayedRetryCount = 0; - if (context.Headers.TryGetValue(MessageHeaders.Retry.DelayedRetryCount.Key, out var headerValue) - && headerValue is int count) + if (context.Headers.TryGetValue(MessageHeaders.Retry.DelayedRetryCount.Key, out var headerValue)) { - delayedRetryCount = count; + delayedRetryCount = headerValue switch + { + int i => i, + long l => (int)l, + double d => (int)d, + _ => 0 + }; } // Expose retry state to handlers via features. diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs index 88ca2dd900c..902c0bda118 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ExceptionPolicyBuilder.cs @@ -79,7 +79,7 @@ public IAfterRetryBuilder Retry(TimeSpan[] intervals) { EnsureCommitted(); - rule.Retry = new RetryPolicyConfig { Intervals = intervals, Attempts = intervals.Length }; + rule.Retry = new RetryPolicyConfig { Intervals = [.. intervals], Attempts = intervals.Length }; rule.Redelivery = new RedeliveryPolicyConfig { Enabled = false }; return this; @@ -124,7 +124,7 @@ public IAfterRedeliveryBuilder Redeliver(TimeSpan[] intervals) rule.Retry = new RetryPolicyConfig { Enabled = false }; rule.Redelivery = new RedeliveryPolicyConfig { - Intervals = intervals, + Intervals = [.. intervals], Attempts = intervals.Length, UseJitter = RedeliveryPolicyDefaults.UseJitter, MaxDelay = RedeliveryPolicyDefaults.MaxDelay @@ -160,7 +160,7 @@ public IAfterRedeliveryBuilder ThenRedeliver(TimeSpan[] intervals) { rule.Redelivery = new RedeliveryPolicyConfig { - Intervals = intervals, + Intervals = [.. intervals], Attempts = intervals.Length, UseJitter = RedeliveryPolicyDefaults.UseJitter, MaxDelay = RedeliveryPolicyDefaults.MaxDelay diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyConfig.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyConfig.cs index f8ad6f3f47a..5c9f2d74f5f 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyConfig.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryPolicyConfig.cs @@ -40,5 +40,5 @@ public sealed class RetryPolicyConfig /// /// Gets the explicit retry intervals. /// - public ImmutableArray[]? Intervals { get; init; } + public ImmutableArray? Intervals { get; init; } } diff --git a/src/Mocha/src/Mocha/Middlewares/Extensions/MiddlewareConfigurationExtensions.cs b/src/Mocha/src/Mocha/Middlewares/Extensions/MiddlewareConfigurationExtensions.cs index 16f27b0b22a..d9f0f1d3bff 100644 --- a/src/Mocha/src/Mocha/Middlewares/Extensions/MiddlewareConfigurationExtensions.cs +++ b/src/Mocha/src/Mocha/Middlewares/Extensions/MiddlewareConfigurationExtensions.cs @@ -220,12 +220,6 @@ public void Prepend(ConsumerMiddlewareConfiguration configuration, string? befor configurations.Add(pipeline => { - // Skip if a middleware with the same key is already registered. - if (configuration.Key is not null && pipeline.Exists(m => m.Key == configuration.Key)) - { - return; - } - var index = pipeline.FindIndex(m => m.Key == before); if (index == -1) { diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs index 24d89f8f155..a403718fc95 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs @@ -31,6 +31,7 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next { int i => i, long l => (int)l, + double d => (int)d, _ => 0 }; } @@ -41,6 +42,8 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next } catch (Exception ex) { + var consumerFeature = context.Features.GetOrSet(); + // Request/reply messages must not be redelivered -- the caller is waiting. if (context.Envelope?.ResponseAddress is not null) { @@ -50,7 +53,7 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next // Match exception against policy rules. var rule = ExceptionPolicyMatcher.Match(exceptionPolicyRules, ex); - // No matching rule — no policy for this exception, let it propagate. + // No matching rule - no policy for this exception, let it propagate. if (rule is null) { throw; @@ -59,12 +62,7 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next // Discard: swallow at receive level. if (rule.Terminal == TerminalAction.Discard) { - var consumerFeature = context.Features.Get(); - - if (consumerFeature is not null) - { - consumerFeature.MessageConsumed = true; - } + consumerFeature.MessageConsumed = true; return; } @@ -84,9 +82,7 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next var redeliveryConfig = rule.Redelivery; // Check if redelivery attempts remain. - var maxAttempts = redeliveryConfig.Attempts - ?? redeliveryConfig.Intervals?.Length - ?? 0; + var maxAttempts = redeliveryConfig.Attempts ?? redeliveryConfig.Intervals?.Length ?? 0; if (delayedRetryCount >= maxAttempts) { @@ -94,7 +90,7 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next } // Calculate the delay for this redelivery attempt. - var delay = CalculateDelay(delayedRetryCount, redeliveryConfig); + var delay = RedeliveryExecutor.CalculateDelay(delayedRetryCount, redeliveryConfig); var scheduledTime = timeProvider.GetUtcNow().Add(delay); // Update the header on the envelope so the next delivery round sees the incremented count. @@ -139,47 +135,10 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next } // Mark the message as consumed so Fault/DeadLetter don't also handle it. - context.Features.GetOrSet().MessageConsumed = true; + consumerFeature.MessageConsumed = true; } } - private static TimeSpan CalculateDelay(int attempt, RedeliveryPolicyConfig config) - { - TimeSpan baseDelay; - - if (config.Intervals is { Length: > 0 } intervals) - { - // Explicit intervals: use array index, clamp to last. - baseDelay = intervals[Math.Min(attempt, intervals.Length - 1)]; - } - else - { - // Calculated: BaseDelay * (attempt + 1). - var configuredBaseDelay = config.BaseDelay ?? RedeliveryPolicyDefaults.Intervals[0]; - baseDelay = configuredBaseDelay * (attempt + 1); - } - - // Cap by MaxDelay. - var maxDelay = config.MaxDelay ?? RedeliveryPolicyDefaults.MaxDelay; - - if (baseDelay > maxDelay) - { - baseDelay = maxDelay; - } - - // Add jitter: +/- 25%. - var useJitter = config.UseJitter ?? RedeliveryPolicyDefaults.UseJitter; - - if (useJitter) - { - var jitterRange = baseDelay.TotalMilliseconds * 0.25; - var jitter = (Random.Shared.NextDouble() * 2 - 1) * jitterRange; - baseDelay = TimeSpan.FromMilliseconds(Math.Max(0, baseDelay.TotalMilliseconds + jitter)); - } - - return baseDelay; - } - public static ReceiveMiddlewareConfiguration Create() => new( static (context, next) => @@ -189,17 +148,14 @@ public static ReceiveMiddlewareConfiguration Create() if (feature is null) { - // No exception policy configured — skip redelivery middleware entirely. + // No exception policy configured - skip redelivery middleware entirely. return next; } var timeProvider = context.Services.GetRequiredService(); var pools = context.Services.GetRequiredService(); - var middleware = new ReceiveRedeliveryMiddleware( - feature.Rules.ToImmutableArray(), - timeProvider, - pools); + var middleware = new ReceiveRedeliveryMiddleware(feature.Rules.ToImmutableArray(), timeProvider, pools); return ctx => middleware.InvokeAsync(ctx, next); }, diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs new file mode 100644 index 00000000000..86d9f032ab7 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs @@ -0,0 +1,44 @@ +namespace Mocha; + +/// +/// Provides delay calculation logic for the redelivery middleware. +/// +internal static class RedeliveryExecutor +{ + internal static TimeSpan CalculateDelay(int attempt, RedeliveryPolicyConfig config) + { + TimeSpan baseDelay; + + if (config.Intervals is { Length: > 0 } intervals) + { + // Explicit intervals: use array index, clamp to last. + baseDelay = intervals[Math.Min(attempt, intervals.Length - 1)]; + } + else + { + // Calculated: BaseDelay * (attempt + 1). + var configuredBaseDelay = config.BaseDelay ?? RedeliveryPolicyDefaults.Intervals[0]; + baseDelay = configuredBaseDelay * (attempt + 1); + } + + // Cap by MaxDelay. + var maxDelay = config.MaxDelay ?? RedeliveryPolicyDefaults.MaxDelay; + + if (baseDelay > maxDelay) + { + baseDelay = maxDelay; + } + + // Add jitter: +/- 25%. + var useJitter = config.UseJitter ?? RedeliveryPolicyDefaults.UseJitter; + + if (useJitter) + { + var jitterRange = baseDelay.TotalMilliseconds * 0.25; + var jitter = ((Random.Shared.NextDouble() * 2) - 1) * jitterRange; + baseDelay = TimeSpan.FromMilliseconds(Math.Max(0, baseDelay.TotalMilliseconds + jitter)); + } + + return baseDelay; + } +} diff --git a/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/ExceptionPolicyMatcherTests.cs b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/ExceptionPolicyMatcherTests.cs index aa5c0dfa7ae..8396879e4a4 100644 --- a/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/ExceptionPolicyMatcherTests.cs +++ b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/ExceptionPolicyMatcherTests.cs @@ -140,14 +140,9 @@ public void Match_Should_ReturnPassingRule_When_TwoRulesForSameTypeAndOnePredica Assert.Same(passingRule, result); } - private static ExceptionPolicyRule CreateRule( - Func? predicate = null) + private static ExceptionPolicyRule CreateRule(Func? predicate = null) where TException : Exception { - return new ExceptionPolicyRule - { - ExceptionType = typeof(TException), - Predicate = predicate - }; + return new ExceptionPolicyRule { ExceptionType = typeof(TException), Predicate = predicate }; } } diff --git a/src/Mocha/test/Mocha.Tests/Middlewares/Receive/Redelivery/RedeliveryExecutorTests.cs b/src/Mocha/test/Mocha.Tests/Middlewares/Receive/Redelivery/RedeliveryExecutorTests.cs new file mode 100644 index 00000000000..11fa5350aeb --- /dev/null +++ b/src/Mocha/test/Mocha.Tests/Middlewares/Receive/Redelivery/RedeliveryExecutorTests.cs @@ -0,0 +1,237 @@ +using System.Collections.Immutable; + +namespace Mocha.Tests.Middlewares.Receive.Redelivery; + +public sealed class RedeliveryExecutorTests +{ + [Fact] + public void CalculateDelay_Should_ReturnCorrectInterval_When_ExplicitIntervalsProvided() + { + // arrange + var config = new RedeliveryPolicyConfig + { + Intervals = ImmutableArray.Create( + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(15), + TimeSpan.FromMinutes(30)), + UseJitter = false + }; + + // act & assert + Assert.Equal(TimeSpan.FromMinutes(5), RedeliveryExecutor.CalculateDelay(0, config)); + Assert.Equal(TimeSpan.FromMinutes(15), RedeliveryExecutor.CalculateDelay(1, config)); + Assert.Equal(TimeSpan.FromMinutes(30), RedeliveryExecutor.CalculateDelay(2, config)); + } + + [Fact] + public void CalculateDelay_Should_ClampToLastInterval_When_AttemptExceedsIntervalCount() + { + // arrange + var config = new RedeliveryPolicyConfig + { + Intervals = ImmutableArray.Create( + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(15)), + UseJitter = false + }; + + // act + var delay = RedeliveryExecutor.CalculateDelay(5, config); + + // assert + Assert.Equal(TimeSpan.FromMinutes(15), delay); + } + + [Theory] + [InlineData(0, 10)] // BaseDelay * (0 + 1) = 10min + [InlineData(1, 20)] // BaseDelay * (1 + 1) = 20min + [InlineData(2, 30)] // BaseDelay * (2 + 1) = 30min + public void CalculateDelay_Should_ReturnLinearDelay_When_NoIntervalsProvided( + int attempt, + int expectedMinutes) + { + // arrange + var config = new RedeliveryPolicyConfig + { + BaseDelay = TimeSpan.FromMinutes(10), + UseJitter = false, + MaxDelay = TimeSpan.FromHours(2) + }; + + // act + var delay = RedeliveryExecutor.CalculateDelay(attempt, config); + + // assert + Assert.Equal(TimeSpan.FromMinutes(expectedMinutes), delay); + } + + [Fact] + public void CalculateDelay_Should_UseDefaultBaseDelay_When_NoBaseDelayAndNoIntervals() + { + // arrange - no BaseDelay, no Intervals => uses RedeliveryPolicyDefaults.Intervals[0] (5 min) + var config = new RedeliveryPolicyConfig + { + UseJitter = false, + MaxDelay = TimeSpan.FromHours(2) + }; + + // act + var delay = RedeliveryExecutor.CalculateDelay(0, config); + + // assert - 5min * (0 + 1) = 5min + Assert.Equal(TimeSpan.FromMinutes(5), delay); + } + + [Fact] + public void CalculateDelay_Should_ScaleDefaultBaseDelay_When_AttemptIncreases() + { + // arrange + var config = new RedeliveryPolicyConfig + { + UseJitter = false, + MaxDelay = TimeSpan.FromHours(2) + }; + + // act & assert - 5min * (attempt + 1) + Assert.Equal(TimeSpan.FromMinutes(5), RedeliveryExecutor.CalculateDelay(0, config)); + Assert.Equal(TimeSpan.FromMinutes(10), RedeliveryExecutor.CalculateDelay(1, config)); + Assert.Equal(TimeSpan.FromMinutes(15), RedeliveryExecutor.CalculateDelay(2, config)); + } + + [Fact] + public void CalculateDelay_Should_CapAtMaxDelay_When_ComputedDelayExceedsMax() + { + // arrange + var config = new RedeliveryPolicyConfig + { + BaseDelay = TimeSpan.FromMinutes(30), + MaxDelay = TimeSpan.FromMinutes(45), + UseJitter = false + }; + + // act - 30min * (1 + 1) = 60min, capped at 45min + var delay = RedeliveryExecutor.CalculateDelay(1, config); + + // assert + Assert.Equal(TimeSpan.FromMinutes(45), delay); + } + + [Fact] + public void CalculateDelay_Should_UseDefaultMaxDelay_When_NoMaxDelayConfigured() + { + // arrange - no MaxDelay => uses RedeliveryPolicyDefaults.MaxDelay (1 hour) + var config = new RedeliveryPolicyConfig + { + BaseDelay = TimeSpan.FromMinutes(30), + UseJitter = false + }; + + // act - 30min * (5 + 1) = 180min, capped at 60min (default) + var delay = RedeliveryExecutor.CalculateDelay(5, config); + + // assert + Assert.Equal(TimeSpan.FromHours(1), delay); + } + + [Fact] + public void CalculateDelay_Should_ReturnExactDelay_When_JitterDisabled() + { + // arrange + var config = new RedeliveryPolicyConfig + { + BaseDelay = TimeSpan.FromMinutes(10), + UseJitter = false, + MaxDelay = TimeSpan.FromHours(2) + }; + + // act - run multiple times to confirm determinism + var delay1 = RedeliveryExecutor.CalculateDelay(0, config); + var delay2 = RedeliveryExecutor.CalculateDelay(0, config); + var delay3 = RedeliveryExecutor.CalculateDelay(0, config); + + // assert + Assert.Equal(TimeSpan.FromMinutes(10), delay1); + Assert.Equal(delay1, delay2); + Assert.Equal(delay2, delay3); + } + + [Fact] + public void CalculateDelay_Should_ReturnDelayWithinJitterBounds_When_JitterEnabled() + { + // arrange + var config = new RedeliveryPolicyConfig + { + BaseDelay = TimeSpan.FromMinutes(10), + UseJitter = true, + MaxDelay = TimeSpan.FromHours(2) + }; + + var baseExpected = TimeSpan.FromMinutes(10); // 10min * (0 + 1) + var lowerBound = baseExpected.TotalMilliseconds * 0.75; + var upperBound = baseExpected.TotalMilliseconds * 1.25; + + // act - run multiple times to increase confidence + for (var i = 0; i < 100; i++) + { + var delay = RedeliveryExecutor.CalculateDelay(0, config); + + // assert + Assert.InRange(delay.TotalMilliseconds, lowerBound, upperBound); + } + } + + [Fact] + public void CalculateDelay_Should_ApplyJitterByDefault_When_UseJitterIsNull() + { + // arrange - UseJitter not set => defaults to RedeliveryPolicyDefaults.UseJitter (true) + var config = new RedeliveryPolicyConfig + { + BaseDelay = TimeSpan.FromMinutes(10), + MaxDelay = TimeSpan.FromHours(2) + }; + + var baseExpected = TimeSpan.FromMinutes(10); + var lowerBound = baseExpected.TotalMilliseconds * 0.75; + var upperBound = baseExpected.TotalMilliseconds * 1.25; + var hasVariation = false; + TimeSpan? firstDelay = null; + + // act - run multiple times; at least one should differ (proving jitter is active) + for (var i = 0; i < 100; i++) + { + var delay = RedeliveryExecutor.CalculateDelay(0, config); + + Assert.InRange(delay.TotalMilliseconds, lowerBound, upperBound); + + firstDelay ??= delay; + + if (delay != firstDelay) + { + hasVariation = true; + } + } + + // assert - with jitter, we expect variation across 100 runs + Assert.True(hasVariation, "Expected jitter to produce varying delays, but all 100 values were identical."); + } + + [Fact] + public void CalculateDelay_Should_CapExplicitIntervalAtMaxDelay_When_IntervalExceedsMax() + { + // arrange + var config = new RedeliveryPolicyConfig + { + Intervals = ImmutableArray.Create( + TimeSpan.FromMinutes(5), + TimeSpan.FromHours(2)), + MaxDelay = TimeSpan.FromHours(1), + UseJitter = false + }; + + // act + var delay = RedeliveryExecutor.CalculateDelay(1, config); + + // assert + Assert.Equal(TimeSpan.FromHours(1), delay); + } +} diff --git a/website/src/docs/mocha/v1/exception-policies.md b/website/src/docs/mocha/v1/exception-policies.md index 850a1f1a7cc..5020817eec1 100644 --- a/website/src/docs/mocha/v1/exception-policies.md +++ b/website/src/docs/mocha/v1/exception-policies.md @@ -3,20 +3,20 @@ title: "Exception Policies" description: "Configure per-exception handling with composable retry, redelivery, and terminal actions." --- -Not every exception deserves the same treatment. A database deadlock might resolve on immediate retry. A downstream service outage needs minutes to recover. A validation error will never succeed no matter how many times you retry. Exception policies let you define per-exception handling strategies — retry, redeliver, dead-letter, or discard — as composable escalation chains in a single `AddResilience` call. +Not every exception deserves the same treatment. A database deadlock might resolve on immediate retry. A downstream service outage needs minutes to recover. A validation error will never succeed no matter how many times you retry. Exception policies let you define per-exception handling strategies - retry, redeliver, dead-letter, or discard - as composable escalation chains in a single `AddResilience` call. ```csharp builder.Services .AddMessageBus() .AddResilience(policy => { - // Validation errors are permanent — route straight to the error endpoint + // Validation errors are permanent - route straight to the error endpoint policy.On().DeadLetter(); // Duplicate messages are safe to drop policy.On().Discard(); - // Database deadlocks resolve quickly — retry then redeliver + // Database deadlocks resolve quickly - retry then redeliver policy.On(ex => ex.IsTransient) .Retry(5, TimeSpan.FromMilliseconds(200)) .ThenRedeliver(); @@ -33,7 +33,7 @@ builder.Services # How exception handling works -When a handler throws an exception, Mocha evaluates exception policies to determine what happens next. The decision flows through two pipeline stages — retry in the consumer pipeline and redelivery in the receive pipeline — before reaching the fault middleware as a last resort. +When a handler throws an exception, Mocha evaluates exception policies to determine what happens next. The decision flows through two pipeline stages - retry in the consumer pipeline and redelivery in the receive pipeline - before reaching the fault middleware as a last resort. ```mermaid flowchart TD @@ -60,14 +60,14 @@ flowchart TD Each exception policy rule targets a specific exception type and defines an escalation chain. The chain controls which stages the message passes through and with what settings. -Exception matching respects inheritance. A policy on `NpgsqlException` also matches any subclass. When multiple rules could match, the most specific type wins — the same precedence as C# `catch` blocks. +Exception matching respects inheritance. A policy on `NpgsqlException` also matches any subclass. When multiple rules could match, the most specific type wins - the same precedence as C# `catch` blocks. # Configure exception policies -`AddResilience` is the single entry point for all exception handling configuration. There is no separate `AddRetry` or `AddRedelivery` call — retry and redelivery settings are configured per-exception within the policy. +`AddResilience` is the single entry point for all exception handling configuration. There is no separate `AddRetry` or `AddRedelivery` call - retry and redelivery settings are configured per-exception within the policy. :::note Replacement semantics -Calling `On()` for the same exception type replaces the previous rule for that type — last write wins. If you call `On()` twice without a predicate, the second call overwrites the first. The same applies to `Default()`: calling it again replaces the previous default rule. For example, the parameterless `AddResilience()` registers `Default().Retry().ThenRedeliver()`. If you later call `AddResilience(p => p.Default().Retry(5))`, the new default replaces the one registered by the parameterless overload. +Calling `On()` for the same exception type replaces the previous rule for that type - last write wins. If you call `On()` twice without a predicate, the second call overwrites the first. The same applies to `Default()`: calling it again replaces the previous default rule. For example, the parameterless `AddResilience()` registers `Default().Retry().ThenRedeliver()`. If you later call `AddResilience(p => p.Default().Retry(5))`, the new default replaces the one registered by the parameterless overload. ::: ## Parameterless defaults @@ -95,9 +95,9 @@ This is equivalent to: `ExceptionPolicyOptions` exposes two methods for creating rules: -- **`Default()`** — shorthand for `On()`. Configures the catch-all behavior for any exception that does not match a more specific rule. -- **`On()`** — configures behavior for a specific exception type. -- **`On(predicate)`** — configures behavior for a specific exception type when a predicate matches. +- **`Default()`** - shorthand for `On()`. Configures the catch-all behavior for any exception that does not match a more specific rule. +- **`On()`** - configures behavior for a specific exception type. +- **`On(predicate)`** - configures behavior for a specific exception type when a predicate matches. ```csharp .AddResilience(policy => @@ -127,7 +127,7 @@ builder.Services ## Transport-level policies -Override bus-level policies for a specific transport. Transport-level policies replace the bus-level policies entirely for all endpoints on that transport — they are not merged. +Override bus-level policies for a specific transport. Transport-level policies replace the bus-level policies entirely for all endpoints on that transport - they are not merged. ```csharp builder.Services @@ -175,7 +175,7 @@ The `PaymentHandler` gets 5 retries with exponential backoff. All other consumer ## Scope hierarchy -Exception policies resolve at four levels. The most specific scope wins, and replacement is atomic — the entire set of rules is replaced, not individual rules. +Exception policies resolve at four levels. The most specific scope wins, and replacement is atomic - the entire set of rules is replaced, not individual rules. | Scope | Applies to | Configured on | | --------- | ------------------------------------- | ---------------------------- | @@ -193,11 +193,11 @@ If a consumer defines exception policies, the bus-level and transport-level poli # Terminal actions -Terminal actions end the message's lifecycle immediately. No retry, no redelivery — the message is either routed to the error endpoint or discarded. +Terminal actions end the message's lifecycle immediately. No retry, no redelivery - the message is either routed to the error endpoint or discarded. ## DeadLetter -`DeadLetter()` routes the message to the error endpoint, skipping both retry and redelivery. Use this for exceptions that are permanent — retrying will never succeed and you want the message preserved for inspection. +`DeadLetter()` routes the message to the error endpoint, skipping both retry and redelivery. Use this for exceptions that are permanent - retrying will never succeed and you want the message preserved for inspection. ```csharp policy.On().DeadLetter(); @@ -216,7 +216,7 @@ policy.On().Discard(); policy.On().Discard(); ``` -**When to use:** Messages that are safe to lose — duplicates you have already processed, stale events that no longer matter. Use with caution: discarded messages leave no audit trail in the error endpoint. +**When to use:** Messages that are safe to lose - duplicates you have already processed, stale events that no longer matter. Use with caution: discarded messages leave no audit trail in the error endpoint. # Retry policies @@ -280,7 +280,7 @@ All strategies apply jitter by default to prevent thundering herd effects. # Redelivery policies -Redelivery schedules the message for later delivery through the transport. The concurrency slot is released, the message re-enters the full receive pipeline on each redelivery attempt, and fresh retry cycles run on each delivery. Use redelivery for failures that need minutes or hours to resolve — a downstream service recovering from an outage, a rate limit resetting, or a database completing a failover. +Redelivery schedules the message for later delivery through the transport. The concurrency slot is released, the message re-enters the full receive pipeline on each redelivery attempt, and fresh retry cycles run on each delivery. Use redelivery for failures that need minutes or hours to resolve - a downstream service recovering from an outage, a rate limit resetting, or a database completing a failover. When you call `.Redeliver()` directly (without `.Retry()` first), retry is disabled for that exception type. The handler failure goes straight to redelivery scheduling. @@ -319,7 +319,7 @@ Specifies the exact delay before each redelivery attempt. The array length deter # Escalation chains -The fluent API composes retry, redelivery, and terminal actions into escalation chains. The interface design enforces valid chains at compile time — you cannot chain `.ThenRedeliver()` after `.Redeliver()`, or `.Retry()` after `.ThenRedeliver()`. +The fluent API composes retry, redelivery, and terminal actions into escalation chains. The interface design enforces valid chains at compile time - you cannot chain `.ThenRedeliver()` after `.Redeliver()`, or `.Retry()` after `.ThenRedeliver()`. ## Retry then redeliver From 31822f54b6daee4098c93ab54b8efc704ea92851 Mon Sep 17 00:00:00 2001 From: Pascal Senn Date: Wed, 8 Apr 2026 01:38:36 +0200 Subject: [PATCH 11/14] Refactor redelivery logic to use RedeliveryExecutor for decision-making and enhance exception handling; add new RedeliveryAction and RedeliveryDecision types for clarity and maintainability --- .../Consume/Retry/ConsumerRetryMiddleware.cs | 8 +- .../Redelivery/ReceiveRedeliveryMiddleware.cs | 136 +++----- .../Receive/Redelivery/RedeliveryAction.cs | 22 ++ .../Receive/Redelivery/RedeliveryDecision.cs | 41 +++ .../Receive/Redelivery/RedeliveryExecutor.cs | 74 +++- .../Redelivery/RedeliveryExecutorTests.cs | 318 ++++++++++++++++-- .../src/docs/mocha/v1/exception-policies.md | 2 +- website/src/docs/mocha/v1/reliability.md | 2 +- 8 files changed, 481 insertions(+), 122 deletions(-) create mode 100644 src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryAction.cs create mode 100644 src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryDecision.cs diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs index d0c333fb6a0..5a55bf4ac41 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/ConsumerRetryMiddleware.cs @@ -17,13 +17,7 @@ public async ValueTask InvokeAsync(IConsumeContext context, ConsumerDelegate nex if (context.Headers.TryGetValue(MessageHeaders.Retry.DelayedRetryCount.Key, out var headerValue)) { - delayedRetryCount = headerValue switch - { - int i => i, - long l => (int)l, - double d => (int)d, - _ => 0 - }; + delayedRetryCount = RedeliveryExecutor.ParseDelayedRetryCount(headerValue); } // Expose retry state to handlers via features. diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs index a403718fc95..fc978445b3e 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs @@ -27,13 +27,7 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next if (context.Headers.TryGetValue(MessageHeaders.Retry.DelayedRetryCount.Key, out var headerValue)) { - delayedRetryCount = headerValue switch - { - int i => i, - long l => (int)l, - double d => (int)d, - _ => 0 - }; + delayedRetryCount = RedeliveryExecutor.ParseDelayedRetryCount(headerValue); } try @@ -42,100 +36,72 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next } catch (Exception ex) { - var consumerFeature = context.Features.GetOrSet(); - // Request/reply messages must not be redelivered -- the caller is waiting. if (context.Envelope?.ResponseAddress is not null) { throw; } - // Match exception against policy rules. - var rule = ExceptionPolicyMatcher.Match(exceptionPolicyRules, ex); - - // No matching rule - no policy for this exception, let it propagate. - if (rule is null) - { - throw; - } - - // Discard: swallow at receive level. - if (rule.Terminal == TerminalAction.Discard) - { - consumerFeature.MessageConsumed = true; - - return; - } - - // DeadLetter: skip redelivery, let fault middleware handle. - if (rule.Terminal == TerminalAction.DeadLetter) - { - throw; - } - - // No redelivery configured for this rule, or redelivery explicitly disabled. - if (rule.Redelivery is null or { Enabled: false }) - { - throw; - } - - var redeliveryConfig = rule.Redelivery; - - // Check if redelivery attempts remain. - var maxAttempts = redeliveryConfig.Attempts ?? redeliveryConfig.Intervals?.Length ?? 0; - - if (delayedRetryCount >= maxAttempts) - { - throw; - } - - // Calculate the delay for this redelivery attempt. - var delay = RedeliveryExecutor.CalculateDelay(delayedRetryCount, redeliveryConfig); - var scheduledTime = timeProvider.GetUtcNow().Add(delay); - - // Update the header on the envelope so the next delivery round sees the incremented count. - var envelope = context.Envelope; + var decision = RedeliveryExecutor.Evaluate(exceptionPolicyRules, ex, delayedRetryCount); - if (envelope is null) + switch (decision.Action) { - throw; + case RedeliveryAction.Discard: + context.Features.GetOrSet().MessageConsumed = true; + return; + + case RedeliveryAction.Redeliver: + var scheduledTime = timeProvider.GetUtcNow().Add(decision.Delay); + await DispatchRedeliveryAsync(context, delayedRetryCount, scheduledTime); + context.Features.GetOrSet().MessageConsumed = true; + return; + + default: + throw; } + } + } - if (envelope.Headers is null) - { - throw new InvalidOperationException( - "Cannot increment delayed retry count because the envelope has no headers collection."); - } + private async ValueTask DispatchRedeliveryAsync( + IReceiveContext context, + int delayedRetryCount, + DateTimeOffset scheduledTime) + { + var envelope = context.Envelope + ?? throw new InvalidOperationException( + "Cannot redeliver because the receive context has no envelope."); - envelope.Headers.Set(MessageHeaders.Retry.DelayedRetryCount.Key, delayedRetryCount + 1); + if (envelope.Headers is null) + { + throw new InvalidOperationException( + "Cannot increment delayed retry count because the envelope has no headers collection."); + } - // Dispatch the envelope back to the same endpoint with the scheduled time. - // Use the Source address (queue/topic) rather than the endpoint address, because - // transports resolve dispatch endpoints from the topology resource address. - var dispatchEndpoint = context.Runtime.GetDispatchEndpoint(context.Endpoint.Source.Address); - var dispatchContext = pools.DispatchContext.Get(); + envelope.Headers.Set(MessageHeaders.Retry.DelayedRetryCount.Key, delayedRetryCount + 1); - try - { - dispatchContext.Initialize( - context.Services, - dispatchEndpoint, - context.Runtime, - context.MessageType, - context.CancellationToken); + // Dispatch the envelope back to the same endpoint with the scheduled time. + // Use the Source address (queue/topic) rather than the endpoint address, because + // transports resolve dispatch endpoints from the topology resource address. + var dispatchEndpoint = context.Runtime.GetDispatchEndpoint(context.Endpoint.Source.Address); + var dispatchContext = pools.DispatchContext.Get(); - dispatchContext.Envelope = envelope; - dispatchContext.ScheduledTime = scheduledTime; + try + { + dispatchContext.Initialize( + context.Services, + dispatchEndpoint, + context.Runtime, + context.MessageType, + context.CancellationToken); - await dispatchEndpoint.ExecuteAsync(dispatchContext); - } - finally - { - pools.DispatchContext.Return(dispatchContext); - } + dispatchContext.Envelope = envelope; + dispatchContext.ScheduledTime = scheduledTime; - // Mark the message as consumed so Fault/DeadLetter don't also handle it. - consumerFeature.MessageConsumed = true; + await dispatchEndpoint.ExecuteAsync(dispatchContext); + } + finally + { + pools.DispatchContext.Return(dispatchContext); } } diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryAction.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryAction.cs new file mode 100644 index 00000000000..91205429ae8 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryAction.cs @@ -0,0 +1,22 @@ +namespace Mocha; + +/// +/// Specifies the action the redelivery middleware should take for a failed message. +/// +internal enum RedeliveryAction +{ + /// + /// Re-throw the exception; let outer middleware handle it. + /// + Rethrow, + + /// + /// Discard the message; swallow the exception. + /// + Discard, + + /// + /// Redeliver the message after a delay. + /// + Redeliver +} diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryDecision.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryDecision.cs new file mode 100644 index 00000000000..e613bc92343 --- /dev/null +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryDecision.cs @@ -0,0 +1,41 @@ +namespace Mocha; + +/// +/// The result of evaluating exception policy rules for a redelivery decision. +/// +internal readonly struct RedeliveryDecision +{ + private RedeliveryDecision(RedeliveryAction action, TimeSpan delay = default) + { + Action = action; + Delay = delay; + } + + /// + /// Gets the action to take. + /// + public RedeliveryAction Action { get; } + + /// + /// Gets the delay before redelivery. Only meaningful when is + /// . + /// + public TimeSpan Delay { get; } + + /// + /// A decision to re-throw the exception. + /// + public static readonly RedeliveryDecision Rethrow = new(RedeliveryAction.Rethrow); + + /// + /// A decision to discard the message. + /// + public static readonly RedeliveryDecision Discard = new(RedeliveryAction.Discard); + + /// + /// Creates a decision to redeliver the message after the specified delay. + /// + /// The delay before redelivery. + /// A redeliver decision. + public static RedeliveryDecision Redeliver(TimeSpan delay) => new(RedeliveryAction.Redeliver, delay); +} diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs index 86d9f032ab7..c2897fb69f9 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs @@ -1,10 +1,82 @@ +using System.Collections.Immutable; + namespace Mocha; /// -/// Provides delay calculation logic for the redelivery middleware. +/// Provides delay calculation and exception evaluation logic for the redelivery middleware. /// internal static class RedeliveryExecutor { + /// + /// Evaluates exception policy rules and determines whether to rethrow, discard, or redeliver. + /// + /// The exception policy rules to evaluate. + /// The exception that was thrown. + /// The current delayed retry count. + /// A indicating the action to take. + internal static RedeliveryDecision Evaluate( + ImmutableArray rules, + Exception exception, + int delayedRetryCount) + { + // Match exception against policy rules. + var rule = ExceptionPolicyMatcher.Match(rules, exception); + + // No matching rule — no policy for this exception. + if (rule is null) + { + return RedeliveryDecision.Rethrow; + } + + // Discard: swallow at receive level. + if (rule.Terminal == TerminalAction.Discard) + { + return RedeliveryDecision.Discard; + } + + // DeadLetter: skip redelivery, let fault middleware handle. + if (rule.Terminal == TerminalAction.DeadLetter) + { + return RedeliveryDecision.Rethrow; + } + + // No redelivery configured for this rule, or redelivery explicitly disabled. + if (rule.Redelivery is null or { Enabled: false }) + { + return RedeliveryDecision.Rethrow; + } + + var redeliveryConfig = rule.Redelivery; + + // Check if redelivery attempts remain. + var maxAttempts = redeliveryConfig.Attempts ?? redeliveryConfig.Intervals?.Length ?? 0; + + if (delayedRetryCount >= maxAttempts) + { + return RedeliveryDecision.Rethrow; + } + + // Calculate the delay for this redelivery attempt. + var delay = CalculateDelay(delayedRetryCount, redeliveryConfig); + return RedeliveryDecision.Redeliver(delay); + } + + /// + /// Parses a delayed retry count from a message header value. + /// + /// The raw header value. + /// The parsed integer count, or 0 if the value cannot be parsed. + internal static int ParseDelayedRetryCount(object? headerValue) + { + return headerValue switch + { + int i => i, + long l => (int)l, + double d => (int)d, + _ => 0 + }; + } + internal static TimeSpan CalculateDelay(int attempt, RedeliveryPolicyConfig config) { TimeSpan baseDelay; diff --git a/src/Mocha/test/Mocha.Tests/Middlewares/Receive/Redelivery/RedeliveryExecutorTests.cs b/src/Mocha/test/Mocha.Tests/Middlewares/Receive/Redelivery/RedeliveryExecutorTests.cs index 11fa5350aeb..cd5141b0560 100644 --- a/src/Mocha/test/Mocha.Tests/Middlewares/Receive/Redelivery/RedeliveryExecutorTests.cs +++ b/src/Mocha/test/Mocha.Tests/Middlewares/Receive/Redelivery/RedeliveryExecutorTests.cs @@ -29,9 +29,7 @@ public void CalculateDelay_Should_ClampToLastInterval_When_AttemptExceedsInterva // arrange var config = new RedeliveryPolicyConfig { - Intervals = ImmutableArray.Create( - TimeSpan.FromMinutes(5), - TimeSpan.FromMinutes(15)), + Intervals = ImmutableArray.Create(TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(15)), UseJitter = false }; @@ -43,12 +41,10 @@ public void CalculateDelay_Should_ClampToLastInterval_When_AttemptExceedsInterva } [Theory] - [InlineData(0, 10)] // BaseDelay * (0 + 1) = 10min - [InlineData(1, 20)] // BaseDelay * (1 + 1) = 20min - [InlineData(2, 30)] // BaseDelay * (2 + 1) = 30min - public void CalculateDelay_Should_ReturnLinearDelay_When_NoIntervalsProvided( - int attempt, - int expectedMinutes) + [InlineData(0, 10)] // BaseDelay * (0 + 1) = 10min + [InlineData(1, 20)] // BaseDelay * (1 + 1) = 20min + [InlineData(2, 30)] // BaseDelay * (2 + 1) = 30min + public void CalculateDelay_Should_ReturnLinearDelay_When_NoIntervalsProvided(int attempt, int expectedMinutes) { // arrange var config = new RedeliveryPolicyConfig @@ -69,11 +65,7 @@ public void CalculateDelay_Should_ReturnLinearDelay_When_NoIntervalsProvided( public void CalculateDelay_Should_UseDefaultBaseDelay_When_NoBaseDelayAndNoIntervals() { // arrange - no BaseDelay, no Intervals => uses RedeliveryPolicyDefaults.Intervals[0] (5 min) - var config = new RedeliveryPolicyConfig - { - UseJitter = false, - MaxDelay = TimeSpan.FromHours(2) - }; + var config = new RedeliveryPolicyConfig { UseJitter = false, MaxDelay = TimeSpan.FromHours(2) }; // act var delay = RedeliveryExecutor.CalculateDelay(0, config); @@ -86,11 +78,7 @@ public void CalculateDelay_Should_UseDefaultBaseDelay_When_NoBaseDelayAndNoInter public void CalculateDelay_Should_ScaleDefaultBaseDelay_When_AttemptIncreases() { // arrange - var config = new RedeliveryPolicyConfig - { - UseJitter = false, - MaxDelay = TimeSpan.FromHours(2) - }; + var config = new RedeliveryPolicyConfig { UseJitter = false, MaxDelay = TimeSpan.FromHours(2) }; // act & assert - 5min * (attempt + 1) Assert.Equal(TimeSpan.FromMinutes(5), RedeliveryExecutor.CalculateDelay(0, config)); @@ -120,11 +108,7 @@ public void CalculateDelay_Should_CapAtMaxDelay_When_ComputedDelayExceedsMax() public void CalculateDelay_Should_UseDefaultMaxDelay_When_NoMaxDelayConfigured() { // arrange - no MaxDelay => uses RedeliveryPolicyDefaults.MaxDelay (1 hour) - var config = new RedeliveryPolicyConfig - { - BaseDelay = TimeSpan.FromMinutes(30), - UseJitter = false - }; + var config = new RedeliveryPolicyConfig { BaseDelay = TimeSpan.FromMinutes(30), UseJitter = false }; // act - 30min * (5 + 1) = 180min, capped at 60min (default) var delay = RedeliveryExecutor.CalculateDelay(5, config); @@ -221,9 +205,7 @@ public void CalculateDelay_Should_CapExplicitIntervalAtMaxDelay_When_IntervalExc // arrange var config = new RedeliveryPolicyConfig { - Intervals = ImmutableArray.Create( - TimeSpan.FromMinutes(5), - TimeSpan.FromHours(2)), + Intervals = ImmutableArray.Create(TimeSpan.FromMinutes(5), TimeSpan.FromHours(2)), MaxDelay = TimeSpan.FromHours(1), UseJitter = false }; @@ -234,4 +216,286 @@ public void CalculateDelay_Should_CapExplicitIntervalAtMaxDelay_When_IntervalExc // assert Assert.Equal(TimeSpan.FromHours(1), delay); } + + [Fact] + public void ParseDelayedRetryCount_Should_ReturnIntValue_When_HeaderValueIsInt() + { + // act + var result = RedeliveryExecutor.ParseDelayedRetryCount(42); + + // assert + Assert.Equal(42, result); + } + + [Fact] + public void ParseDelayedRetryCount_Should_ReturnConvertedValue_When_HeaderValueIsLong() + { + // act + var result = RedeliveryExecutor.ParseDelayedRetryCount(7L); + + // assert + Assert.Equal(7, result); + } + + [Fact] + public void ParseDelayedRetryCount_Should_ReturnConvertedValue_When_HeaderValueIsDouble() + { + // act + var result = RedeliveryExecutor.ParseDelayedRetryCount(3.0); + + // assert + Assert.Equal(3, result); + } + + [Fact] + public void ParseDelayedRetryCount_Should_ReturnZero_When_HeaderValueIsString() + { + // act + var result = RedeliveryExecutor.ParseDelayedRetryCount("not a number"); + + // assert + Assert.Equal(0, result); + } + + [Fact] + public void ParseDelayedRetryCount_Should_ReturnZero_When_HeaderValueIsNull() + { + // act + var result = RedeliveryExecutor.ParseDelayedRetryCount(null); + + // assert + Assert.Equal(0, result); + } + + [Fact] + public void Evaluate_Should_ReturnRethrow_When_NoRuleMatchesException() + { + // arrange - rules for ArgumentException, but we throw InvalidOperationException + var rules = BuildRules(p => p.On().Redeliver()); + + // act + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("no match"), 0); + + // assert + Assert.Equal(RedeliveryAction.Rethrow, decision.Action); + } + + [Fact] + public void Evaluate_Should_ReturnDiscard_When_TerminalIsDiscard() + { + // arrange + var rules = BuildRules(p => p.On().Discard()); + + // act + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("discard me"), 0); + + // assert + Assert.Equal(RedeliveryAction.Discard, decision.Action); + } + + [Fact] + public void Evaluate_Should_ReturnRethrow_When_TerminalIsDeadLetter() + { + // arrange + var rules = BuildRules(p => p.On().DeadLetter()); + + // act + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("dead letter"), 0); + + // assert + Assert.Equal(RedeliveryAction.Rethrow, decision.Action); + } + + [Fact] + public void Evaluate_Should_ReturnRethrow_When_RedeliveryIsDisabled() + { + // arrange - Retry() sets Redelivery.Enabled = false + var rules = BuildRules(p => p.On().Retry()); + + // act + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("no redelivery"), 0); + + // assert + Assert.Equal(RedeliveryAction.Rethrow, decision.Action); + } + + [Fact] + public void Evaluate_Should_ReturnRethrow_When_RedeliveryIsNull() + { + // arrange - manually build a rule with no redelivery config + var rules = ImmutableArray.Create( + new ExceptionPolicyRule + { + ExceptionType = typeof(Exception), + Predicate = null, + Redelivery = null + }); + + // act + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("no config"), 0); + + // assert + Assert.Equal(RedeliveryAction.Rethrow, decision.Action); + } + + [Fact] + public void Evaluate_Should_ReturnRedeliver_When_RedeliveryConfigured() + { + // arrange - Redeliver() uses defaults (3 intervals, jitter enabled) + var rules = BuildRules(p => p.On().Redeliver()); + + // act + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("redeliver me"), 0); + + // assert + Assert.Equal(RedeliveryAction.Redeliver, decision.Action); + Assert.True(decision.Delay > TimeSpan.Zero); + } + + [Fact] + public void Evaluate_Should_ReturnRethrow_When_AllRedeliveryAttemptsExhausted() + { + // arrange - 2 attempts configured, delayedRetryCount already at 2 + var rules = BuildRules(p => p.On().Redeliver(2, TimeSpan.FromMinutes(5))); + + // act + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("exhausted"), 2); + + // assert + Assert.Equal(RedeliveryAction.Rethrow, decision.Action); + } + + [Fact] + public void Evaluate_Should_ReturnRedeliver_When_AttemptsRemain() + { + // arrange - 3 attempts configured, delayedRetryCount at 1 + var rules = BuildRules(p => p.On().Redeliver(3, TimeSpan.FromMinutes(5))); + + // act + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("retry"), 1); + + // assert + Assert.Equal(RedeliveryAction.Redeliver, decision.Action); + Assert.True(decision.Delay > TimeSpan.Zero); + } + + [Fact] + public void Evaluate_Should_ReturnDiscard_When_MostSpecificRuleIsDiscard() + { + // arrange - base rule redelivers, but more specific rule discards + var rules = BuildRules(p => + { + p.On().Redeliver(); + p.On().Discard(); + }); + + // act + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("specific discard"), 0); + + // assert + Assert.Equal(RedeliveryAction.Discard, decision.Action); + } + + [Fact] + public void Evaluate_Should_ReturnRethrow_When_RulesListIsEmpty() + { + // arrange + var rules = BuildRules(_ => { }); + + // act + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("no rules"), 0); + + // assert + Assert.Equal(RedeliveryAction.Rethrow, decision.Action); + } + + [Fact] + public void Evaluate_Should_ReturnRethrow_When_PredicateDoesNotMatch() + { + // arrange + var rules = BuildRules(p => + p.On(static ex => ex.Message.Contains("transient")).Discard() + ); + + // act + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("permanent failure"), 0); + + // assert + Assert.Equal(RedeliveryAction.Rethrow, decision.Action); + } + + [Fact] + public void Evaluate_Should_ReturnDiscard_When_PredicateMatches() + { + // arrange + var rules = BuildRules(p => + p.On(static ex => ex.Message.Contains("transient")).Discard() + ); + + // act + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("transient failure"), 0); + + // assert + Assert.Equal(RedeliveryAction.Discard, decision.Action); + } + + [Fact] + public void Evaluate_Should_UseIntervalLengthAsMaxAttempts_When_AttemptsNotSet() + { + // arrange - 3 explicit intervals, no Attempts property + var rules = BuildRules(p => + p.On() + .Redeliver([TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(30)]) + ); + + // act - attempt 2 (0-based), which is the 3rd interval → should still redeliver + var decision2 = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("retry"), 2); + + // act - attempt 3 (0-based), all 3 intervals exhausted → should rethrow + var decision3 = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("retry"), 3); + + // assert + Assert.Equal(RedeliveryAction.Redeliver, decision2.Action); + Assert.Equal(RedeliveryAction.Rethrow, decision3.Action); + } + + [Fact] + public void Evaluate_Should_ReturnRethrow_When_MaxAttemptsIsZero() + { + // arrange - rule with Redelivery having no Attempts and no Intervals → maxAttempts = 0 + var rules = ImmutableArray.Create( + new ExceptionPolicyRule + { + ExceptionType = typeof(Exception), + Predicate = null, + Redelivery = new RedeliveryPolicyConfig { Enabled = true, UseJitter = false } + }); + + // act + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("no attempts"), 0); + + // assert + Assert.Equal(RedeliveryAction.Rethrow, decision.Action); + } + + [Fact] + public void Evaluate_Should_ReturnRedeliver_When_DefaultRuleFallsThrough() + { + // arrange - Default() matches all exceptions, including derived types + var rules = BuildRules(p => p.Default().Redeliver()); + + // act - throw a derived exception + var decision = RedeliveryExecutor.Evaluate(rules, new InvalidOperationException("derived"), 0); + + // assert + Assert.Equal(RedeliveryAction.Redeliver, decision.Action); + Assert.True(decision.Delay > TimeSpan.Zero); + } + + private static ImmutableArray BuildRules(Action configure) + { + var feature = new ExceptionPolicyFeature(); + feature.Configure(configure); + return [.. feature.Rules]; + } } diff --git a/website/src/docs/mocha/v1/exception-policies.md b/website/src/docs/mocha/v1/exception-policies.md index 5020817eec1..b40e78c042c 100644 --- a/website/src/docs/mocha/v1/exception-policies.md +++ b/website/src/docs/mocha/v1/exception-policies.md @@ -184,7 +184,7 @@ Exception policies resolve at four levels. The most specific scope wins, and rep | Transport | All endpoints on a specific transport | `IReceiveMiddlewareProvider` | | Consumer | A single consumer | `IConsumerDescriptor` | -``` +```text Consumer policies → Transport policies → Bus policies → Host policies (highest priority) (lowest priority) ``` diff --git a/website/src/docs/mocha/v1/reliability.md b/website/src/docs/mocha/v1/reliability.md index da44c413666..cbd8d77d849 100644 --- a/website/src/docs/mocha/v1/reliability.md +++ b/website/src/docs/mocha/v1/reliability.md @@ -454,7 +454,7 @@ Message arrives The total number of handler invocations before a message reaches the error endpoint: -``` +```text Total attempts = (retry attempts + 1) x (redelivery attempts + 1) ``` From fd0f858026fbb55c494eab36387e8a749bf5a5dd Mon Sep 17 00:00:00 2001 From: Pascal Senn Date: Wed, 8 Apr 2026 01:55:29 +0200 Subject: [PATCH 12/14] cleanup --- .../Exceptions/Exceptions.cs | 14 +++++----- .../ExceptionPolicies/Handlers/Handlers.cs | 8 +++--- .../examples/ExceptionPolicies/Program.cs | 18 ++++++------ .../Receive/Redelivery/RedeliveryExecutor.cs | 2 +- .../NotificationStrategyTests.cs | 2 +- .../Behaviors/RedeliveryTests.cs | 2 +- website/src/docs/mocha/v1/diagnostics.md | 28 +++++++++---------- .../src/docs/mocha/v1/handler-registration.md | 2 +- 8 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/Mocha/examples/ExceptionPolicies/Exceptions/Exceptions.cs b/src/Mocha/examples/ExceptionPolicies/Exceptions/Exceptions.cs index 4154a60a0ee..09d0894b7c1 100644 --- a/src/Mocha/examples/ExceptionPolicies/Exceptions/Exceptions.cs +++ b/src/Mocha/examples/ExceptionPolicies/Exceptions/Exceptions.cs @@ -1,32 +1,32 @@ namespace ExceptionPolicies.Exceptions; /// -/// Transient database failure — worth retrying because it usually resolves quickly. +/// Transient database failure worth retrying because it usually resolves quickly. /// public class TransientDatabaseException(string message) : Exception(message); /// -/// The message payload is malformed — retrying will never help. +/// The message payload is malformed retrying will never help. /// public class MessageValidationException(string message) : Exception(message); /// -/// The message was already processed — expected in at-least-once delivery. +/// The message was already processed expected in at-least-once delivery. /// public class DuplicateMessageException(string message) : Exception(message); /// -/// Payment gateway returned an error — flaky but usually recovers. +/// Payment gateway returned an error flaky but usually recovers. /// public class PaymentGatewayException(string message) : Exception(message); /// -/// Auth token expired — immediate retry is pointless, need to wait for refresh. +/// Auth token expired immediate retry is pointless, need to wait for refresh. /// public class AuthTokenExpiredException(string message) : Exception(message); /// -/// External service is completely unavailable — needs time to recover. +/// External service is completely unavailable needs time to recover. /// public class ExternalServiceUnavailableException(string message) : Exception(message); @@ -39,6 +39,6 @@ public class HttpServiceException(string message, int statusCode) : Exception(me } /// -/// Corrupt or unparseable message payload — a poison message. +/// Corrupt or unparseable message payload a poison message. /// public class PoisonMessageException(string message) : Exception(message); diff --git a/src/Mocha/examples/ExceptionPolicies/Handlers/Handlers.cs b/src/Mocha/examples/ExceptionPolicies/Handlers/Handlers.cs index 7b48de3e008..8d93c767c0e 100644 --- a/src/Mocha/examples/ExceptionPolicies/Handlers/Handlers.cs +++ b/src/Mocha/examples/ExceptionPolicies/Handlers/Handlers.cs @@ -29,7 +29,7 @@ public ValueTask HandleAsync(ProcessPayment message, CancellationToken cancellat /// /// Receives a message with an invalid payload. -/// Policy: DeadLetter immediately — the message is permanently bad. +/// Policy: DeadLetter immediately the message is permanently bad. /// public class ValidateOrderHandler : IEventHandler { @@ -42,7 +42,7 @@ public ValueTask HandleAsync(ValidateOrder message, CancellationToken cancellati /// /// Detects a duplicate message that was already processed. -/// Policy: Discard silently — no retry, no dead-letter. +/// Policy: Discard silently no retry, no dead-letter. /// public class DeduplicateMessageHandler : IEventHandler { @@ -71,7 +71,7 @@ public ValueTask HandleAsync(CallExternalApi message, CancellationToken cancella /// /// Service with an expired auth token. -/// Policy: Redeliver only (skip retry) — immediate retry won't help. +/// Policy: Redeliver only (skip retry) immediate retry won't help. /// public class RefreshAuthTokenHandler : IEventHandler { @@ -110,7 +110,7 @@ public ValueTask HandleAsync(ProcessBatch message, CancellationToken cancellatio /// /// Ingests telemetry data from a device, encountering various HTTP errors. -/// Policy: Conditional — different behavior for 404, 429, and 503 status codes. +/// Policy: Conditional different behavior for 404, 429, and 503 status codes. /// public class IngestTelemetryHandler : IEventHandler { diff --git a/src/Mocha/examples/ExceptionPolicies/Program.cs b/src/Mocha/examples/ExceptionPolicies/Program.cs index 597d209fde5..d7752e8c5af 100644 --- a/src/Mocha/examples/ExceptionPolicies/Program.cs +++ b/src/Mocha/examples/ExceptionPolicies/Program.cs @@ -9,7 +9,7 @@ // Exception Policies Demo // // Demonstrates all per-exception policy configurations available in Mocha. -// Uses the InMemory transport for simplicity — no external dependencies. +// Uses the InMemory transport for simplicity no external dependencies. // --------------------------------------------------------------------------- var builder = Host.CreateApplicationBuilder(args); @@ -18,7 +18,7 @@ .AddMessageBus() // ----------------------------------------------------------------------- - // Exception Policies — the main showcase + // Exception Policies the main showcase // // Per-exception rules are configured in a single AddResilience call. // The On() catch-all provides global retry/redelivery defaults. @@ -26,13 +26,13 @@ .AddResilience(policy => { // --- Terminal: DeadLetter --- - // Validation errors are permanent — the message payload is bad. + // Validation errors are permanent the message payload is bad. // Skip retry and redelivery entirely; route straight to the error endpoint. policy.On().DeadLetter(); // --- Terminal: Discard --- // Duplicate messages are expected in at-least-once delivery systems. - // Silently drop them — no retry, no redelivery, no error endpoint. + // Silently drop them no retry, no redelivery, no error endpoint. policy.On().Discard(); // --- Retry only (skip redelivery) --- @@ -45,26 +45,26 @@ backoff: RetryBackoffType.Exponential); // --- Redeliver only (skip retry) --- - // Auth token expired — immediate retry is pointless because the token + // Auth token expired immediate retry is pointless because the token // won't refresh in milliseconds. Wait for redelivery instead. policy.On().Redeliver(); // --- Escalation: Retry then Redeliver --- - // Transient DB errors — try a few times quickly (connection hiccup), + // Transient DB errors try a few times quickly (connection hiccup), // then back off with redelivery if the database is truly struggling. policy.On() .Retry(attempts: 3) .ThenRedeliver(); // --- Escalation: Retry then DeadLetter (skip redelivery) --- - // Poison messages — try once in case it was a transient parse glitch, + // Poison messages try once in case it was a transient parse glitch, // then give up immediately. Redelivery won't fix a corrupt payload. policy.On() .Retry(attempts: 1) .ThenDeadLetter(); // --- Full chain: Retry -> Redeliver -> DeadLetter --- - // External service completely down — aggressive retry first, then + // External service completely down aggressive retry first, then // patient redelivery with increasing intervals, then dead-letter // as the last resort so operators can investigate. policy.On() @@ -117,7 +117,7 @@ .AddEventHandler() // ----------------------------------------------------------------------- - // Transport — InMemory for this demo (no external dependencies) + // Transport InMemory for this demo (no external dependencies) // ----------------------------------------------------------------------- .AddInMemory(); diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs index c2897fb69f9..31b8e86efe4 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs @@ -22,7 +22,7 @@ internal static RedeliveryDecision Evaluate( // Match exception against policy rules. var rule = ExceptionPolicyMatcher.Match(rules, exception); - // No matching rule — no policy for this exception. + // No matching rule no policy for this exception. if (rule is null) { return RedeliveryDecision.Rethrow; diff --git a/src/Mocha/test/Mocha.Mediator.Tests/NotificationStrategyTests.cs b/src/Mocha/test/Mocha.Mediator.Tests/NotificationStrategyTests.cs index bbd0f582004..c6c67464334 100644 --- a/src/Mocha/test/Mocha.Mediator.Tests/NotificationStrategyTests.cs +++ b/src/Mocha/test/Mocha.Mediator.Tests/NotificationStrategyTests.cs @@ -143,7 +143,7 @@ public async Task PublishAsync_Should_PropagateException_When_TaskWhenAllHandler using var scope = provider.CreateScope(); var mediator = scope.ServiceProvider.GetRequiredService(); - // Act & Assert — concurrent mode surfaces AggregateException with all failures + // Act & Assert concurrent mode surfaces AggregateException with all failures var ex = await Assert.ThrowsAsync( () => mediator.PublishAsync(new StrategyTestNotification("throw")).AsTask()); diff --git a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs index 482d9f6bd4b..e936d5f6193 100644 --- a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs +++ b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs @@ -78,7 +78,7 @@ public async Task Redelivery_Should_SkipRedelivery_When_ExceptionIsIgnored() [Fact] public async Task Redelivery_Should_PassThrough_When_Disabled() { - // arrange — no exception policy configured, so retry and redelivery are both no-ops + // arrange no exception policy configured, so retry and redelivery are both no-ops var counter = new InvocationCounter(); await using var provider = await new ServiceCollection() diff --git a/website/src/docs/mocha/v1/diagnostics.md b/website/src/docs/mocha/v1/diagnostics.md index f49249288cc..3428b2ff8ac 100644 --- a/website/src/docs/mocha/v1/diagnostics.md +++ b/website/src/docs/mocha/v1/diagnostics.md @@ -5,7 +5,7 @@ description: "Reference for all compile-time diagnostics emitted by the Mocha so # Diagnostics -Mocha uses a Roslyn source generator to validate your message handlers, consumers, and sagas at compile time. When the generator detects a problem — a missing handler, a duplicate registration, an invalid type — it emits a diagnostic that appears as a compiler warning or error in your IDE and build output. You can fix these issues before your code ever runs. +Mocha uses a Roslyn source generator to validate your message handlers, consumers, and sagas at compile time. When the generator detects a problem a missing handler, a duplicate registration, an invalid type it emits a diagnostic that appears as a compiler warning or error in your IDE and build output. You can fix these issues before your code ever runs. # Quick reference @@ -23,7 +23,7 @@ Mocha uses a Roslyn source generator to validate your message handlers, consumer # Mediator diagnostics -These diagnostics apply to the in-process [mediator](/docs/mocha/v1/mediator) — commands, queries, and notifications dispatched within a single process. +These diagnostics apply to the in-process [mediator](/docs/mocha/v1/mediator) commands, queries, and notifications dispatched within a single process. ## MO0001 @@ -43,7 +43,7 @@ A command or query type is declared but no corresponding handler implementation ```csharp using Mocha.Mediator; -// Command with no handler — triggers MO0001 +// Command with no handler triggers MO0001 public record PlaceOrder(Guid OrderId, decimal Total) : ICommand; ``` @@ -79,7 +79,7 @@ public class PlaceOrderHandler : ICommandHandler ### Cause -A command or query type has more than one handler implementation. Commands and queries require exactly one handler — the mediator cannot decide which one to call. This diagnostic does not apply to notifications, which support multiple handlers by design. +A command or query type has more than one handler implementation. Commands and queries require exactly one handler the mediator cannot decide which one to call. This diagnostic does not apply to notifications, which support multiple handlers by design. ### Example @@ -88,7 +88,7 @@ using Mocha.Mediator; public record PlaceOrder(Guid OrderId, decimal Total) : ICommand; -// Two handlers for the same command — triggers MO0002 +// Two handlers for the same command triggers MO0002 public class PlaceOrderHandler : ICommandHandler { public ValueTask HandleAsync(PlaceOrder command, CancellationToken ct) @@ -138,7 +138,7 @@ using Mocha.Mediator; public record GetOrderTotal(Guid OrderId) : IQuery; -// Abstract handler — triggers MO0003 +// Abstract handler triggers MO0003 public abstract class GetOrderTotalHandler : IQueryHandler { public abstract ValueTask HandleAsync( @@ -185,7 +185,7 @@ A command or query type has unbound type parameters. The mediator dispatches con ```csharp using Mocha.Mediator; -// Open generic command — triggers MO0004 +// Open generic command triggers MO0004 public record ProcessItem(T Item) : ICommand; ``` @@ -221,7 +221,7 @@ using Mocha.Mediator; public record PlaceOrder(Guid OrderId) : ICommand; public record GetOrder(Guid OrderId) : IQuery; -// Implements both command and query handler — triggers MO0005 +// Implements both command and query handler triggers MO0005 public class OrderHandler : ICommandHandler, IQueryHandler @@ -259,7 +259,7 @@ public class GetOrderHandler : IQueryHandler # Messaging diagnostics -These diagnostics apply to the [message bus](/docs/mocha/v1/handlers-and-consumers) — event handlers, request handlers, batch handlers, consumers, and sagas that communicate across service boundaries. +These diagnostics apply to the [message bus](/docs/mocha/v1/handlers-and-consumers) event handlers, request handlers, batch handlers, consumers, and sagas that communicate across service boundaries. ## MO0011 @@ -272,7 +272,7 @@ These diagnostics apply to the [message bus](/docs/mocha/v1/handlers-and-consume ### Cause -A request type (used with `SendAsync` or `RequestAsync`) has more than one [handler](/docs/mocha/v1/handlers-and-consumers) implementation. Request types require exactly one handler — the bus cannot route to multiple targets. +A request type (used with `SendAsync` or `RequestAsync`) has more than one [handler](/docs/mocha/v1/handlers-and-consumers) implementation. Request types require exactly one handler the bus cannot route to multiple targets. ### Example @@ -281,7 +281,7 @@ using Mocha; public record ProcessPayment(decimal Amount); -// Two handlers for the same request type — triggers MO0011 +// Two handlers for the same request type triggers MO0011 public class PaymentHandlerA : IEventRequestHandler { public ValueTask HandleAsync( @@ -335,7 +335,7 @@ A messaging handler (`IEventHandler`, `IEventRequestHandler`, `IBatchEvent ```csharp using Mocha; -// Open generic handler — triggers MO0012 +// Open generic handler triggers MO0012 public class GenericEventHandler : IEventHandler { public ValueTask HandleAsync( @@ -385,7 +385,7 @@ using Mocha; public record OrderPlaced(Guid OrderId); -// Abstract handler — triggers MO0013 +// Abstract handler triggers MO0013 public abstract class OrderEventHandler : IEventHandler { public abstract ValueTask HandleAsync( @@ -435,7 +435,7 @@ public class RefundSagaState : SagaStateBase public Guid OrderId { get; set; } } -// Constructor requires a parameter — triggers MO0014 +// Constructor requires a parameter triggers MO0014 public class RefundSaga : Saga { private readonly ILogger _logger; diff --git a/website/src/docs/mocha/v1/handler-registration.md b/website/src/docs/mocha/v1/handler-registration.md index 08183ffde05..12952344aa7 100644 --- a/website/src/docs/mocha/v1/handler-registration.md +++ b/website/src/docs/mocha/v1/handler-registration.md @@ -79,7 +79,7 @@ builder.Services .AddRabbitMQ(); ``` -You can mix source-generated and manual registration freely. If both the source generator and manual code register the same handler type, the configurations are composed — the source generator sets up the base registration and your manual call layers additional configuration (such as consumer middleware) on top. +You can mix source-generated and manual registration freely. If both the source generator and manual code register the same handler type, the configurations are composed the source generator sets up the base registration and your manual call layers additional configuration (such as consumer middleware) on top. > **Prefer the source generator.** Manual registration methods use runtime reflection to create handler consumers. The source generator produces direct, reflection-free factory calls. We guarantee backwards compatibility for the source-generated registration path; the manual registration API is stable at the surface level but its internal behavior may evolve. From 69d117bbce06cd893f5d7a03b7884aaa1587f612 Mon Sep 17 00:00:00 2001 From: Pascal Senn Date: Wed, 8 Apr 2026 19:11:38 +0200 Subject: [PATCH 13/14] Fix test hang by replacing custom DelayAsync with Task.Delay and revert cosmetic changes The RetryExecutor used a custom DelayAsync method with TaskCompletionSource that caused thread pool starvation hangs when tests ran in parallel. Replaced with the built-in Task.Delay(TimeSpan, TimeProvider, CancellationToken) overload. Rewrote two FakeTimeProvider-dependent tests to avoid unawaited async operations that deadlock under thread pool contention. Also reverts unintentional em-dash removal in examples, docs, and unrelated test files. --- .../Exceptions/Exceptions.cs | 14 ++-- .../ExceptionPolicies/Handlers/Handlers.cs | 8 +- .../examples/ExceptionPolicies/Program.cs | 22 ++--- .../Consume/Retry/RetryExecutor.cs | 16 +--- .../Receive/Redelivery/RedeliveryExecutor.cs | 2 +- .../NotificationStrategyTests.cs | 2 +- .../Consume/Retry/RetryExecutorTests.cs | 81 ++++++++++--------- .../Behaviors/RedeliveryTests.cs | 2 +- website/src/docs/mocha/v1/diagnostics.md | 28 +++---- .../src/docs/mocha/v1/handler-registration.md | 2 +- 10 files changed, 85 insertions(+), 92 deletions(-) diff --git a/src/Mocha/examples/ExceptionPolicies/Exceptions/Exceptions.cs b/src/Mocha/examples/ExceptionPolicies/Exceptions/Exceptions.cs index 09d0894b7c1..4154a60a0ee 100644 --- a/src/Mocha/examples/ExceptionPolicies/Exceptions/Exceptions.cs +++ b/src/Mocha/examples/ExceptionPolicies/Exceptions/Exceptions.cs @@ -1,32 +1,32 @@ namespace ExceptionPolicies.Exceptions; /// -/// Transient database failure worth retrying because it usually resolves quickly. +/// Transient database failure — worth retrying because it usually resolves quickly. /// public class TransientDatabaseException(string message) : Exception(message); /// -/// The message payload is malformed retrying will never help. +/// The message payload is malformed — retrying will never help. /// public class MessageValidationException(string message) : Exception(message); /// -/// The message was already processed expected in at-least-once delivery. +/// The message was already processed — expected in at-least-once delivery. /// public class DuplicateMessageException(string message) : Exception(message); /// -/// Payment gateway returned an error flaky but usually recovers. +/// Payment gateway returned an error — flaky but usually recovers. /// public class PaymentGatewayException(string message) : Exception(message); /// -/// Auth token expired immediate retry is pointless, need to wait for refresh. +/// Auth token expired — immediate retry is pointless, need to wait for refresh. /// public class AuthTokenExpiredException(string message) : Exception(message); /// -/// External service is completely unavailable needs time to recover. +/// External service is completely unavailable — needs time to recover. /// public class ExternalServiceUnavailableException(string message) : Exception(message); @@ -39,6 +39,6 @@ public class HttpServiceException(string message, int statusCode) : Exception(me } /// -/// Corrupt or unparseable message payload a poison message. +/// Corrupt or unparseable message payload — a poison message. /// public class PoisonMessageException(string message) : Exception(message); diff --git a/src/Mocha/examples/ExceptionPolicies/Handlers/Handlers.cs b/src/Mocha/examples/ExceptionPolicies/Handlers/Handlers.cs index 8d93c767c0e..7b48de3e008 100644 --- a/src/Mocha/examples/ExceptionPolicies/Handlers/Handlers.cs +++ b/src/Mocha/examples/ExceptionPolicies/Handlers/Handlers.cs @@ -29,7 +29,7 @@ public ValueTask HandleAsync(ProcessPayment message, CancellationToken cancellat /// /// Receives a message with an invalid payload. -/// Policy: DeadLetter immediately the message is permanently bad. +/// Policy: DeadLetter immediately — the message is permanently bad. /// public class ValidateOrderHandler : IEventHandler { @@ -42,7 +42,7 @@ public ValueTask HandleAsync(ValidateOrder message, CancellationToken cancellati /// /// Detects a duplicate message that was already processed. -/// Policy: Discard silently no retry, no dead-letter. +/// Policy: Discard silently — no retry, no dead-letter. /// public class DeduplicateMessageHandler : IEventHandler { @@ -71,7 +71,7 @@ public ValueTask HandleAsync(CallExternalApi message, CancellationToken cancella /// /// Service with an expired auth token. -/// Policy: Redeliver only (skip retry) immediate retry won't help. +/// Policy: Redeliver only (skip retry) — immediate retry won't help. /// public class RefreshAuthTokenHandler : IEventHandler { @@ -110,7 +110,7 @@ public ValueTask HandleAsync(ProcessBatch message, CancellationToken cancellatio /// /// Ingests telemetry data from a device, encountering various HTTP errors. -/// Policy: Conditional different behavior for 404, 429, and 503 status codes. +/// Policy: Conditional — different behavior for 404, 429, and 503 status codes. /// public class IngestTelemetryHandler : IEventHandler { diff --git a/src/Mocha/examples/ExceptionPolicies/Program.cs b/src/Mocha/examples/ExceptionPolicies/Program.cs index d7752e8c5af..27892d97516 100644 --- a/src/Mocha/examples/ExceptionPolicies/Program.cs +++ b/src/Mocha/examples/ExceptionPolicies/Program.cs @@ -9,7 +9,7 @@ // Exception Policies Demo // // Demonstrates all per-exception policy configurations available in Mocha. -// Uses the InMemory transport for simplicity no external dependencies. +// Uses the InMemory transport for simplicity — no external dependencies. // --------------------------------------------------------------------------- var builder = Host.CreateApplicationBuilder(args); @@ -18,21 +18,21 @@ .AddMessageBus() // ----------------------------------------------------------------------- - // Exception Policies the main showcase + // Exception Policies — the main showcase // - // Per-exception rules are configured in a single AddResilience call. + // Per-exception rules are configured in a single AddExceptionPolicy call. // The On() catch-all provides global retry/redelivery defaults. // ----------------------------------------------------------------------- - .AddResilience(policy => + .AddExceptionPolicy(policy => { // --- Terminal: DeadLetter --- - // Validation errors are permanent the message payload is bad. + // Validation errors are permanent — the message payload is bad. // Skip retry and redelivery entirely; route straight to the error endpoint. policy.On().DeadLetter(); // --- Terminal: Discard --- // Duplicate messages are expected in at-least-once delivery systems. - // Silently drop them no retry, no redelivery, no error endpoint. + // Silently drop them — no retry, no redelivery, no error endpoint. policy.On().Discard(); // --- Retry only (skip redelivery) --- @@ -45,26 +45,26 @@ backoff: RetryBackoffType.Exponential); // --- Redeliver only (skip retry) --- - // Auth token expired immediate retry is pointless because the token + // Auth token expired — immediate retry is pointless because the token // won't refresh in milliseconds. Wait for redelivery instead. policy.On().Redeliver(); // --- Escalation: Retry then Redeliver --- - // Transient DB errors try a few times quickly (connection hiccup), + // Transient DB errors — try a few times quickly (connection hiccup), // then back off with redelivery if the database is truly struggling. policy.On() .Retry(attempts: 3) .ThenRedeliver(); // --- Escalation: Retry then DeadLetter (skip redelivery) --- - // Poison messages try once in case it was a transient parse glitch, + // Poison messages — try once in case it was a transient parse glitch, // then give up immediately. Redelivery won't fix a corrupt payload. policy.On() .Retry(attempts: 1) .ThenDeadLetter(); // --- Full chain: Retry -> Redeliver -> DeadLetter --- - // External service completely down aggressive retry first, then + // External service completely down — aggressive retry first, then // patient redelivery with increasing intervals, then dead-letter // as the last resort so operators can investigate. policy.On() @@ -117,7 +117,7 @@ .AddEventHandler() // ----------------------------------------------------------------------- - // Transport InMemory for this demo (no external dependencies) + // Transport — InMemory for this demo (no external dependencies) // ----------------------------------------------------------------------- .AddInMemory(); diff --git a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs index a13201f7bdc..6e2afa3cf8d 100644 --- a/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs +++ b/src/Mocha/src/Mocha/Middlewares/Consume/Retry/RetryExecutor.cs @@ -77,26 +77,12 @@ public static async ValueTask ExecuteAsync( if (delay > TimeSpan.Zero) { - await DelayAsync(timeProvider, delay, cancellationToken); + await Task.Delay(delay, timeProvider, cancellationToken).ConfigureAwait(false); } } } } - private static async Task DelayAsync(TimeProvider timeProvider, TimeSpan delay, CancellationToken cancellationToken) - { - var tcs = new TaskCompletionSource(); - await using var timer = timeProvider.CreateTimer( - static s => ((TaskCompletionSource)s!).TrySetResult(), - tcs, - delay, - Timeout.InfiniteTimeSpan); - await using var registration = cancellationToken.Register( - static s => ((TaskCompletionSource)s!).TrySetCanceled(), - tcs); - await tcs.Task.ConfigureAwait(false); - } - internal static TimeSpan CalculateDelay(int attempt, RetryPolicyConfig config) { // Explicit intervals take precedence. diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs index 31b8e86efe4..c2897fb69f9 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/RedeliveryExecutor.cs @@ -22,7 +22,7 @@ internal static RedeliveryDecision Evaluate( // Match exception against policy rules. var rule = ExceptionPolicyMatcher.Match(rules, exception); - // No matching rule no policy for this exception. + // No matching rule — no policy for this exception. if (rule is null) { return RedeliveryDecision.Rethrow; diff --git a/src/Mocha/test/Mocha.Mediator.Tests/NotificationStrategyTests.cs b/src/Mocha/test/Mocha.Mediator.Tests/NotificationStrategyTests.cs index c6c67464334..bbd0f582004 100644 --- a/src/Mocha/test/Mocha.Mediator.Tests/NotificationStrategyTests.cs +++ b/src/Mocha/test/Mocha.Mediator.Tests/NotificationStrategyTests.cs @@ -143,7 +143,7 @@ public async Task PublishAsync_Should_PropagateException_When_TaskWhenAllHandler using var scope = provider.CreateScope(); var mediator = scope.ServiceProvider.GetRequiredService(); - // Act & Assert concurrent mode surfaces AggregateException with all failures + // Act & Assert — concurrent mode surfaces AggregateException with all failures var ex = await Assert.ThrowsAsync( () => mediator.PublishAsync(new StrategyTestNotification("throw")).AsTask()); diff --git a/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs index 9c0bee8f74f..2e5de6b8f33 100644 --- a/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs +++ b/src/Mocha/test/Mocha.Tests/Middlewares/Consume/Retry/RetryExecutorTests.cs @@ -1,5 +1,5 @@ using System.Collections.Immutable; -using Microsoft.Extensions.Time.Testing; +using System.Diagnostics; namespace Mocha.Tests.Middlewares.Consume.Retry; @@ -643,58 +643,65 @@ await Assert.ThrowsAsync(() => [Fact] public async Task ExecuteAsync_Should_ThrowOperationCanceled_When_CancellationRequestedDuringDelay() { - // arrange - var timeProvider = new FakeTimeProvider(); - var rules = BuildRules(p => p.On().Retry(3, TimeSpan.FromSeconds(10), RetryBackoffType.Constant)); + // arrange - use a short real delay with pre-cancelled token to verify cancellation propagation + var rules = BuildRules(p => p.On().Retry(3, TimeSpan.FromSeconds(1), RetryBackoffType.Constant)); using var cts = new CancellationTokenSource(); var counter = new Counter(); - // act - start the executor, it will block on the first retry delay - var task = RetryExecutor.ExecuteAsync( - rules, - counter, - static (s) => - { - s.Increment(); - throw new InvalidOperationException("fail"); - }, - onRetry: null, - timeProvider, - cts.Token); - - // Cancel while waiting for the delay + // Pre-cancel so the delay will throw OperationCanceledException immediately cts.Cancel(); - // assert - await Assert.ThrowsAnyAsync(() => task.AsTask()); + // act & assert + await Assert.ThrowsAnyAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new InvalidOperationException("fail"); + }, + onRetry: null, + cts.Token) + .AsTask() + ); + Assert.Equal(1, counter.Count); } [Fact] - public async Task ExecuteAsync_Should_NotComplete_When_DelayHasNotElapsed() + public async Task ExecuteAsync_Should_WaitForDelay_When_RetryDelayIsConfigured() { // arrange - var timeProvider = new FakeTimeProvider(); var rules = BuildRules(p => - p.On().Retry(1, TimeSpan.FromSeconds(5), RetryBackoffType.Constant) + p.On().Retry(1, TimeSpan.FromMilliseconds(200), RetryBackoffType.Constant) ); + var counter = new Counter(); + var sw = Stopwatch.StartNew(); - // act - start the executor; it will block on the retry delay - var task = RetryExecutor.ExecuteAsync( - rules, - 0, - static (_) => throw new InvalidOperationException("fail"), - onRetry: null, - timeProvider, - cancellationToken: default); + // act & assert - the executor should wait for the retry delay before the second attempt + await Assert.ThrowsAsync(() => + RetryExecutor + .ExecuteAsync( + rules, + counter, + static (s) => + { + s.Increment(); + throw new InvalidOperationException("fail"); + }, + onRetry: null, + cancellationToken: default) + .AsTask() + ); - // assert - task should not complete until time advances - Assert.False(task.IsCompleted); + sw.Stop(); - // Advance past the delay to unblock - timeProvider.Advance(TimeSpan.FromSeconds(5)); - // Second attempt also fails, retries exhausted - should throw - await Assert.ThrowsAsync(() => task.AsTask()); + // assert - delay of at least 200ms should have been applied + Assert.Equal(2, counter.Count); + Assert.True(sw.Elapsed >= TimeSpan.FromMilliseconds(150), + $"Expected delay of ~200ms but elapsed was {sw.Elapsed.TotalMilliseconds}ms"); } [Fact] diff --git a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs index e936d5f6193..482d9f6bd4b 100644 --- a/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs +++ b/src/Mocha/test/Mocha.Transport.InMemory.Tests/Behaviors/RedeliveryTests.cs @@ -78,7 +78,7 @@ public async Task Redelivery_Should_SkipRedelivery_When_ExceptionIsIgnored() [Fact] public async Task Redelivery_Should_PassThrough_When_Disabled() { - // arrange no exception policy configured, so retry and redelivery are both no-ops + // arrange — no exception policy configured, so retry and redelivery are both no-ops var counter = new InvocationCounter(); await using var provider = await new ServiceCollection() diff --git a/website/src/docs/mocha/v1/diagnostics.md b/website/src/docs/mocha/v1/diagnostics.md index 3428b2ff8ac..f49249288cc 100644 --- a/website/src/docs/mocha/v1/diagnostics.md +++ b/website/src/docs/mocha/v1/diagnostics.md @@ -5,7 +5,7 @@ description: "Reference for all compile-time diagnostics emitted by the Mocha so # Diagnostics -Mocha uses a Roslyn source generator to validate your message handlers, consumers, and sagas at compile time. When the generator detects a problem a missing handler, a duplicate registration, an invalid type it emits a diagnostic that appears as a compiler warning or error in your IDE and build output. You can fix these issues before your code ever runs. +Mocha uses a Roslyn source generator to validate your message handlers, consumers, and sagas at compile time. When the generator detects a problem — a missing handler, a duplicate registration, an invalid type — it emits a diagnostic that appears as a compiler warning or error in your IDE and build output. You can fix these issues before your code ever runs. # Quick reference @@ -23,7 +23,7 @@ Mocha uses a Roslyn source generator to validate your message handlers, consumer # Mediator diagnostics -These diagnostics apply to the in-process [mediator](/docs/mocha/v1/mediator) commands, queries, and notifications dispatched within a single process. +These diagnostics apply to the in-process [mediator](/docs/mocha/v1/mediator) — commands, queries, and notifications dispatched within a single process. ## MO0001 @@ -43,7 +43,7 @@ A command or query type is declared but no corresponding handler implementation ```csharp using Mocha.Mediator; -// Command with no handler triggers MO0001 +// Command with no handler — triggers MO0001 public record PlaceOrder(Guid OrderId, decimal Total) : ICommand; ``` @@ -79,7 +79,7 @@ public class PlaceOrderHandler : ICommandHandler ### Cause -A command or query type has more than one handler implementation. Commands and queries require exactly one handler the mediator cannot decide which one to call. This diagnostic does not apply to notifications, which support multiple handlers by design. +A command or query type has more than one handler implementation. Commands and queries require exactly one handler — the mediator cannot decide which one to call. This diagnostic does not apply to notifications, which support multiple handlers by design. ### Example @@ -88,7 +88,7 @@ using Mocha.Mediator; public record PlaceOrder(Guid OrderId, decimal Total) : ICommand; -// Two handlers for the same command triggers MO0002 +// Two handlers for the same command — triggers MO0002 public class PlaceOrderHandler : ICommandHandler { public ValueTask HandleAsync(PlaceOrder command, CancellationToken ct) @@ -138,7 +138,7 @@ using Mocha.Mediator; public record GetOrderTotal(Guid OrderId) : IQuery; -// Abstract handler triggers MO0003 +// Abstract handler — triggers MO0003 public abstract class GetOrderTotalHandler : IQueryHandler { public abstract ValueTask HandleAsync( @@ -185,7 +185,7 @@ A command or query type has unbound type parameters. The mediator dispatches con ```csharp using Mocha.Mediator; -// Open generic command triggers MO0004 +// Open generic command — triggers MO0004 public record ProcessItem(T Item) : ICommand; ``` @@ -221,7 +221,7 @@ using Mocha.Mediator; public record PlaceOrder(Guid OrderId) : ICommand; public record GetOrder(Guid OrderId) : IQuery; -// Implements both command and query handler triggers MO0005 +// Implements both command and query handler — triggers MO0005 public class OrderHandler : ICommandHandler, IQueryHandler @@ -259,7 +259,7 @@ public class GetOrderHandler : IQueryHandler # Messaging diagnostics -These diagnostics apply to the [message bus](/docs/mocha/v1/handlers-and-consumers) event handlers, request handlers, batch handlers, consumers, and sagas that communicate across service boundaries. +These diagnostics apply to the [message bus](/docs/mocha/v1/handlers-and-consumers) — event handlers, request handlers, batch handlers, consumers, and sagas that communicate across service boundaries. ## MO0011 @@ -272,7 +272,7 @@ These diagnostics apply to the [message bus](/docs/mocha/v1/handlers-and-consume ### Cause -A request type (used with `SendAsync` or `RequestAsync`) has more than one [handler](/docs/mocha/v1/handlers-and-consumers) implementation. Request types require exactly one handler the bus cannot route to multiple targets. +A request type (used with `SendAsync` or `RequestAsync`) has more than one [handler](/docs/mocha/v1/handlers-and-consumers) implementation. Request types require exactly one handler — the bus cannot route to multiple targets. ### Example @@ -281,7 +281,7 @@ using Mocha; public record ProcessPayment(decimal Amount); -// Two handlers for the same request type triggers MO0011 +// Two handlers for the same request type — triggers MO0011 public class PaymentHandlerA : IEventRequestHandler { public ValueTask HandleAsync( @@ -335,7 +335,7 @@ A messaging handler (`IEventHandler`, `IEventRequestHandler`, `IBatchEvent ```csharp using Mocha; -// Open generic handler triggers MO0012 +// Open generic handler — triggers MO0012 public class GenericEventHandler : IEventHandler { public ValueTask HandleAsync( @@ -385,7 +385,7 @@ using Mocha; public record OrderPlaced(Guid OrderId); -// Abstract handler triggers MO0013 +// Abstract handler — triggers MO0013 public abstract class OrderEventHandler : IEventHandler { public abstract ValueTask HandleAsync( @@ -435,7 +435,7 @@ public class RefundSagaState : SagaStateBase public Guid OrderId { get; set; } } -// Constructor requires a parameter triggers MO0014 +// Constructor requires a parameter — triggers MO0014 public class RefundSaga : Saga { private readonly ILogger _logger; diff --git a/website/src/docs/mocha/v1/handler-registration.md b/website/src/docs/mocha/v1/handler-registration.md index 12952344aa7..08183ffde05 100644 --- a/website/src/docs/mocha/v1/handler-registration.md +++ b/website/src/docs/mocha/v1/handler-registration.md @@ -79,7 +79,7 @@ builder.Services .AddRabbitMQ(); ``` -You can mix source-generated and manual registration freely. If both the source generator and manual code register the same handler type, the configurations are composed the source generator sets up the base registration and your manual call layers additional configuration (such as consumer middleware) on top. +You can mix source-generated and manual registration freely. If both the source generator and manual code register the same handler type, the configurations are composed — the source generator sets up the base registration and your manual call layers additional configuration (such as consumer middleware) on top. > **Prefer the source generator.** Manual registration methods use runtime reflection to create handler consumers. The source generator produces direct, reflection-free factory calls. We guarantee backwards compatibility for the source-generated registration path; the manual registration API is stable at the surface level but its internal behavior may evolve. From 60c299257e31c32c9935eb781ab0b5f59a57c003 Mon Sep 17 00:00:00 2001 From: Pascal Senn Date: Thu, 9 Apr 2026 08:27:11 +0200 Subject: [PATCH 14/14] cleanup --- .../Redelivery/ReceiveRedeliveryMiddleware.cs | 8 +-- .../src/docs/mocha/v1/exception-policies.md | 52 ------------------- website/src/docs/mocha/v1/reliability.md | 10 ++-- website/src/docs/mocha/v1/scheduling.md | 51 ------------------ 4 files changed, 9 insertions(+), 112 deletions(-) diff --git a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs index fc978445b3e..78cf4df766a 100644 --- a/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs +++ b/src/Mocha/src/Mocha/Middlewares/Receive/Redelivery/ReceiveRedeliveryMiddleware.cs @@ -36,7 +36,7 @@ public async ValueTask InvokeAsync(IReceiveContext context, ReceiveDelegate next } catch (Exception ex) { - // Request/reply messages must not be redelivered -- the caller is waiting. + // Request/reply messages must not be redelivered - the caller is waiting. if (context.Envelope?.ResponseAddress is not null) { throw; @@ -67,9 +67,9 @@ private async ValueTask DispatchRedeliveryAsync( int delayedRetryCount, DateTimeOffset scheduledTime) { - var envelope = context.Envelope - ?? throw new InvalidOperationException( - "Cannot redeliver because the receive context has no envelope."); + var envelope = + context.Envelope + ?? throw new InvalidOperationException("Cannot redeliver because the receive context has no envelope."); if (envelope.Headers is null) { diff --git a/website/src/docs/mocha/v1/exception-policies.md b/website/src/docs/mocha/v1/exception-policies.md index b40e78c042c..c5ce18f2fa6 100644 --- a/website/src/docs/mocha/v1/exception-policies.md +++ b/website/src/docs/mocha/v1/exception-policies.md @@ -431,55 +431,3 @@ policy.On(ex => // Catch-all for other HTTP errors policy.On().Retry(3); ``` - -# API reference - -## ExceptionPolicyOptions - -| Method | Parameters | Returns | Description | -| --------------------------- | ------------------------- | ------------------------------------- | -------------------------------------------------------------------------- | -| `Default()` | - | `IExceptionPolicyBuilder` | Configure the catch-all default behavior. Equivalent to `On()`. | -| `On()` | - | `IExceptionPolicyBuilder` | Configure behavior for an exception type | -| `On(predicate)` | `Func?` | `IExceptionPolicyBuilder` | Configure behavior for an exception type matching a condition | - -## IExceptionPolicyBuilder<TException> - -| Method | Parameters | Returns | Description | -| --------------------------------- | ------------------------------------- | ------------------------- | ------------------------------------------------------------------------- | -| `Discard()` | - | `void` | Swallow the exception, discard the message | -| `DeadLetter()` | - | `void` | Route to error endpoint, skip retry and redelivery | -| `Retry()` | - | `IAfterRetryBuilder` | Retry with defaults (3 attempts, 200 ms, exponential), disable redelivery | -| `Retry(attempts)` | `int` | `IAfterRetryBuilder` | Retry with custom attempts, disable redelivery | -| `Retry(attempts, delay, backoff)` | `int`, `TimeSpan`, `RetryBackoffType` | `IAfterRetryBuilder` | Retry with full configuration, disable redelivery | -| `Retry(intervals)` | `TimeSpan[]` | `IAfterRetryBuilder` | Retry with explicit intervals, disable redelivery | -| `Redeliver()` | - | `IAfterRedeliveryBuilder` | Redeliver with defaults (5, 15, 30 min), disable retry | -| `Redeliver(attempts, baseDelay)` | `int`, `TimeSpan` | `IAfterRedeliveryBuilder` | Redeliver with custom attempts and delay, disable retry | -| `Redeliver(intervals)` | `TimeSpan[]` | `IAfterRedeliveryBuilder` | Redeliver with explicit intervals, disable retry | - -## IAfterRetryBuilder - -| Method | Parameters | Returns | Description | -| ------------------------------------ | ----------------- | ------------------------- | --------------------------------------------------------------- | -| `ThenRedeliver()` | - | `IAfterRedeliveryBuilder` | Chain redelivery with defaults after retry exhaustion | -| `ThenRedeliver(attempts, baseDelay)` | `int`, `TimeSpan` | `IAfterRedeliveryBuilder` | Chain redelivery with custom settings after retry exhaustion | -| `ThenRedeliver(intervals)` | `TimeSpan[]` | `IAfterRedeliveryBuilder` | Chain redelivery with explicit intervals after retry exhaustion | -| `ThenDeadLetter()` | - | `void` | Route to error endpoint after retry exhaustion | - -## IAfterRedeliveryBuilder - -| Method | Parameters | Returns | Description | -| ------------------ | ---------- | ------- | --------------------------------------------------- | -| `ThenDeadLetter()` | - | `void` | Route to error endpoint after redelivery exhaustion | - -## AddResilience extensions - -| Target | Method | Description | -| ---------------------------- | ----------------------------------------------- | ---------------------------------------------------------------------- | -| `IMessageBusBuilder` | `AddResilience()` | Register default exception policies at the bus level | -| `IMessageBusBuilder` | `AddResilience(Action)` | Configure exception policies at the bus level | -| `IMessageBusHostBuilder` | `AddResilience()` | Register default exception policies at the host level | -| `IMessageBusHostBuilder` | `AddResilience(Action)` | Configure exception policies at the host level | -| `IReceiveMiddlewareProvider` | `AddResilience()` | Register default exception policies at the transport or endpoint level | -| `IReceiveMiddlewareProvider` | `AddResilience(Action)` | Configure exception policies at the transport or endpoint level | -| `IConsumerDescriptor` | `AddResilience()` | Register default exception policies at the consumer level | -| `IConsumerDescriptor` | `AddResilience(Action)` | Configure exception policies at the consumer level | diff --git a/website/src/docs/mocha/v1/reliability.md b/website/src/docs/mocha/v1/reliability.md index cbd8d77d849..830902871e1 100644 --- a/website/src/docs/mocha/v1/reliability.md +++ b/website/src/docs/mocha/v1/reliability.md @@ -283,11 +283,11 @@ Consumer-level policies replace the bus-level policies entirely for that consume ### RetryBackoffType values -| Value | Formula | Example (Delay = 200 ms) | -| ------------- | ----------------------- | ------------------------ | -| `Constant` | `Delay` | 200 ms, 200 ms, 200 ms | -| `Linear` | `Delay * (attempt + 1)` | 400 ms, 600 ms, 800 ms | -| `Exponential` | `Delay * 2^attempt` | 400 ms, 800 ms, 1600 ms | +| Value | Formula | Example (Delay = 200 ms) | +| ------------- | ------------------------- | ------------------------ | +| `Constant` | `Delay` | 200 ms, 200 ms, 200 ms | +| `Linear` | `Delay * attempt` | 200 ms, 400 ms, 600 ms | +| `Exponential` | `Delay * 2^(attempt - 1)` | 200 ms, 400 ms, 800 ms | # Redeliver failed messages diff --git a/website/src/docs/mocha/v1/scheduling.md b/website/src/docs/mocha/v1/scheduling.md index 5217fa570d8..91668c567b1 100644 --- a/website/src/docs/mocha/v1/scheduling.md +++ b/website/src/docs/mocha/v1/scheduling.md @@ -197,57 +197,6 @@ Both `ScheduledPublish` and `ScheduledSend` are available on `ISagaTransitionDes See [Sagas](/docs/mocha/v1/sagas) for the full saga configuration guide. -# API reference - -## Extension methods on `IMessageBus` - -| Method | Parameters | Description | -| ------------------------- | -------------------------------------------------------------------- | ----------------------------------------------------- | -| `SchedulePublishAsync` | `T message, DateTimeOffset scheduledTime, CancellationToken ct` | Publishes a message for delivery at an absolute time. | -| `ScheduleSendAsync` | `object message, DateTimeOffset scheduledTime, CancellationToken ct` | Sends a message for delivery at an absolute time. | - -All methods return `ValueTask` and complete when the message has been handed to the scheduling infrastructure. - -## Scheduling properties on options - -| Struct | Property | Type | Default | Description | -| ---------------- | --------------- | ----------------- | ------- | --------------------------------------------------------- | -| `PublishOptions` | `ScheduledTime` | `DateTimeOffset?` | `null` | Scheduled delivery time. `null` means immediate delivery. | -| `SendOptions` | `ScheduledTime` | `DateTimeOffset?` | `null` | Scheduled delivery time. `null` means immediate delivery. | - -`PublishOptions` and `SendOptions` have additional properties for expiration, headers, and other dispatch behavior. `ScheduledTime` can be combined with any of them. - -## Saga extensions - -| Method | Available on | Parameters | Description | -| ------------------ | ------------------------------------------------------- | ------------------------------------------------ | ------------------------------------------- | -| `ScheduledPublish` | `ISagaTransitionDescriptor`, `ISagaLifeCycleDescriptor` | `TimeSpan delay, Func factory` | Publishes a message with a scheduled delay. | -| `ScheduledSend` | `ISagaTransitionDescriptor`, `ISagaLifeCycleDescriptor` | `TimeSpan delay, Func factory` | Sends a message with a scheduled delay. | - -## `ScheduledMessage` entity columns - -| Column | Type | Description | -| ---------------- | ----------- | --------------------------------------------------------------------- | -| `id` | `uuid` | Primary key. | -| `envelope` | `json` | Serialized message envelope with headers and payload. | -| `scheduled_time` | `timestamp` | UTC time when the message becomes eligible for dispatch. | -| `times_sent` | `integer` | Number of dispatch attempts. | -| `max_attempts` | `integer` | Maximum dispatch attempts before the message is dropped. Default: 10. | -| `last_error` | `jsonb` | Last dispatch error (exception type, message, stack trace). | -| `created_at` | `timestamp` | UTC time when the scheduled message was created. | - -## EF Core model builder - -| Method | Description | -| --------------------------------------------- | ---------------------------------------------------------------------------------------- | -| `modelBuilder.AddPostgresScheduledMessages()` | Applies the `ScheduledMessage` entity configuration with default table and column names. | - -## Scheduling service registration - -| Method | Description | -| ------------------------- | ----------------------------------------------------------------------------------------------------------- | -| `UsePostgresScheduling()` | Registers the Postgres scheduling pipeline: store, dispatcher, background worker, and EF Core interceptors. | - # Troubleshooting **Scheduled messages are not being delivered.**