diff --git a/CHANGELOG.md b/CHANGELOG.md index 60948d37..4920f5a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Middleware / interceptor (#93):** a composable per-item hook — `IItemMiddleware` returning + `MiddlewareResult` (`Continue` to keep/replace an item, `Drop` to remove it) — attached to any + stream with the `WithMiddleware(...)` extensions (single or ordered chain). Lets cross-cutting + concerns (logging, validation, metrics, throttling, dedup) decorate an extractor/transformer output + or loader input without changing the component, and composes inside an `EtlPipeline` via + `Through(s => s.WithMiddleware(...))`. Dependency-free. - **Retry seam (#94):** `ExtractorBase`, `LoaderBase`, and `TransformerBase` gained a `protected virtual WrapWorkerExecution(...)` hook wrapped around every worker invocation. The default implementation is a no-op, so behaviour is unchanged; override it to run the worker through diff --git a/src/Wolfgang.Etl.Abstractions/IItemMiddleware.cs b/src/Wolfgang.Etl.Abstractions/IItemMiddleware.cs new file mode 100644 index 00000000..511f51df --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/IItemMiddleware.cs @@ -0,0 +1,25 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Wolfgang.Etl.Abstractions; + +/// +/// A composable, reusable hook for cross-cutting per-item behaviour (logging, validation, metrics, +/// throttling, deduplication) that can be attached to any stream without modifying the extractor, +/// transformer, or loader that produced it. Attach one or more with +/// ; +/// they run in the order attached, each seeing the item the previous one passed on. +/// +/// The item type flowing through the pipeline. +public interface IItemMiddleware +{ + /// + /// Invoked once per item. Return to keep the item + /// flowing (optionally replacing it), or to remove it from + /// the stream. + /// + /// The item to process. + /// A to observe. + /// The outcome describing whether to keep or drop the item. + ValueTask> OnItemAsync(T item, CancellationToken token); +} diff --git a/src/Wolfgang.Etl.Abstractions/MiddlewareExtensions.cs b/src/Wolfgang.Etl.Abstractions/MiddlewareExtensions.cs new file mode 100644 index 00000000..fb4565b1 --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/MiddlewareExtensions.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Wolfgang.Etl.Abstractions; + +/// +/// Extension methods that attach to an +/// stream, so cross-cutting per-item behaviour composes onto any +/// extractor / transformer output or loader input — and inside an EtlPipeline via +/// Through(stream => stream.WithMiddleware(...)) — without changing the component itself. +/// +public static class MiddlewareExtensions +{ + /// + /// Pipes every item of through . Items the + /// middleware drops () are removed from the stream; otherwise + /// the (possibly replaced) item is yielded. + /// + /// The item type. + /// The stream to decorate. + /// The middleware to run for each item. + /// A to observe. + /// The decorated stream. + /// or is . + public static async IAsyncEnumerable WithMiddleware + ( + this IAsyncEnumerable source, + IItemMiddleware middleware, + [EnumeratorCancellation] CancellationToken token = default + ) + { + if (source is null) + { + throw new ArgumentNullException(nameof(source)); + } + + if (middleware is null) + { + throw new ArgumentNullException(nameof(middleware)); + } + + await foreach (var item in source.WithCancellation(token)) + { + // Stryker disable once Boolean: equivalent — with no synchronization context in play, ConfigureAwait(false) and (true) are indistinguishable. + var result = await middleware.OnItemAsync(item, token).ConfigureAwait(false); + if (!result.Skip) + { + yield return result.Item; + } + } + } + + + + /// + /// Pipes every item of through in order: + /// each middleware sees the item the previous one passed on. If any middleware drops the item + /// (), the remaining middleware is not run and the item is + /// removed from the stream. + /// + /// The item type. + /// The stream to decorate. + /// The middleware chain, applied in enumeration order. + /// A to observe. + /// The decorated stream. + /// or is , or a member of is . + public static async IAsyncEnumerable WithMiddleware + ( + this IAsyncEnumerable source, + IEnumerable> middlewares, + [EnumeratorCancellation] CancellationToken token = default + ) + { + if (source is null) + { + throw new ArgumentNullException(nameof(source)); + } + + if (middlewares is null) + { + throw new ArgumentNullException(nameof(middlewares)); + } + + // Snapshot the chain once so the same ordered set runs for every item. + var chain = new List>(middlewares); + foreach (var middleware in chain) + { + if (middleware is null) + { + // Stryker disable once String: the exception message is diagnostic-only, not a behavioural contract asserted by tests. + throw new ArgumentNullException(nameof(middlewares), "A middleware in the chain is null."); + } + } + + await foreach (var item in source.WithCancellation(token)) + { + var current = item; + var dropped = false; + + foreach (var middleware in chain) + { + // Stryker disable once Boolean: equivalent — with no synchronization context in play, ConfigureAwait(false) and (true) are indistinguishable. + var result = await middleware.OnItemAsync(current, token).ConfigureAwait(false); + if (result.Skip) + { + dropped = true; + break; + } + + current = result.Item; + } + + if (!dropped) + { + yield return current; + } + } + } +} diff --git a/src/Wolfgang.Etl.Abstractions/MiddlewareResult.Factory.cs b/src/Wolfgang.Etl.Abstractions/MiddlewareResult.Factory.cs new file mode 100644 index 00000000..b8f9a1c6 --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/MiddlewareResult.Factory.cs @@ -0,0 +1,25 @@ +namespace Wolfgang.Etl.Abstractions; + +/// +/// Factory methods for creating values from an +/// implementation. +/// +public static class MiddlewareResult +{ + /// + /// Keeps the item in the stream, optionally replacing it with a transformed value. + /// + /// The item type. + /// The item to pass on. + /// A result that keeps flowing. + public static MiddlewareResult Continue(T item) => new(item, skip: false); + + + + /// + /// Drops the current item from the stream. + /// + /// The item type. + /// A result that discards the current item. + public static MiddlewareResult Drop() => new(default!, skip: true); +} diff --git a/src/Wolfgang.Etl.Abstractions/MiddlewareResult.cs b/src/Wolfgang.Etl.Abstractions/MiddlewareResult.cs new file mode 100644 index 00000000..ec86e59b --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/MiddlewareResult.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; + +namespace Wolfgang.Etl.Abstractions; + +/// +/// The outcome of running a single item through an : the +/// (possibly replaced) item to pass on, and whether the item should be dropped from the stream. +/// Create one with to keep an item flowing or +/// to discard it. +/// +/// The item type flowing through the pipeline. +public readonly struct MiddlewareResult : IEquatable> +{ + internal MiddlewareResult(T item, bool skip) + { + Item = item; + Skip = skip; + } + + + + /// + /// The item to pass on to the next middleware (or to the stream). Meaningful only when + /// is . + /// + public T Item { get; } + + + + /// + /// to drop the item from the stream (later middleware is not run and the + /// item is not yielded); to keep it. + /// + public bool Skip { get; } + + + + /// + public bool Equals(MiddlewareResult other) => + Skip == other.Skip && EqualityComparer.Default.Equals(Item, other.Item); + + + + /// + public override bool Equals(object? obj) => obj is MiddlewareResult other && Equals(other); + + + + /// + // Stryker disable once all: equivalent — any change to the hash formula still yields equal hash codes for equal values (the only GetHashCode contract), so no behavioural test can distinguish it. + public override int GetHashCode() => + unchecked(((Skip ? 1 : 0) * 397) ^ (Item is null ? 0 : EqualityComparer.Default.GetHashCode(Item))); + + + + /// Indicates whether two results are equal. + public static bool operator ==(MiddlewareResult left, MiddlewareResult right) => left.Equals(right); + + + + /// Indicates whether two results are not equal. + public static bool operator !=(MiddlewareResult left, MiddlewareResult right) => !left.Equals(right); +} diff --git a/src/Wolfgang.Etl.Abstractions/PublicAPI.Unshipped.txt b/src/Wolfgang.Etl.Abstractions/PublicAPI.Unshipped.txt index fa63bcc8..e53b6f35 100644 --- a/src/Wolfgang.Etl.Abstractions/PublicAPI.Unshipped.txt +++ b/src/Wolfgang.Etl.Abstractions/PublicAPI.Unshipped.txt @@ -3,3 +3,20 @@ virtual Wolfgang.Etl.Abstractions.ExtractorBase.WrapWorkerEx virtual Wolfgang.Etl.Abstractions.LoaderBase.WrapWorkerExecution(System.Func! workerFactory, System.Threading.CancellationToken token) -> System.Threading.Tasks.Task! virtual Wolfgang.Etl.Abstractions.TransformerBase.WrapWorkerExecution(System.Func!>! workerFactory, System.Threading.CancellationToken token) -> System.Collections.Generic.IAsyncEnumerable! Wolfgang.Etl.Abstractions.Report.Report(int currentItemCount, System.DateTimeOffset? startedAt, System.TimeSpan elapsed, int? totalItemCount = null) -> void +Wolfgang.Etl.Abstractions.IItemMiddleware +Wolfgang.Etl.Abstractions.IItemMiddleware.OnItemAsync(T item, System.Threading.CancellationToken token) -> System.Threading.Tasks.ValueTask> +Wolfgang.Etl.Abstractions.MiddlewareResult +static Wolfgang.Etl.Abstractions.MiddlewareResult.Continue(T item) -> Wolfgang.Etl.Abstractions.MiddlewareResult +static Wolfgang.Etl.Abstractions.MiddlewareResult.Drop() -> Wolfgang.Etl.Abstractions.MiddlewareResult +Wolfgang.Etl.Abstractions.MiddlewareResult +Wolfgang.Etl.Abstractions.MiddlewareResult.MiddlewareResult() -> void +Wolfgang.Etl.Abstractions.MiddlewareResult.Item.get -> T +Wolfgang.Etl.Abstractions.MiddlewareResult.Skip.get -> bool +Wolfgang.Etl.Abstractions.MiddlewareResult.Equals(Wolfgang.Etl.Abstractions.MiddlewareResult other) -> bool +override Wolfgang.Etl.Abstractions.MiddlewareResult.Equals(object? obj) -> bool +override Wolfgang.Etl.Abstractions.MiddlewareResult.GetHashCode() -> int +static Wolfgang.Etl.Abstractions.MiddlewareResult.operator ==(Wolfgang.Etl.Abstractions.MiddlewareResult left, Wolfgang.Etl.Abstractions.MiddlewareResult right) -> bool +static Wolfgang.Etl.Abstractions.MiddlewareResult.operator !=(Wolfgang.Etl.Abstractions.MiddlewareResult left, Wolfgang.Etl.Abstractions.MiddlewareResult right) -> bool +Wolfgang.Etl.Abstractions.MiddlewareExtensions +static Wolfgang.Etl.Abstractions.MiddlewareExtensions.WithMiddleware(this System.Collections.Generic.IAsyncEnumerable! source, Wolfgang.Etl.Abstractions.IItemMiddleware! middleware, System.Threading.CancellationToken token = default(System.Threading.CancellationToken)) -> System.Collections.Generic.IAsyncEnumerable! +static Wolfgang.Etl.Abstractions.MiddlewareExtensions.WithMiddleware(this System.Collections.Generic.IAsyncEnumerable! source, System.Collections.Generic.IEnumerable!>! middlewares, System.Threading.CancellationToken token = default(System.Threading.CancellationToken)) -> System.Collections.Generic.IAsyncEnumerable! diff --git a/tests/Wolfgang.Etl.Abstractions.Tests.Unit/MiddlewareTests.cs b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/MiddlewareTests.cs new file mode 100644 index 00000000..92ae8342 --- /dev/null +++ b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/MiddlewareTests.cs @@ -0,0 +1,266 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; +using Xunit; + +namespace Wolfgang.Etl.Abstractions.Tests.Unit; + +/// +/// Covers the #93 middleware mechanism: , +/// , and the WithMiddleware stream decorators — including +/// composition inside an Through stage. +/// +public class MiddlewareTests +{ + // ---------- MiddlewareResult value type ---------- + + [Fact] + public void MiddlewareResult_Continue_keeps_the_item() + { + var result = MiddlewareResult.Continue(42); + + Assert.False(result.Skip); + Assert.Equal(42, result.Item); + } + + + [Fact] + public void MiddlewareResult_Drop_marks_the_item_skipped() + { + var result = MiddlewareResult.Drop(); + + Assert.True(result.Skip); + } + + + [Fact] + public void MiddlewareResult_has_value_equality() + { + var a = MiddlewareResult.Continue(7); + var b = MiddlewareResult.Continue(7); + var different = MiddlewareResult.Continue(8); + var dropped = MiddlewareResult.Drop(); + + Assert.Equal(a, b); + Assert.True(a == b); + Assert.False(a != b); + Assert.Equal(a.GetHashCode(), b.GetHashCode()); + Assert.NotEqual(a, different); + Assert.True(a != different); + Assert.NotEqual(a, dropped); + } + + + // ---------- single middleware ---------- + + [Fact] + public async Task WithMiddleware_transforms_each_item() + { + var items = await Drain(AsyncSource(1, 2, 3).WithMiddleware(new TimesTenMiddleware())); + + Assert.Equal(new[] { 10, 20, 30 }, items); + } + + + [Fact] + public async Task WithMiddleware_drops_items_the_middleware_skips() + { + var items = await Drain(AsyncSource(1, 2, 3, 4).WithMiddleware(new DropOddMiddleware())); + + Assert.Equal(new[] { 2, 4 }, items); + } + + + [Fact] + public async Task WithMiddleware_flows_the_cancellation_token_to_the_middleware() + { + using var cts = new CancellationTokenSource(); + var capturing = new TokenCapturingMiddleware(); + + await Drain(AsyncSource(1).WithMiddleware(capturing), cts.Token); + + Assert.Equal(cts.Token, capturing.LastToken); + } + + + [Fact] + public async Task WithMiddleware_composes_inside_an_EtlPipeline_Through_stage() + { + var stream = EtlPipeline + .Create() + .From(AsyncSource(1, 2, 3, 4)) + .Through(s => s.WithMiddleware(new DropOddMiddleware())) + .Through(s => s.WithMiddleware(new TimesTenMiddleware())) + .AsAsyncEnumerable(); + + var items = await Drain(stream); + + Assert.Equal(new[] { 20, 40 }, items); + } + + + [Fact] + public async Task WithMiddleware_when_source_is_null_throws_ArgumentNullException() + { + await Assert.ThrowsAsync( + () => Drain(((IAsyncEnumerable)null!).WithMiddleware(new TimesTenMiddleware()))); + } + + + [Fact] + public async Task WithMiddleware_when_middleware_is_null_throws_ArgumentNullException() + { + await Assert.ThrowsAsync( + () => Drain(AsyncSource(1).WithMiddleware((IItemMiddleware)null!))); + } + + + // ---------- middleware chain ---------- + + [Fact] + public async Task WithMiddleware_chain_runs_in_registration_order_each_seeing_the_previous_output() + { + var log = new List(); + var chain = new IItemMiddleware[] + { + new RecordingMiddleware(log, "A", add: 10), + new RecordingMiddleware(log, "B", add: 100), + }; + + var items = await Drain(AsyncSource(1).WithMiddleware(chain)); + + Assert.Equal(new[] { 111 }, items); // 1 -> +10 -> +100 + Assert.Equal(new[] { "A:1", "B:11" }, log); // B saw A's output + } + + + [Fact] + public async Task WithMiddleware_chain_stops_at_the_first_drop() + { + var log = new List(); + var chain = new IItemMiddleware[] + { + new DropOddMiddleware(), + new RecordingMiddleware(log, "R", add: 0), + }; + + var items = await Drain(AsyncSource(1, 2, 3).WithMiddleware(chain)); + + Assert.Equal(new[] { 2 }, items); // odds dropped before reaching R + Assert.Equal(new[] { "R:2" }, log); // R only ran for the surviving even item + } + + + [Fact] + public async Task WithMiddleware_empty_chain_passes_items_through() + { + var items = await Drain(AsyncSource(1, 2, 3).WithMiddleware(Array.Empty>())); + + Assert.Equal(new[] { 1, 2, 3 }, items); + } + + + [Fact] + public async Task WithMiddleware_chain_when_source_is_null_throws_ArgumentNullException() + { + await Assert.ThrowsAsync( + () => Drain(((IAsyncEnumerable)null!).WithMiddleware(new IItemMiddleware[] { new TimesTenMiddleware() }))); + } + + + [Fact] + public async Task WithMiddleware_chain_when_middlewares_is_null_throws_ArgumentNullException() + { + var ex = await Assert.ThrowsAsync( + () => Drain(AsyncSource(1).WithMiddleware((IEnumerable>)null!))); + + // The explicit guard names "middlewares"; without it the fallback List ctor would name "collection". + Assert.Equal("middlewares", ex.ParamName); + } + + + [Fact] + public async Task WithMiddleware_chain_when_a_member_is_null_throws_ArgumentNullException() + { + var chain = new IItemMiddleware[] { new TimesTenMiddleware(), null! }; + + await Assert.ThrowsAsync( + () => Drain(AsyncSource(1).WithMiddleware(chain))); + } + + + // ---------- helpers ---------- + + private static async IAsyncEnumerable AsyncSource(params int[] items) + { + foreach (var item in items) + { + await Task.Yield(); + yield return item; + } + } + + + private static async Task> Drain(IAsyncEnumerable source, CancellationToken token = default) + { + var result = new List(); + await foreach (var item in source.WithCancellation(token).ConfigureAwait(false)) + { + result.Add(item); + } + + return result; + } + + + // ---------- doubles ---------- + + private sealed class TimesTenMiddleware : IItemMiddleware + { + public ValueTask> OnItemAsync(int item, CancellationToken token) => + new(MiddlewareResult.Continue(item * 10)); + } + + + private sealed class DropOddMiddleware : IItemMiddleware + { + public ValueTask> OnItemAsync(int item, CancellationToken token) => + new(item % 2 == 0 ? MiddlewareResult.Continue(item) : MiddlewareResult.Drop()); + } + + + private sealed class TokenCapturingMiddleware : IItemMiddleware + { + public CancellationToken LastToken { get; private set; } + + public ValueTask> OnItemAsync(int item, CancellationToken token) + { + LastToken = token; + return new(MiddlewareResult.Continue(item)); + } + } + + + private sealed class RecordingMiddleware : IItemMiddleware + { + private readonly List _log; + private readonly string _name; + private readonly int _add; + + public RecordingMiddleware(List log, string name, int add) + { + _log = log; + _name = name; + _add = add; + } + + public ValueTask> OnItemAsync(int item, CancellationToken token) + { + _log.Add($"{_name}:{item}"); + return new(MiddlewareResult.Continue(item + _add)); + } + } +}