diff --git a/ETL.png b/ETL.png new file mode 100644 index 0000000..2970f9e Binary files /dev/null and b/ETL.png differ diff --git a/README.md b/README.md index a439ae8..3c9bfcd 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,31 @@ -# ETL-Abstractions -Interfaces and base classes for creating ETLs +# Wolfgang.Etl.Abstractions + +This package contains interfaces and base classes for building ETLs using a specific design pattern + +The ETL design pattern is a common approach in data processing that involves three main stages: +- **Extract**: Retrieving data from various sources. +- **Transform**: Processing and transforming the extracted data into a desired format. +- **Load**: Storing the transformed data into a target system. + +The abstractions in this package provide a way to define and implement these stages +in a flexible and reusable manner. Each stage can be implemented with or without +support for cancellation and progress reporting, allowing for greater control +over the ETL process. + +To build an ETL using this package, you would typically need to create 5 classes: +- One class for each of the three stages: Extract, Transform, and Load. +- One class representing the source data. +- One class representing the target data. +- One class that acts as the ETL orchestrator, coordinating the execution of the three stages. + +The design uses lazy loading and lazy evaluation to ensure that data is processed only when needed. +This allows for efficient memory usage and can handle large datasets without loading everything into memory at once. + +The process uses a pull method rather than a push method to move data through the pipeline. +The process starts when the ETL orchestrator calls the `LoadAsyc` method of the `Loader` class. +The loader will start enumerating through the list of items passed into its `LoadAsync` method. +This will intern trigger the `TransformAsync` method of the `Transformer` class, which will process each item +and yield the transformed results. The process of transformation will also trigger the `ExtractAsync` method of the `Extractor` class, +which will retrieve the necessary data from the source. + +For more information check out the [documentation](https://github.com/Chris-Wolfgang/ETL-Abstractions/wiki) \ No newline at end of file diff --git a/Wolfgang.Etl.Abstractions.sln b/Wolfgang.Etl.Abstractions.sln new file mode 100644 index 0000000..b9d04c5 --- /dev/null +++ b/Wolfgang.Etl.Abstractions.sln @@ -0,0 +1,167 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wolfgang.Etl.Abstractions", "src\Wolfgang.Etl.Abstractions\Wolfgang.Etl.Abstractions.csproj", "{C4987BAD-4513-955F-B3C1-7563D0C1A7A3}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{02EA681E-C7D8-13C7-8484-4AC65E1B71E8}" + ProjectSection(SolutionItems) = preProject + README.md = README.md + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{8220BC33-6632-4D4C-9A50-B7978141A4E3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wolfgang.Etl.Abstractions.Tests.Unit", "tests\Wolfgang.Etl.Abstractions.Tests.Unit\Wolfgang.Etl.Abstractions.Tests.Unit.csproj", "{B8558C7F-934B-3DC1-AAED-D668CF964C8E}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "examples", "examples", "{207971F1-432C-4AB4-9DC4-16A9E5ACF812}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example1-BasicETL", "examples\Net8.0\Example1-BasicETL\Example1-BasicETL.csproj", "{F1BA1AF5-9A71-421B-963C-E277610FBD40}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Net80", "Net80", "{3C48157B-5E90-489E-9444-E01F51D59F86}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Net48", "Net48", "{336D72A1-8E5E-49DE-83D9-DF6BE458BA24}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example1-BasicETL", "examples\Net4.8\Example1-BasicETL\Example1-BasicETL.csproj", "{B715D0C5-3F1A-485C-92D1-FFB87A801AC3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example2-WithCancellationToken", "examples\Net8.0\Example2-WithCancellationToken\Example2-WithCancellationToken.csproj", "{F4A2F47C-9687-0ABE-FEAE-D07873E0133A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example3-WithGracefulCancellation", "examples\Net8.0\Example3-WithGracefulCancellation\Example3-WithGracefulCancellation.csproj", "{733AD8E6-D170-789A-6F61-13C041011037}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example2-WithCancellationToken", "examples\Net4.8\Example2-WithCancellationToken\Example2-WithCancellationToken.csproj", "{2DC16382-F59F-4024-B560-D400E09EF91F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example3-WithGracefulCancellation", "examples\Net4.8\Example3-WithGracefulCancellation\Example3-WithGracefulCancellation.csproj", "{5A7CFA9D-DCE7-43FD-AF89-7418C36AAE14}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example4a-WithExtractorProgress", "examples\Net8.0\Example4a-WithExtractorProgress\Example4a-WithExtractorProgress.csproj", "{483AE567-071E-4797-B13B-84B142D4ED44}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example4b-WithTransformerProgress", "examples\Net8.0\Example4b-WithTransformerProgress\Example4b-WithTransformerProgress.csproj", "{6C19204D-7CC8-48D2-A475-49CD98931105}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example4c-WithLoaderProgress", "examples\Net8.0\Example4c-WithLoaderProgress\Example4c-WithLoaderProgress.csproj", "{E858F22E-01D7-4BE9-B8DE-0E4AFBA5B33A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example4a-WithExtractorProgress", "examples\Net4.8\Example4a-WithExtractorProgress\Example4a-WithExtractorProgress.csproj", "{8068B622-46A9-4ABA-BDA3-6AB259D1682C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example4c-WithLoaderProgress", "examples\Net4.8\Example4c-WithLoaderProgress\Example4c-WithLoaderProgress.csproj", "{6DA63E99-692F-461C-983A-270E5CBE8D45}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example4b-WithTransformerProgress", "examples\Net4.8\Example4b-WithTransformerProgress\Example4b-WithTransformerProgress.csproj", "{6BD7828E-55B4-4213-8DF9-7BFBA891B8EE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example5a-ExtractorWithProgressAndCancellation", "examples\Net8.0\Example5a-ExtractorWithProgressAndCancellation\Example5a-ExtractorWithProgressAndCancellation.csproj", "{2D14E222-4E44-40AB-82EF-1E8ABCED0476}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example5a-ExtractorWithProgressAndCancellation", "examples\Net4.8\Example5a-ExtractorWithProgressAndCancellation\Example5a-ExtractorWithProgressAndCancellation.csproj", "{861EA36D-970E-4CFE-9E72-D3D12F0BBB60}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example6-ReducingDuplicateCode", "examples\Net8.0\Example6-ReducingDuplicateCode\Example6-ReducingDuplicateCode.csproj", "{80E49C71-1073-4208-B48F-E0F399946B3B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example6-ReducingDuplicateCode", "examples\Net4.8\Example6-ReducingDuplicateCode\Example6-ReducingDuplicateCode.csproj", "{85A15A15-D528-4542-A546-63BE0EAED986}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Docs", "Docs", "{8EC462FD-D22E-90A8-E5CE-7E832BA40C5D}" + ProjectSection(SolutionItems) = preProject + docs\readme.md = docs\readme.md + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {C4987BAD-4513-955F-B3C1-7563D0C1A7A3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C4987BAD-4513-955F-B3C1-7563D0C1A7A3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C4987BAD-4513-955F-B3C1-7563D0C1A7A3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C4987BAD-4513-955F-B3C1-7563D0C1A7A3}.Release|Any CPU.Build.0 = Release|Any CPU + {B8558C7F-934B-3DC1-AAED-D668CF964C8E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B8558C7F-934B-3DC1-AAED-D668CF964C8E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B8558C7F-934B-3DC1-AAED-D668CF964C8E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B8558C7F-934B-3DC1-AAED-D668CF964C8E}.Release|Any CPU.Build.0 = Release|Any CPU + {F1BA1AF5-9A71-421B-963C-E277610FBD40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F1BA1AF5-9A71-421B-963C-E277610FBD40}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F1BA1AF5-9A71-421B-963C-E277610FBD40}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F1BA1AF5-9A71-421B-963C-E277610FBD40}.Release|Any CPU.Build.0 = Release|Any CPU + {B715D0C5-3F1A-485C-92D1-FFB87A801AC3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B715D0C5-3F1A-485C-92D1-FFB87A801AC3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B715D0C5-3F1A-485C-92D1-FFB87A801AC3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B715D0C5-3F1A-485C-92D1-FFB87A801AC3}.Release|Any CPU.Build.0 = Release|Any CPU + {F4A2F47C-9687-0ABE-FEAE-D07873E0133A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F4A2F47C-9687-0ABE-FEAE-D07873E0133A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F4A2F47C-9687-0ABE-FEAE-D07873E0133A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F4A2F47C-9687-0ABE-FEAE-D07873E0133A}.Release|Any CPU.Build.0 = Release|Any CPU + {733AD8E6-D170-789A-6F61-13C041011037}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {733AD8E6-D170-789A-6F61-13C041011037}.Debug|Any CPU.Build.0 = Debug|Any CPU + {733AD8E6-D170-789A-6F61-13C041011037}.Release|Any CPU.ActiveCfg = Release|Any CPU + {733AD8E6-D170-789A-6F61-13C041011037}.Release|Any CPU.Build.0 = Release|Any CPU + {2DC16382-F59F-4024-B560-D400E09EF91F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2DC16382-F59F-4024-B560-D400E09EF91F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2DC16382-F59F-4024-B560-D400E09EF91F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2DC16382-F59F-4024-B560-D400E09EF91F}.Release|Any CPU.Build.0 = Release|Any CPU + {5A7CFA9D-DCE7-43FD-AF89-7418C36AAE14}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5A7CFA9D-DCE7-43FD-AF89-7418C36AAE14}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5A7CFA9D-DCE7-43FD-AF89-7418C36AAE14}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5A7CFA9D-DCE7-43FD-AF89-7418C36AAE14}.Release|Any CPU.Build.0 = Release|Any CPU + {483AE567-071E-4797-B13B-84B142D4ED44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {483AE567-071E-4797-B13B-84B142D4ED44}.Debug|Any CPU.Build.0 = Debug|Any CPU + {483AE567-071E-4797-B13B-84B142D4ED44}.Release|Any CPU.ActiveCfg = Release|Any CPU + {483AE567-071E-4797-B13B-84B142D4ED44}.Release|Any CPU.Build.0 = Release|Any CPU + {6C19204D-7CC8-48D2-A475-49CD98931105}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6C19204D-7CC8-48D2-A475-49CD98931105}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6C19204D-7CC8-48D2-A475-49CD98931105}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6C19204D-7CC8-48D2-A475-49CD98931105}.Release|Any CPU.Build.0 = Release|Any CPU + {E858F22E-01D7-4BE9-B8DE-0E4AFBA5B33A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E858F22E-01D7-4BE9-B8DE-0E4AFBA5B33A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E858F22E-01D7-4BE9-B8DE-0E4AFBA5B33A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E858F22E-01D7-4BE9-B8DE-0E4AFBA5B33A}.Release|Any CPU.Build.0 = Release|Any CPU + {8068B622-46A9-4ABA-BDA3-6AB259D1682C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8068B622-46A9-4ABA-BDA3-6AB259D1682C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8068B622-46A9-4ABA-BDA3-6AB259D1682C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8068B622-46A9-4ABA-BDA3-6AB259D1682C}.Release|Any CPU.Build.0 = Release|Any CPU + {6DA63E99-692F-461C-983A-270E5CBE8D45}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6DA63E99-692F-461C-983A-270E5CBE8D45}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6DA63E99-692F-461C-983A-270E5CBE8D45}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6DA63E99-692F-461C-983A-270E5CBE8D45}.Release|Any CPU.Build.0 = Release|Any CPU + {6BD7828E-55B4-4213-8DF9-7BFBA891B8EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6BD7828E-55B4-4213-8DF9-7BFBA891B8EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6BD7828E-55B4-4213-8DF9-7BFBA891B8EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6BD7828E-55B4-4213-8DF9-7BFBA891B8EE}.Release|Any CPU.Build.0 = Release|Any CPU + {2D14E222-4E44-40AB-82EF-1E8ABCED0476}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2D14E222-4E44-40AB-82EF-1E8ABCED0476}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2D14E222-4E44-40AB-82EF-1E8ABCED0476}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2D14E222-4E44-40AB-82EF-1E8ABCED0476}.Release|Any CPU.Build.0 = Release|Any CPU + {861EA36D-970E-4CFE-9E72-D3D12F0BBB60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {861EA36D-970E-4CFE-9E72-D3D12F0BBB60}.Debug|Any CPU.Build.0 = Debug|Any CPU + {861EA36D-970E-4CFE-9E72-D3D12F0BBB60}.Release|Any CPU.ActiveCfg = Release|Any CPU + {861EA36D-970E-4CFE-9E72-D3D12F0BBB60}.Release|Any CPU.Build.0 = Release|Any CPU + {80E49C71-1073-4208-B48F-E0F399946B3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {80E49C71-1073-4208-B48F-E0F399946B3B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {80E49C71-1073-4208-B48F-E0F399946B3B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {80E49C71-1073-4208-B48F-E0F399946B3B}.Release|Any CPU.Build.0 = Release|Any CPU + {85A15A15-D528-4542-A546-63BE0EAED986}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {85A15A15-D528-4542-A546-63BE0EAED986}.Debug|Any CPU.Build.0 = Debug|Any CPU + {85A15A15-D528-4542-A546-63BE0EAED986}.Release|Any CPU.ActiveCfg = Release|Any CPU + {85A15A15-D528-4542-A546-63BE0EAED986}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {C4987BAD-4513-955F-B3C1-7563D0C1A7A3} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} + {B8558C7F-934B-3DC1-AAED-D668CF964C8E} = {8220BC33-6632-4D4C-9A50-B7978141A4E3} + {F1BA1AF5-9A71-421B-963C-E277610FBD40} = {3C48157B-5E90-489E-9444-E01F51D59F86} + {3C48157B-5E90-489E-9444-E01F51D59F86} = {207971F1-432C-4AB4-9DC4-16A9E5ACF812} + {336D72A1-8E5E-49DE-83D9-DF6BE458BA24} = {207971F1-432C-4AB4-9DC4-16A9E5ACF812} + {B715D0C5-3F1A-485C-92D1-FFB87A801AC3} = {336D72A1-8E5E-49DE-83D9-DF6BE458BA24} + {F4A2F47C-9687-0ABE-FEAE-D07873E0133A} = {3C48157B-5E90-489E-9444-E01F51D59F86} + {733AD8E6-D170-789A-6F61-13C041011037} = {3C48157B-5E90-489E-9444-E01F51D59F86} + {2DC16382-F59F-4024-B560-D400E09EF91F} = {336D72A1-8E5E-49DE-83D9-DF6BE458BA24} + {5A7CFA9D-DCE7-43FD-AF89-7418C36AAE14} = {336D72A1-8E5E-49DE-83D9-DF6BE458BA24} + {483AE567-071E-4797-B13B-84B142D4ED44} = {3C48157B-5E90-489E-9444-E01F51D59F86} + {6C19204D-7CC8-48D2-A475-49CD98931105} = {3C48157B-5E90-489E-9444-E01F51D59F86} + {E858F22E-01D7-4BE9-B8DE-0E4AFBA5B33A} = {3C48157B-5E90-489E-9444-E01F51D59F86} + {8068B622-46A9-4ABA-BDA3-6AB259D1682C} = {336D72A1-8E5E-49DE-83D9-DF6BE458BA24} + {6DA63E99-692F-461C-983A-270E5CBE8D45} = {336D72A1-8E5E-49DE-83D9-DF6BE458BA24} + {6BD7828E-55B4-4213-8DF9-7BFBA891B8EE} = {336D72A1-8E5E-49DE-83D9-DF6BE458BA24} + {2D14E222-4E44-40AB-82EF-1E8ABCED0476} = {3C48157B-5E90-489E-9444-E01F51D59F86} + {861EA36D-970E-4CFE-9E72-D3D12F0BBB60} = {336D72A1-8E5E-49DE-83D9-DF6BE458BA24} + {80E49C71-1073-4208-B48F-E0F399946B3B} = {3C48157B-5E90-489E-9444-E01F51D59F86} + {85A15A15-D528-4542-A546-63BE0EAED986} = {336D72A1-8E5E-49DE-83D9-DF6BE458BA24} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {F673635D-58CE-48A5-9AE4-31F4484BED9E} + EndGlobalSection +EndGlobal diff --git a/examples/Net4.8/Example1-BasicETL/App.config b/examples/Net4.8/Example1-BasicETL/App.config new file mode 100644 index 0000000..193aecc --- /dev/null +++ b/examples/Net4.8/Example1-BasicETL/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example1-BasicETL/ETL/ConsoleLoader.cs b/examples/Net4.8/Example1-BasicETL/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..3d66ba1 --- /dev/null +++ b/examples/Net4.8/Example1-BasicETL/ETL/ConsoleLoader.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example1_BasicETL.ETL +{ + internal class ConsoleLoader : ILoadAsync + { + public async Task LoadAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example1-BasicETL/ETL/FibonacciExtractor.cs b/examples/Net4.8/Example1-BasicETL/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..5f5726e --- /dev/null +++ b/examples/Net4.8/Example1-BasicETL/ETL/FibonacciExtractor.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example1_BasicETL.ETL +{ + internal class FibonacciExtractor : IExtractAsync + { + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Yield(); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example1-BasicETL/ETL/IntToStringTransformer.cs b/examples/Net4.8/Example1-BasicETL/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..57df63a --- /dev/null +++ b/examples/Net4.8/Example1-BasicETL/ETL/IntToStringTransformer.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example1_BasicETL.ETL +{ + internal class IntToStringTransformer : ITransformAsync + { + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example1-BasicETL/Example1-BasicETL.csproj b/examples/Net4.8/Example1-BasicETL/Example1-BasicETL.csproj new file mode 100644 index 0000000..9524fba --- /dev/null +++ b/examples/Net4.8/Example1-BasicETL/Example1-BasicETL.csproj @@ -0,0 +1,67 @@ + + + + + Debug + AnyCPU + {B715D0C5-3F1A-485C-92D1-FFB87A801AC3} + Exe + Example1_BasicETL + Example1-BasicETL + v4.8 + 512 + true + true + 8 + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.6\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + + + + ..\..\..\packages\System.Runtime.CompilerServices.Unsafe.4.5.3\lib\net461\System.Runtime.CompilerServices.Unsafe.dll + + + ..\..\..\packages\System.Threading.Tasks.Extensions.4.5.4\lib\net461\System.Threading.Tasks.Extensions.dll + + + + + + + + + + + + + + + + {c4987bad-4513-955f-b3c1-7563d0c1a7a3} + Wolfgang.Etl.Abstractions + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example1-BasicETL/Program.cs b/examples/Net4.8/Example1-BasicETL/Program.cs new file mode 100644 index 0000000..73b83c5 --- /dev/null +++ b/examples/Net4.8/Example1-BasicETL/Program.cs @@ -0,0 +1,42 @@ +using System; +using System.Threading.Tasks; +using Example1_BasicETL.ETL; + +namespace Example1_BasicETL +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + var sourceItems = extractor.ExtractAsync(); + var transformedItems = transformer.TransformAsync(sourceItems); + await loader.LoadAsync(transformedItems); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + } +} diff --git a/examples/Net4.8/Example1-BasicETL/Properties/AssemblyInfo.cs b/examples/Net4.8/Example1-BasicETL/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..ba94ce4 --- /dev/null +++ b/examples/Net4.8/Example1-BasicETL/Properties/AssemblyInfo.cs @@ -0,0 +1,32 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Example1-BasicETL")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Example1-BasicETL")] +[assembly: AssemblyCopyright("Copyright © 2025")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("b715d0c5-3f1a-485c-92d1-ffb87a801ac3")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/examples/Net4.8/Example1-BasicETL/packages.config b/examples/Net4.8/Example1-BasicETL/packages.config new file mode 100644 index 0000000..7b7c193 --- /dev/null +++ b/examples/Net4.8/Example1-BasicETL/packages.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example2-WithCancellationToken/App.config b/examples/Net4.8/Example2-WithCancellationToken/App.config new file mode 100644 index 0000000..193aecc --- /dev/null +++ b/examples/Net4.8/Example2-WithCancellationToken/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example2-WithCancellationToken/ETL/ConsoleLoader.cs b/examples/Net4.8/Example2-WithCancellationToken/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..83c62c2 --- /dev/null +++ b/examples/Net4.8/Example2-WithCancellationToken/ETL/ConsoleLoader.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example2_WithCancellationToken.ETL +{ + internal class ConsoleLoader : ILoadWithCancellationAsync + { + public async Task LoadAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items.WithCancellation(token)) + { + // Throw exception if cancellation is requested + // See Example3-ExtractorWithGracefulCancellation for a more graceful cancellation approach + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example2-WithCancellationToken/ETL/FibonacciExtractor.cs b/examples/Net4.8/Example2-WithCancellationToken/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..1150a59 --- /dev/null +++ b/examples/Net4.8/Example2-WithCancellationToken/ETL/FibonacciExtractor.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example2_WithCancellationToken.ETL +{ + internal class FibonacciExtractor : IExtractWithCancellationAsync + { + + + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync([EnumeratorCancellation] CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + // Throw exception if cancellation is requested + // See Example3-ExtractorWithGracefulCancellation for a more graceful cancellation approach + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example2-WithCancellationToken/ETL/IntToStringTransformer.cs b/examples/Net4.8/Example2-WithCancellationToken/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..ffac307 --- /dev/null +++ b/examples/Net4.8/Example2-WithCancellationToken/ETL/IntToStringTransformer.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example2_WithCancellationToken.ETL +{ + internal class IntToStringTransformer : ITransformWithCancellationAsync + { + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items, [EnumeratorCancellation] CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items.WithCancellation(token)) + { + // Throw exception if cancellation is requested + // See Example3-ExtractorWithGracefulCancellation for a more graceful cancellation approach + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example2-WithCancellationToken/Example2-WithCancellationToken.csproj b/examples/Net4.8/Example2-WithCancellationToken/Example2-WithCancellationToken.csproj new file mode 100644 index 0000000..06d7c1e --- /dev/null +++ b/examples/Net4.8/Example2-WithCancellationToken/Example2-WithCancellationToken.csproj @@ -0,0 +1,70 @@ + + + + + Debug + AnyCPU + {2DC16382-F59F-4024-B560-D400E09EF91F} + Exe + Example2_WithCancellationToken + Example2-WithCancellationToken + v4.8 + 512 + true + true + 8 + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + + + + ..\..\..\packages\System.Runtime.CompilerServices.Unsafe.6.1.2\lib\net462\System.Runtime.CompilerServices.Unsafe.dll + + + ..\..\..\packages\System.Threading.Tasks.Extensions.4.6.3\lib\net462\System.Threading.Tasks.Extensions.dll + + + + + + + + + + + + + + + + + + + {c4987bad-4513-955f-b3c1-7563d0c1a7a3} + Wolfgang.Etl.Abstractions + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example2-WithCancellationToken/Program.cs b/examples/Net4.8/Example2-WithCancellationToken/Program.cs new file mode 100644 index 0000000..ce0ce2c --- /dev/null +++ b/examples/Net4.8/Example2-WithCancellationToken/Program.cs @@ -0,0 +1,55 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Example2_WithCancellationToken.ETL; + +namespace Example2_WithCancellationToken +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + // Set a cancellation token to cancel the extraction after 1 second + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var token = cts.Token; + + try + { + var sourceItems = extractor.ExtractAsync(token); + var transformedItems = transformer.TransformAsync(sourceItems, token); + await loader.LoadAsync(transformedItems, token); + } + catch (Exception e) + { + Console.WriteLine(e); + } + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + } +} diff --git a/examples/Net4.8/Example2-WithCancellationToken/Properties/AssemblyInfo.cs b/examples/Net4.8/Example2-WithCancellationToken/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..f095465 --- /dev/null +++ b/examples/Net4.8/Example2-WithCancellationToken/Properties/AssemblyInfo.cs @@ -0,0 +1,32 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Example2-WithCancellationToken")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Example2-WithCancellationToken")] +[assembly: AssemblyCopyright("Copyright © 2025")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("2dc16382-f59f-4024-b560-d400e09ef91f")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/examples/Net4.8/Example2-WithCancellationToken/packages.config b/examples/Net4.8/Example2-WithCancellationToken/packages.config new file mode 100644 index 0000000..27ab3ab --- /dev/null +++ b/examples/Net4.8/Example2-WithCancellationToken/packages.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example3-WithGracefulCancellation/App.config b/examples/Net4.8/Example3-WithGracefulCancellation/App.config new file mode 100644 index 0000000..193aecc --- /dev/null +++ b/examples/Net4.8/Example3-WithGracefulCancellation/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example3-WithGracefulCancellation/ETL/ConsoleLoader.cs b/examples/Net4.8/Example3-WithGracefulCancellation/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..6f594d8 --- /dev/null +++ b/examples/Net4.8/Example3-WithGracefulCancellation/ETL/ConsoleLoader.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example3_WithGracefulCancellation.ETL +{ + internal class ConsoleLoader : ILoadWithCancellationAsync + { + public async Task LoadAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Loading cancelled.{ConsoleColors.Reset}"); + return; // Exit gracefully if cancellation is requested + } + + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example3-WithGracefulCancellation/ETL/FibonacciExtractor.cs b/examples/Net4.8/Example3-WithGracefulCancellation/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..f62ac49 --- /dev/null +++ b/examples/Net4.8/Example3-WithGracefulCancellation/ETL/FibonacciExtractor.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example3_WithGracefulCancellation.ETL +{ + internal class FibonacciExtractor : IExtractWithCancellationAsync + { + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Yield(); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync([EnumeratorCancellation] CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Extraction cancelled{ConsoleColors.Reset}."); + yield break; // Exit the method if cancellation is requested + } + + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example3-WithGracefulCancellation/ETL/IntToStringTransformer.cs b/examples/Net4.8/Example3-WithGracefulCancellation/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..e034118 --- /dev/null +++ b/examples/Net4.8/Example3-WithGracefulCancellation/ETL/IntToStringTransformer.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example3_WithGracefulCancellation.ETL +{ + internal class IntToStringTransformer : ITransformWithCancellationAsync + { + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items, [EnumeratorCancellation] CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items.WithCancellation(token)) + { + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Loading cancelled.{ConsoleColors.Reset}"); + yield break; // Exit gracefully if cancellation is requested + } + + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example3-WithGracefulCancellation/Example3-WithGracefulCancellation.csproj b/examples/Net4.8/Example3-WithGracefulCancellation/Example3-WithGracefulCancellation.csproj new file mode 100644 index 0000000..9d54856 --- /dev/null +++ b/examples/Net4.8/Example3-WithGracefulCancellation/Example3-WithGracefulCancellation.csproj @@ -0,0 +1,70 @@ + + + + + Debug + AnyCPU + {5A7CFA9D-DCE7-43FD-AF89-7418C36AAE14} + Exe + Example3_WithGracefulCancellation + Example3-WithGracefulCancellation + v4.8 + 512 + true + true + 8 + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + + + + ..\..\..\packages\System.Runtime.CompilerServices.Unsafe.6.1.2\lib\net462\System.Runtime.CompilerServices.Unsafe.dll + + + ..\..\..\packages\System.Threading.Tasks.Extensions.4.6.3\lib\net462\System.Threading.Tasks.Extensions.dll + + + + + + + + + + + + + + + + + + + {c4987bad-4513-955f-b3c1-7563d0c1a7a3} + Wolfgang.Etl.Abstractions + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example3-WithGracefulCancellation/Program.cs b/examples/Net4.8/Example3-WithGracefulCancellation/Program.cs new file mode 100644 index 0000000..09f0d5a --- /dev/null +++ b/examples/Net4.8/Example3-WithGracefulCancellation/Program.cs @@ -0,0 +1,48 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Example3_WithGracefulCancellation.ETL; + +namespace Example3_WithGracefulCancellation +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + // Set a cancellation token to cancel the extraction after 1 second + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var token = cts.Token; + + var sourceItems = extractor.ExtractAsync(token); + var transformedItems = transformer.TransformAsync(sourceItems,token); + await loader.LoadAsync(transformedItems, token); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + } +} diff --git a/examples/Net4.8/Example3-WithGracefulCancellation/Properties/AssemblyInfo.cs b/examples/Net4.8/Example3-WithGracefulCancellation/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..3566172 --- /dev/null +++ b/examples/Net4.8/Example3-WithGracefulCancellation/Properties/AssemblyInfo.cs @@ -0,0 +1,32 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Example3-WithGracefulCancellation")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Example3-WithGracefulCancellation")] +[assembly: AssemblyCopyright("Copyright © 2025")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("5a7cfa9d-dce7-43fd-af89-7418c36aae14")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/examples/Net4.8/Example3-WithGracefulCancellation/packages.config b/examples/Net4.8/Example3-WithGracefulCancellation/packages.config new file mode 100644 index 0000000..27ab3ab --- /dev/null +++ b/examples/Net4.8/Example3-WithGracefulCancellation/packages.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example4a-WithExtractorProgress/App.config b/examples/Net4.8/Example4a-WithExtractorProgress/App.config new file mode 100644 index 0000000..193aecc --- /dev/null +++ b/examples/Net4.8/Example4a-WithExtractorProgress/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example4a-WithExtractorProgress/ETL/ConsoleLoader.cs b/examples/Net4.8/Example4a-WithExtractorProgress/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..4b56cc6 --- /dev/null +++ b/examples/Net4.8/Example4a-WithExtractorProgress/ETL/ConsoleLoader.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example4a_WithExtractorProgress.ETL +{ + internal class ConsoleLoader : ILoadAsync, ILoadWithProgressAsync + { + + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async Task LoadAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example4a-WithExtractorProgress/ETL/FibonacciExtractor.cs b/examples/Net4.8/Example4a-WithExtractorProgress/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..7588c65 --- /dev/null +++ b/examples/Net4.8/Example4a-WithExtractorProgress/ETL/FibonacciExtractor.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example4a_WithExtractorProgress.ETL +{ + internal class FibonacciExtractor : IExtractAsync, IExtractWithProgressAsync + { + private int _progressInterval = 1_000; + + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync(IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + count = Interlocked.Increment(ref count); + + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example4a-WithExtractorProgress/ETL/IntToStringTransformer.cs b/examples/Net4.8/Example4a-WithExtractorProgress/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..6821f92 --- /dev/null +++ b/examples/Net4.8/Example4a-WithExtractorProgress/ETL/IntToStringTransformer.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example4a_WithExtractorProgress.ETL +{ + internal class IntToStringTransformer + : ITransformAsync, ITransformWithProgressAsync + { + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items, IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example4a-WithExtractorProgress/Example4a-WithExtractorProgress.csproj b/examples/Net4.8/Example4a-WithExtractorProgress/Example4a-WithExtractorProgress.csproj new file mode 100644 index 0000000..efd9a36 --- /dev/null +++ b/examples/Net4.8/Example4a-WithExtractorProgress/Example4a-WithExtractorProgress.csproj @@ -0,0 +1,73 @@ + + + + + Debug + AnyCPU + {8068B622-46A9-4ABA-BDA3-6AB259D1682C} + Exe + Example4a_WithExtractorProgress + Example4a-WithExtractorProgress + v4.8 + 512 + true + true + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + latest + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + latest + + + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + + + + ..\..\..\packages\System.Runtime.4.3.1\lib\net462\System.Runtime.dll + True + True + + + ..\..\..\packages\System.Runtime.CompilerServices.Unsafe.6.1.2\lib\net462\System.Runtime.CompilerServices.Unsafe.dll + + + ..\..\..\packages\System.Threading.Tasks.Extensions.4.6.3\lib\net462\System.Threading.Tasks.Extensions.dll + + + + + + + + + + + + + + + + {c4987bad-4513-955f-b3c1-7563d0c1a7a3} + Wolfgang.Etl.Abstractions + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example4a-WithExtractorProgress/Program.cs b/examples/Net4.8/Example4a-WithExtractorProgress/Program.cs new file mode 100644 index 0000000..2a59be0 --- /dev/null +++ b/examples/Net4.8/Example4a-WithExtractorProgress/Program.cs @@ -0,0 +1,59 @@ +using System; +using System.Threading.Tasks; +using Example4a_WithExtractorProgress.ETL; + +namespace Example4a_WithExtractorProgress +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + var progress = new Progress(p => + { + Console.WriteLine($"Extracted {ConsoleColors.Cyan}{p.CurrentCount}{ConsoleColors.Reset} items."); + }); + + // Best practice is to only use one progress reporter per ETL process. Using multiple progress reporters + // can lead to confusion and inconsistent reporting. + var sourceItems = extractor.ExtractAsync(progress); + var transformedItems = transformer.TransformAsync(sourceItems); + await loader.LoadAsync(transformedItems); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + + internal class EtlProgress(int currentCount) + { + public int CurrentCount { get; } = currentCount; + } + + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + public const string Cyan = "\u001b[36m"; + } +} diff --git a/examples/Net4.8/Example4a-WithExtractorProgress/Properties/AssemblyInfo.cs b/examples/Net4.8/Example4a-WithExtractorProgress/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..8166221 --- /dev/null +++ b/examples/Net4.8/Example4a-WithExtractorProgress/Properties/AssemblyInfo.cs @@ -0,0 +1,32 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Example4a-WithExtractorProgress")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Example4a-WithExtractorProgress")] +[assembly: AssemblyCopyright("Copyright © 2025")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("8068b622-46a9-4aba-bda3-6ab259d1682c")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/examples/Net4.8/Example4a-WithExtractorProgress/packages.config b/examples/Net4.8/Example4a-WithExtractorProgress/packages.config new file mode 100644 index 0000000..6c61924 --- /dev/null +++ b/examples/Net4.8/Example4a-WithExtractorProgress/packages.config @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example4b-WithTransformerProgress/App.config b/examples/Net4.8/Example4b-WithTransformerProgress/App.config new file mode 100644 index 0000000..193aecc --- /dev/null +++ b/examples/Net4.8/Example4b-WithTransformerProgress/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example4b-WithTransformerProgress/ETL/ConsoleLoader.cs b/examples/Net4.8/Example4b-WithTransformerProgress/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..88a1cff --- /dev/null +++ b/examples/Net4.8/Example4b-WithTransformerProgress/ETL/ConsoleLoader.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example4b_WithTransformerProgress.ETL +{ + internal class ConsoleLoader : ILoadAsync, ILoadWithProgressAsync + { + + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async Task LoadAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example4b-WithTransformerProgress/ETL/FibonacciExtractor.cs b/examples/Net4.8/Example4b-WithTransformerProgress/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..d7c969a --- /dev/null +++ b/examples/Net4.8/Example4b-WithTransformerProgress/ETL/FibonacciExtractor.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example4b_WithTransformerProgress.ETL +{ + internal class FibonacciExtractor : IExtractAsync, IExtractWithProgressAsync + { + private int _progressInterval = 1_000; + + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync(IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + count = Interlocked.Increment(ref count); + + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example4b-WithTransformerProgress/ETL/IntToStringTransformer.cs b/examples/Net4.8/Example4b-WithTransformerProgress/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..6143842 --- /dev/null +++ b/examples/Net4.8/Example4b-WithTransformerProgress/ETL/IntToStringTransformer.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example4b_WithTransformerProgress.ETL +{ + internal class IntToStringTransformer + : ITransformAsync, ITransformWithProgressAsync + { + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items, IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example4b-WithTransformerProgress/Example4b-WithTransformerProgress.csproj b/examples/Net4.8/Example4b-WithTransformerProgress/Example4b-WithTransformerProgress.csproj new file mode 100644 index 0000000..609a437 --- /dev/null +++ b/examples/Net4.8/Example4b-WithTransformerProgress/Example4b-WithTransformerProgress.csproj @@ -0,0 +1,68 @@ + + + + + Debug + AnyCPU + {6BD7828E-55B4-4213-8DF9-7BFBA891B8EE} + Exe + Example4b_WithTransformerProgress + Example4b-WithTransformerProgress + v4.8 + 512 + true + true + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + latest + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + latest + + + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + + + + ..\..\..\packages\System.Runtime.CompilerServices.Unsafe.6.1.2\lib\net462\System.Runtime.CompilerServices.Unsafe.dll + + + ..\..\..\packages\System.Threading.Tasks.Extensions.4.6.3\lib\net462\System.Threading.Tasks.Extensions.dll + + + + + + + + + + + + + + + + {c4987bad-4513-955f-b3c1-7563d0c1a7a3} + Wolfgang.Etl.Abstractions + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example4b-WithTransformerProgress/Program.cs b/examples/Net4.8/Example4b-WithTransformerProgress/Program.cs new file mode 100644 index 0000000..a02055f --- /dev/null +++ b/examples/Net4.8/Example4b-WithTransformerProgress/Program.cs @@ -0,0 +1,58 @@ +using System; +using System.Threading.Tasks; +using Example4b_WithTransformerProgress.ETL; + +namespace Example4b_WithTransformerProgress +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + var progress = new Progress(p => + { + Console.WriteLine($"Transformed {ConsoleColors.Cyan}{p.CurrentCount}{ConsoleColors.Reset} items."); + }); + + // Best practice is to only use one progress reporter per ETL process. Using multiple progress reporters + // can lead to confusion and inconsistent reporting. + var sourceItems = extractor.ExtractAsync(); + var transformedItems = transformer.TransformAsync(sourceItems, progress); + await loader.LoadAsync(transformedItems, progress); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + internal class EtlProgress(int currentCount) + { + public int CurrentCount { get; } = currentCount; + } + + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + public const string Cyan = "\u001b[36m"; + } +} diff --git a/examples/Net4.8/Example4b-WithTransformerProgress/Properties/AssemblyInfo.cs b/examples/Net4.8/Example4b-WithTransformerProgress/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..690ec78 --- /dev/null +++ b/examples/Net4.8/Example4b-WithTransformerProgress/Properties/AssemblyInfo.cs @@ -0,0 +1,32 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Example4b-WithTransformerProgress")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Example4b-WithTransformerProgress")] +[assembly: AssemblyCopyright("Copyright © 2025")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("6bd7828e-55b4-4213-8df9-7bfba891b8ee")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/examples/Net4.8/Example4b-WithTransformerProgress/packages.config b/examples/Net4.8/Example4b-WithTransformerProgress/packages.config new file mode 100644 index 0000000..27ab3ab --- /dev/null +++ b/examples/Net4.8/Example4b-WithTransformerProgress/packages.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example4c-WithLoaderProgress/App.config b/examples/Net4.8/Example4c-WithLoaderProgress/App.config new file mode 100644 index 0000000..193aecc --- /dev/null +++ b/examples/Net4.8/Example4c-WithLoaderProgress/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example4c-WithLoaderProgress/ETL/ConsoleLoader.cs b/examples/Net4.8/Example4c-WithLoaderProgress/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..96efbd8 --- /dev/null +++ b/examples/Net4.8/Example4c-WithLoaderProgress/ETL/ConsoleLoader.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example4c_WithLoaderProgress.ETL +{ + internal class ConsoleLoader : ILoadWithProgressAsync + { + + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async Task LoadAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example4c-WithLoaderProgress/ETL/FibonacciExtractor.cs b/examples/Net4.8/Example4c-WithLoaderProgress/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..ee10b99 --- /dev/null +++ b/examples/Net4.8/Example4c-WithLoaderProgress/ETL/FibonacciExtractor.cs @@ -0,0 +1,85 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example4c_WithLoaderProgress.ETL +{ + internal class FibonacciExtractor : IExtractAsync, IExtractWithProgressAsync + { + private int _progressInterval = 1_000; + + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync(IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var count = 0; + + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + count = Interlocked.Increment(ref count); + + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example4c-WithLoaderProgress/ETL/IntToStringTransformer.cs b/examples/Net4.8/Example4c-WithLoaderProgress/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..d7dff92 --- /dev/null +++ b/examples/Net4.8/Example4c-WithLoaderProgress/ETL/IntToStringTransformer.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example4c_WithLoaderProgress.ETL +{ + internal class IntToStringTransformer + : ITransformAsync, ITransformWithProgressAsync + { + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items, IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example4c-WithLoaderProgress/Example4c-WithLoaderProgress.csproj b/examples/Net4.8/Example4c-WithLoaderProgress/Example4c-WithLoaderProgress.csproj new file mode 100644 index 0000000..e35d699 --- /dev/null +++ b/examples/Net4.8/Example4c-WithLoaderProgress/Example4c-WithLoaderProgress.csproj @@ -0,0 +1,68 @@ + + + + + Debug + AnyCPU + {6DA63E99-692F-461C-983A-270E5CBE8D45} + Exe + Example4c_WithLoaderProgress + Example4c-WithLoaderProgress + v4.8 + 512 + true + true + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + latest + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + latest + + + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + + + + ..\..\..\packages\System.Runtime.CompilerServices.Unsafe.6.1.2\lib\net462\System.Runtime.CompilerServices.Unsafe.dll + + + ..\..\..\packages\System.Threading.Tasks.Extensions.4.6.3\lib\net462\System.Threading.Tasks.Extensions.dll + + + + + + + + + + + + + + + + {c4987bad-4513-955f-b3c1-7563d0c1a7a3} + Wolfgang.Etl.Abstractions + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example4c-WithLoaderProgress/Program.cs b/examples/Net4.8/Example4c-WithLoaderProgress/Program.cs new file mode 100644 index 0000000..99e3e35 --- /dev/null +++ b/examples/Net4.8/Example4c-WithLoaderProgress/Program.cs @@ -0,0 +1,59 @@ +using System; +using System.Threading.Tasks; +using Example4c_WithLoaderProgress.ETL; + +namespace Example4c_WithLoaderProgress +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + var progress = new Progress(p => + { + Console.WriteLine($"Loaded {ConsoleColors.Cyan}{p.CurrentCount}{ConsoleColors.Reset} items."); + }); + + // Best practice is to only use one progress reporter per ETL process. Using multiple progress reporters + // can lead to confusion and inconsistent reporting. + var sourceItems = extractor.ExtractAsync(); + var transformedItems = transformer.TransformAsync(sourceItems); + await loader.LoadAsync(transformedItems, progress); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + + internal class EtlProgress(int currentCount) + { + public int CurrentCount { get; } = currentCount; + }; + + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + public const string Cyan = "\u001b[36m"; + } +} diff --git a/examples/Net4.8/Example4c-WithLoaderProgress/Properties/AssemblyInfo.cs b/examples/Net4.8/Example4c-WithLoaderProgress/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..d0680e3 --- /dev/null +++ b/examples/Net4.8/Example4c-WithLoaderProgress/Properties/AssemblyInfo.cs @@ -0,0 +1,32 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Example4c-WithLoaderProgress")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Example4c-WithLoaderProgress")] +[assembly: AssemblyCopyright("Copyright © 2025")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("6da63e99-692f-461c-983a-270e5cbe8d45")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/examples/Net4.8/Example4c-WithLoaderProgress/packages.config b/examples/Net4.8/Example4c-WithLoaderProgress/packages.config new file mode 100644 index 0000000..27ab3ab --- /dev/null +++ b/examples/Net4.8/Example4c-WithLoaderProgress/packages.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/App.config b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/App.config new file mode 100644 index 0000000..193aecc --- /dev/null +++ b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/ETL/ConsoleLoader.cs b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..1f146f8 --- /dev/null +++ b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/ETL/ConsoleLoader.cs @@ -0,0 +1,165 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example5a_ExtractorWithProgressAndCancellation.ETL +{ + internal class ConsoleLoader : ILoadWithProgressAndCancellationAsync + { + + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async Task LoadAsync(IAsyncEnumerable items) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, CancellationToken token) + { + + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items.WithCancellation(token)) + { + // You can either throw an exception if cancellation is requested + // token.ThrowIfCancellationRequested(); + + // or gracefully handle it. + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Loading cancelled{ConsoleColors.Reset}."); + return; + } + + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, IProgress progress) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, IProgress progress, CancellationToken token) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + } +} diff --git a/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/ETL/FibonacciExtractor.cs b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..994ace0 --- /dev/null +++ b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/ETL/FibonacciExtractor.cs @@ -0,0 +1,165 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example5a_ExtractorWithProgressAndCancellation.ETL +{ + internal class FibonacciExtractor : IExtractWithProgressAndCancellationAsync + { + private int _progressInterval = 1_000; + + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync([EnumeratorCancellation] CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + // Throw exception if cancellation is requested + // See Example3-ExtractorWithGracefulCancellation for a more graceful cancellation approach + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync(IProgress progress) + { + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } + + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + count = Interlocked.Increment(ref count); + + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync(IProgress progress, [EnumeratorCancellation] CancellationToken token) + { + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } + + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + // You can either throw an exception if cancellation is requested + // token.ThrowIfCancellationRequested(); + + // or gracefully handle it. + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Extraction cancelled{ConsoleColors.Reset}."); + yield break; + } + + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + count = Interlocked.Increment(ref count); + + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/ETL/IntToStringTransformer.cs b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..6cf8b62 --- /dev/null +++ b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/ETL/IntToStringTransformer.cs @@ -0,0 +1,183 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example5a_ExtractorWithProgressAndCancellation.ETL +{ + internal class IntToStringTransformer : ITransformWithProgressAndCancellationAsync + { + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items + ) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + [EnumeratorCancellation] CancellationToken token + ) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items.WithCancellation(token)) + { + // Throw exception if cancellation is requested + // See Example3-ExtractorWithGracefulCancellation for a more graceful cancellation approach + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + IProgress progress + ) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } + + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + IProgress progress, + [EnumeratorCancellation] CancellationToken token + ) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } + + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + // You can either throw an exception if cancellation is requested + // token.ThrowIfCancellationRequested(); + + // or gracefully handle it. + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Transformation cancelled{ConsoleColors.Reset}."); + yield break; + } + + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/Example5a-ExtractorWithProgressAndCancellation.csproj b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/Example5a-ExtractorWithProgressAndCancellation.csproj new file mode 100644 index 0000000..003cce7 --- /dev/null +++ b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/Example5a-ExtractorWithProgressAndCancellation.csproj @@ -0,0 +1,71 @@ + + + + + Debug + AnyCPU + {861EA36D-970E-4CFE-9E72-D3D12F0BBB60} + Exe + Example5a_ExtractorWithProgressAndCancellation + Example5a-ExtractorWithProgressAndCancellation + v4.8 + 512 + true + true + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + latest + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + latest + + + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + + + + ..\..\..\packages\System.Runtime.CompilerServices.Unsafe.6.1.2\lib\net462\System.Runtime.CompilerServices.Unsafe.dll + + + ..\..\..\packages\System.Threading.Tasks.Extensions.4.6.3\lib\net462\System.Threading.Tasks.Extensions.dll + + + + + + + + + + + + + + + + + + + {c4987bad-4513-955f-b3c1-7563d0c1a7a3} + Wolfgang.Etl.Abstractions + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/Program.cs b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/Program.cs new file mode 100644 index 0000000..008c02f --- /dev/null +++ b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/Program.cs @@ -0,0 +1,64 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Example5a_ExtractorWithProgressAndCancellation.ETL; + +namespace Example5a_ExtractorWithProgressAndCancellation +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + var progress = new Progress(p => + { + Console.WriteLine($"Extracted {ConsoleColors.Cyan}{p.CurrentCount}{ConsoleColors.Reset} items."); + }); + + // Set a cancellation token to cancel the extraction after 1 second + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var token = cts.Token; + + // Best practice is to only use one progress reporter per ETL process. Using multiple progress reporters + // can lead to confusion and inconsistent reporting. This example passes the progress reporter to + // the extractor, but you could also pass it to the transformer or loader depending on your needs. + var sourceItems = extractor.ExtractAsync(progress, token); + var transformedItems = transformer.TransformAsync(sourceItems, token); + await loader.LoadAsync(transformedItems, token); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + internal class EtlProgress(int currentCount) + { + public int CurrentCount { get; } = currentCount; + }; + + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + public const string Cyan = "\u001b[36m"; + } +} diff --git a/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/Properties/AssemblyInfo.cs b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..21e169a --- /dev/null +++ b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/Properties/AssemblyInfo.cs @@ -0,0 +1,32 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Example5a-ExtractorWithProgressAndCancellation")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Example5a-ExtractorWithProgressAndCancellation")] +[assembly: AssemblyCopyright("Copyright © 2025")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("861ea36d-970e-4cfe-9e72-d3d12f0bbb60")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/packages.config b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/packages.config new file mode 100644 index 0000000..27ab3ab --- /dev/null +++ b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/packages.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example6-ReducingDuplicateCode/App.config b/examples/Net4.8/Example6-ReducingDuplicateCode/App.config new file mode 100644 index 0000000..193aecc --- /dev/null +++ b/examples/Net4.8/Example6-ReducingDuplicateCode/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example6-ReducingDuplicateCode/ETL/ConsoleLoader.cs b/examples/Net4.8/Example6-ReducingDuplicateCode/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..4e5b95a --- /dev/null +++ b/examples/Net4.8/Example6-ReducingDuplicateCode/ETL/ConsoleLoader.cs @@ -0,0 +1,133 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example6_ReducingDuplicateCode.ETL +{ + internal class ConsoleLoader : ILoadWithProgressAndCancellationAsync + { + + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public Task LoadAsync(IAsyncEnumerable items) + { + if (items is null) + { + throw new ArgumentNullException(nameof(items)); + } + + return WorkerAsync(items, null, CancellationToken.None); + } + + + + public Task LoadAsync(IAsyncEnumerable items, CancellationToken token) + { + if (items is null) + { + throw new ArgumentNullException(nameof(items)); + } + + return WorkerAsync(items, null, token); + } + + + + public Task LoadAsync(IAsyncEnumerable items, IProgress progress) + { + if (items is null) + { + throw new ArgumentNullException(nameof(items)); + } + + if (progress is null) + { + throw new ArgumentNullException(nameof(progress)); + } + + return WorkerAsync(items, progress, CancellationToken.None); + } + + + + public Task LoadAsync(IAsyncEnumerable items, IProgress progress, CancellationToken token) + { + if (items is null) + { + throw new ArgumentNullException(nameof(items)); + } + + if (progress is null) + { + throw new ArgumentNullException(nameof(progress)); + } + + return WorkerAsync(items, progress, token); + } + + + + private async Task WorkerAsync + ( + IAsyncEnumerable items, + IProgress? progress, + CancellationToken token + ) + { + if (items is null) + { + throw new ArgumentNullException(nameof(items)); + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress?.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items.WithCancellation(token)) + { + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + count = Interlocked.Increment(ref count); + + } + + progress?.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + } +} diff --git a/examples/Net4.8/Example6-ReducingDuplicateCode/ETL/FibonacciExtractor.cs b/examples/Net4.8/Example6-ReducingDuplicateCode/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..311f73e --- /dev/null +++ b/examples/Net4.8/Example6-ReducingDuplicateCode/ETL/FibonacciExtractor.cs @@ -0,0 +1,121 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example6_ReducingDuplicateCode.ETL +{ + internal class FibonacciExtractor : IExtractWithProgressAndCancellationAsync + { + private int _progressInterval = 1_000; + + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public IAsyncEnumerable ExtractAsync() + { + return WorkerAsync(null, CancellationToken.None); + } + + + + public IAsyncEnumerable ExtractAsync(CancellationToken token) + { + return WorkerAsync(null, token); + } + + + + public IAsyncEnumerable ExtractAsync(IProgress progress) + { + if (progress is null) + { + throw new ArgumentNullException(nameof(progress)); + } + + return WorkerAsync(progress, CancellationToken.None); + } + + + + public IAsyncEnumerable ExtractAsync + ( + IProgress progress, + CancellationToken token + ) + { + if (progress is null) + { + throw new ArgumentNullException(nameof(progress)); + } + + return WorkerAsync(progress, token); + } + + + + private async IAsyncEnumerable WorkerAsync + ( + IProgress? progress, + [EnumeratorCancellation] CancellationToken token + ) + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress?.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + // You can either throw an exception if cancellation is requested + // token.ThrowIfCancellationRequested(); + + // or gracefully handle it. + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Extraction cancelled{ConsoleColors.Reset}."); + yield break; + } + + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + count = Interlocked.Increment(ref count); + + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + progress?.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example6-ReducingDuplicateCode/ETL/IntToStringTransformer.cs b/examples/Net4.8/Example6-ReducingDuplicateCode/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..9e2a0b6 --- /dev/null +++ b/examples/Net4.8/Example6-ReducingDuplicateCode/ETL/IntToStringTransformer.cs @@ -0,0 +1,158 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Wolfgang.Etl.Abstractions; + +namespace Example6_ReducingDuplicateCode.ETL +{ + internal class IntToStringTransformer : ITransformWithProgressAndCancellationAsync + { + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items + ) + { + if (items is null) + { + throw new ArgumentNullException(nameof(items)); + } + + + return WorkerAsync(items, null, CancellationToken.None); + } + + + + public IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + CancellationToken token + ) + { + if (items is null) + { + throw new ArgumentNullException(nameof(items)); + } + + + return WorkerAsync(items, null, token); + } + + + + public IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + IProgress progress + ) + { + if (items is null) + { + throw new ArgumentNullException(nameof(items)); + } + + if (progress is null) + { + throw new ArgumentNullException(nameof(progress)); + } + + return WorkerAsync(items, progress, CancellationToken.None); + } + + + + public IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + IProgress progress, + CancellationToken token + ) + { + if (items is null) + { + throw new ArgumentNullException(nameof(items)); + } + + if (progress is null) + { + throw new ArgumentNullException(nameof(progress)); + } + + return WorkerAsync(items, progress, token); + + } + + + + private async IAsyncEnumerable WorkerAsync + ( + IAsyncEnumerable items, + IProgress? progress, + [EnumeratorCancellation] CancellationToken token + ) + { + if (items is null) + { + throw new ArgumentNullException(nameof(items)); + } + + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + var count = 0; + using var timer = new Timer + ( + _ => progress?.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items.WithCancellation(token)) + { + // You can either throw an exception if cancellation is requested + // token.ThrowIfCancellationRequested(); + + // or gracefully handle it. + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Transformation cancelled{ConsoleColors.Reset}."); + yield break; + } + + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + count = Interlocked.Increment(ref count); + + } + + progress?.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net4.8/Example6-ReducingDuplicateCode/Example6-ReducingDuplicateCode.csproj b/examples/Net4.8/Example6-ReducingDuplicateCode/Example6-ReducingDuplicateCode.csproj new file mode 100644 index 0000000..0023993 --- /dev/null +++ b/examples/Net4.8/Example6-ReducingDuplicateCode/Example6-ReducingDuplicateCode.csproj @@ -0,0 +1,71 @@ + + + + + Debug + AnyCPU + {85A15A15-D528-4542-A546-63BE0EAED986} + Exe + Example6_ReducingDuplicateCode + Example6-ReducingDuplicateCode + v4.8 + 512 + true + true + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + 12 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + 12 + + + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + + + + ..\..\..\packages\System.Runtime.CompilerServices.Unsafe.6.1.2\lib\net462\System.Runtime.CompilerServices.Unsafe.dll + + + ..\..\..\packages\System.Threading.Tasks.Extensions.4.6.3\lib\net462\System.Threading.Tasks.Extensions.dll + + + + + + + + + + + + + + + + + + + {c4987bad-4513-955f-b3c1-7563d0c1a7a3} + Wolfgang.Etl.Abstractions + + + + \ No newline at end of file diff --git a/examples/Net4.8/Example6-ReducingDuplicateCode/Program.cs b/examples/Net4.8/Example6-ReducingDuplicateCode/Program.cs new file mode 100644 index 0000000..891037b --- /dev/null +++ b/examples/Net4.8/Example6-ReducingDuplicateCode/Program.cs @@ -0,0 +1,155 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Example6_ReducingDuplicateCode.ETL; + +namespace Example6_ReducingDuplicateCode +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + await EtlWithNoProgressOrCancellation(); + + + await EtlWithCancellationToken(); + + + await EtlWithProgress(); + + + await EtlWithProgressAndCancellationToken(); + + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}All ETLs completed.{ConsoleColors.Reset}"); + } + + + + private static async Task EtlWithNoProgressOrCancellation() + { + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process with no progress or cancellation...{ConsoleColors.Reset}\n\n"); + + var sourceItems = extractor.ExtractAsync(); + var transformedItems = transformer.TransformAsync(sourceItems); + await loader.LoadAsync(transformedItems); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + + } + + + + + private static async Task EtlWithCancellationToken() + { + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process wit cancellation...{ConsoleColors.Reset}\n\n"); + + // Set a cancellation token to cancel the extraction after 1 second + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var token = cts.Token; + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + + var sourceItems = extractor.ExtractAsync(token); + var transformedItems = transformer.TransformAsync(sourceItems, token); + await loader.LoadAsync(transformedItems, token); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + + } + + + + + private static async Task EtlWithProgress() + { + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process wit cancellation...{ConsoleColors.Reset}\n\n"); + + var progress = new Progress(p => + { + Console.WriteLine($"Loaded {ConsoleColors.Cyan}{p.CurrentCount}{ConsoleColors.Reset} items."); + }); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + // Best practice is to only use one progress reporter per ETL process. Using multiple progress reporters + // can lead to confusion and inconsistent reporting. This example passes the progress reporter to + // the loader, but you could also pass it to the extractor or transformer depending on your needs. + var sourceItems = extractor.ExtractAsync(); + var transformedItems = transformer.TransformAsync(sourceItems); + await loader.LoadAsync(transformedItems, progress); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + + } + + + + private static async Task EtlWithProgressAndCancellationToken() + { + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process wit cancellation...{ConsoleColors.Reset}\n\n"); + + // Set a cancellation token to cancel the extraction after 1 second + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var token = cts.Token; + + var progress = new Progress(p => + { + Console.WriteLine($"Loaded {ConsoleColors.Cyan}{p.CurrentCount}{ConsoleColors.Reset} items."); + }); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + // Best practice is to only use one progress reporter per ETL process. Using multiple progress reporters + // can lead to confusion and inconsistent reporting. This example passes the progress reporter to + // the loader, but you could also pass it to the extractor or transformer depending on your needs. + var sourceItems = extractor.ExtractAsync(token); + var transformedItems = transformer.TransformAsync(sourceItems, token); + await loader.LoadAsync(transformedItems, progress, token); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + + } + } + + + + internal class EtlProgress(int currentCount) + { + public int CurrentCount { get; } = currentCount; + }; + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + public const string Cyan = "\u001b[36m"; + } +} diff --git a/examples/Net4.8/Example6-ReducingDuplicateCode/Properties/AssemblyInfo.cs b/examples/Net4.8/Example6-ReducingDuplicateCode/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..1699bc0 --- /dev/null +++ b/examples/Net4.8/Example6-ReducingDuplicateCode/Properties/AssemblyInfo.cs @@ -0,0 +1,32 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Example6-ReducingDuplicateCode")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Example6-ReducingDuplicateCode")] +[assembly: AssemblyCopyright("Copyright © 2025")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("85a15a15-d528-4542-a546-63be0eaed986")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/examples/Net4.8/Example6-ReducingDuplicateCode/packages.config b/examples/Net4.8/Example6-ReducingDuplicateCode/packages.config new file mode 100644 index 0000000..27ab3ab --- /dev/null +++ b/examples/Net4.8/Example6-ReducingDuplicateCode/packages.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/Net8.0/Example1-BasicETL/ETL/ConsoleLoader.cs b/examples/Net8.0/Example1-BasicETL/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..c550ace --- /dev/null +++ b/examples/Net8.0/Example1-BasicETL/ETL/ConsoleLoader.cs @@ -0,0 +1,20 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example1_BasicETL.ETL +{ + internal class ConsoleLoader : ILoadAsync + { + public async Task LoadAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example1-BasicETL/ETL/FibonacciExtractor.cs b/examples/Net8.0/Example1-BasicETL/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..256af8e --- /dev/null +++ b/examples/Net8.0/Example1-BasicETL/ETL/FibonacciExtractor.cs @@ -0,0 +1,26 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example1_BasicETL.ETL +{ + internal class FibonacciExtractor : IExtractAsync + { + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example1-BasicETL/ETL/IntToStringTransformer.cs b/examples/Net8.0/Example1-BasicETL/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..4db43fc --- /dev/null +++ b/examples/Net8.0/Example1-BasicETL/ETL/IntToStringTransformer.cs @@ -0,0 +1,21 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example1_BasicETL.ETL +{ + internal class IntToStringTransformer : ITransformAsync + { + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example1-BasicETL/Example1-BasicETL.csproj b/examples/Net8.0/Example1-BasicETL/Example1-BasicETL.csproj new file mode 100644 index 0000000..0bb7c75 --- /dev/null +++ b/examples/Net8.0/Example1-BasicETL/Example1-BasicETL.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + Example1_BasicETL + enable + enable + 1.0.0 + + + + + + + + diff --git a/examples/Net8.0/Example1-BasicETL/Program.cs b/examples/Net8.0/Example1-BasicETL/Program.cs new file mode 100644 index 0000000..31d27f4 --- /dev/null +++ b/examples/Net8.0/Example1-BasicETL/Program.cs @@ -0,0 +1,40 @@ +using Example1_BasicETL.ETL; + +namespace Example1_BasicETL +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + var sourceItems = extractor.ExtractAsync(); + var transformedItems = transformer.TransformAsync(sourceItems); + await loader.LoadAsync(transformedItems); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + } +} diff --git a/examples/Net8.0/Example2-WithCancellationToken/ETL/ConsoleLoader.cs b/examples/Net8.0/Example2-WithCancellationToken/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..0ead20b --- /dev/null +++ b/examples/Net8.0/Example2-WithCancellationToken/ETL/ConsoleLoader.cs @@ -0,0 +1,40 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example2_WithCancellationToken.ETL +{ + internal class ConsoleLoader : ILoadWithCancellationAsync + { + + public async Task LoadAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items.WithCancellation(token)) + { + // Throw exception if cancellation is requested + // See Example3-ExtractorWithGracefulCancellation for a more graceful cancellation approach + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example2-WithCancellationToken/ETL/FibonacciExtractor.cs b/examples/Net8.0/Example2-WithCancellationToken/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..f8e1b3b --- /dev/null +++ b/examples/Net8.0/Example2-WithCancellationToken/ETL/FibonacciExtractor.cs @@ -0,0 +1,52 @@ +using System.Runtime.CompilerServices; +using Wolfgang.Etl.Abstractions; + +namespace Example2_WithCancellationToken.ETL +{ + internal class FibonacciExtractor : IExtractWithCancellationAsync + { + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync([EnumeratorCancellation] CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + // Throw exception if cancellation is requested + // See Example3-ExtractorWithGracefulCancellation for a more graceful cancellation approach + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example2-WithCancellationToken/ETL/IntToStringTransformer.cs b/examples/Net8.0/Example2-WithCancellationToken/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..9e37816 --- /dev/null +++ b/examples/Net8.0/Example2-WithCancellationToken/ETL/IntToStringTransformer.cs @@ -0,0 +1,45 @@ +using System.Runtime.CompilerServices; +using Wolfgang.Etl.Abstractions; + +namespace Example2_WithCancellationToken.ETL +{ + internal class IntToStringTransformer : ITransformWithCancellationAsync + { + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items, [EnumeratorCancellation] CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items.WithCancellation(token)) + { + // Throw exception if cancellation is requested + // See Example3-ExtractorWithGracefulCancellation for a more graceful cancellation approach + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + } +} diff --git a/examples/Net8.0/Example2-WithCancellationToken/Example2-WithCancellationToken.csproj b/examples/Net8.0/Example2-WithCancellationToken/Example2-WithCancellationToken.csproj new file mode 100644 index 0000000..c7472aa --- /dev/null +++ b/examples/Net8.0/Example2-WithCancellationToken/Example2-WithCancellationToken.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + Example2_WithCancellationToken + enable + enable + 1.0.0 + + + + + + + + diff --git a/examples/Net8.0/Example2-WithCancellationToken/Program.cs b/examples/Net8.0/Example2-WithCancellationToken/Program.cs new file mode 100644 index 0000000..33a3515 --- /dev/null +++ b/examples/Net8.0/Example2-WithCancellationToken/Program.cs @@ -0,0 +1,52 @@ +using Example2_WithCancellationToken.ETL; + +namespace Example2_WithCancellationToken +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + // Set a cancellation token to cancel the extraction after 1 second + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var token = cts.Token; + + try + { + var sourceItems = extractor.ExtractAsync(token); + var transformedItems = transformer.TransformAsync(sourceItems, token); + await loader.LoadAsync(transformedItems, token); + } + catch (Exception e) + { + Console.WriteLine(e); + } + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + } +} diff --git a/examples/Net8.0/Example3-WithGracefulCancellation/ETL/ConsoleLoader.cs b/examples/Net8.0/Example3-WithGracefulCancellation/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..ee26128 --- /dev/null +++ b/examples/Net8.0/Example3-WithGracefulCancellation/ETL/ConsoleLoader.cs @@ -0,0 +1,42 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example3_WithGracefulCancellation.ETL +{ + internal class ConsoleLoader : ILoadWithCancellationAsync + { + + public async Task LoadAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Extraction cancelled{ConsoleColors.Reset}."); + return; // Exit the method if cancellation is requested + } + + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example3-WithGracefulCancellation/ETL/FibonacciExtractor.cs b/examples/Net8.0/Example3-WithGracefulCancellation/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..cba272b --- /dev/null +++ b/examples/Net8.0/Example3-WithGracefulCancellation/ETL/FibonacciExtractor.cs @@ -0,0 +1,54 @@ +using System.Runtime.CompilerServices; +using Wolfgang.Etl.Abstractions; + +namespace Example3_WithGracefulCancellation.ETL +{ + internal class FibonacciExtractor : IExtractWithCancellationAsync + { + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync([EnumeratorCancellation] CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Extraction cancelled{ConsoleColors.Reset}."); + yield break; // Exit the method if cancellation is requested + } + + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example3-WithGracefulCancellation/ETL/IntToStringTransformer.cs b/examples/Net8.0/Example3-WithGracefulCancellation/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..21f1d55 --- /dev/null +++ b/examples/Net8.0/Example3-WithGracefulCancellation/ETL/IntToStringTransformer.cs @@ -0,0 +1,45 @@ +using System.Runtime.CompilerServices; +using Wolfgang.Etl.Abstractions; + +namespace Example3_WithGracefulCancellation.ETL +{ + internal class IntToStringTransformer : ITransformWithCancellationAsync + { + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items, [EnumeratorCancellation] CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Extraction cancelled{ConsoleColors.Reset}."); + yield break; // Exit the method if cancellation is requested + } + + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example3-WithGracefulCancellation/Example3-WithGracefulCancellation.csproj b/examples/Net8.0/Example3-WithGracefulCancellation/Example3-WithGracefulCancellation.csproj new file mode 100644 index 0000000..a80a4b7 --- /dev/null +++ b/examples/Net8.0/Example3-WithGracefulCancellation/Example3-WithGracefulCancellation.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + Example3_WithGracefulCancellation + enable + enable + 1.0.0 + + + + + + + + diff --git a/examples/Net8.0/Example3-WithGracefulCancellation/Program.cs b/examples/Net8.0/Example3-WithGracefulCancellation/Program.cs new file mode 100644 index 0000000..41c5e7b --- /dev/null +++ b/examples/Net8.0/Example3-WithGracefulCancellation/Program.cs @@ -0,0 +1,45 @@ +using Example3_WithGracefulCancellation.ETL; + +namespace Example3_WithGracefulCancellation +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + // Set a cancellation token to cancel the extraction after 1 second + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var token = cts.Token; + + var sourceItems = extractor.ExtractAsync(token); + var transformedItems = transformer.TransformAsync(sourceItems, token); + await loader.LoadAsync(transformedItems, token); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + } +} diff --git a/examples/Net8.0/Example4a-WithExtractorProgress/ETL/ConsoleLoader.cs b/examples/Net8.0/Example4a-WithExtractorProgress/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..7fb8954 --- /dev/null +++ b/examples/Net8.0/Example4a-WithExtractorProgress/ETL/ConsoleLoader.cs @@ -0,0 +1,72 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example4a_WithExtractorProgress.ETL +{ + internal class ConsoleLoader : ILoadAsync, ILoadWithProgressAsync + { + + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async Task LoadAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example4a-WithExtractorProgress/ETL/FibonacciExtractor.cs b/examples/Net8.0/Example4a-WithExtractorProgress/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..d536157 --- /dev/null +++ b/examples/Net8.0/Example4a-WithExtractorProgress/ETL/FibonacciExtractor.cs @@ -0,0 +1,82 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example4a_WithExtractorProgress.ETL +{ + internal class FibonacciExtractor : IExtractAsync, IExtractWithProgressAsync + { + private int _progressInterval = 1_000; + + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync(IProgress progress) + { + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + count = Interlocked.Increment(ref count); + + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example4a-WithExtractorProgress/ETL/IntToStringTransformer.cs b/examples/Net8.0/Example4a-WithExtractorProgress/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..8f0978f --- /dev/null +++ b/examples/Net8.0/Example4a-WithExtractorProgress/ETL/IntToStringTransformer.cs @@ -0,0 +1,73 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example4a_WithExtractorProgress.ETL +{ + internal class IntToStringTransformer + : ITransformAsync, ITransformWithProgressAsync + { + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items, IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example4a-WithExtractorProgress/Example4a-WithExtractorProgress.csproj b/examples/Net8.0/Example4a-WithExtractorProgress/Example4a-WithExtractorProgress.csproj new file mode 100644 index 0000000..9d1f6e0 --- /dev/null +++ b/examples/Net8.0/Example4a-WithExtractorProgress/Example4a-WithExtractorProgress.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + Example4a_WithExtractorProgress + enable + enable + 1.0.0 + + + + + + + + diff --git a/examples/Net8.0/Example4a-WithExtractorProgress/Program.cs b/examples/Net8.0/Example4a-WithExtractorProgress/Program.cs new file mode 100644 index 0000000..1fd33f2 --- /dev/null +++ b/examples/Net8.0/Example4a-WithExtractorProgress/Program.cs @@ -0,0 +1,53 @@ +using Example4a_WithExtractorProgress.ETL; + +namespace Example4a_WithExtractorProgress +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + var progress = new Progress(p => + { + Console.WriteLine($"Extracted {ConsoleColors.Cyan}{p.CurrentCount}{ConsoleColors.Reset} items."); + }); + + // Best practice is to only use one progress reporter per ETL process. Using multiple progress reporters + // can lead to confusion and inconsistent reporting. + var sourceItems = extractor.ExtractAsync(progress); + var transformedItems = transformer.TransformAsync(sourceItems); + await loader.LoadAsync(transformedItems); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + internal record EtlProgress(int CurrentCount); + + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + public const string Cyan = "\u001b[36m"; + } +} diff --git a/examples/Net8.0/Example4b-WithTransformerProgress/ETL/ConsoleLoader.cs b/examples/Net8.0/Example4b-WithTransformerProgress/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..bb44712 --- /dev/null +++ b/examples/Net8.0/Example4b-WithTransformerProgress/ETL/ConsoleLoader.cs @@ -0,0 +1,72 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example4b_WithTransformerProgress.ETL +{ + internal class ConsoleLoader : ILoadAsync, ILoadWithProgressAsync + { + + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async Task LoadAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example4b-WithTransformerProgress/ETL/FibonacciExtractor.cs b/examples/Net8.0/Example4b-WithTransformerProgress/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..bd34041 --- /dev/null +++ b/examples/Net8.0/Example4b-WithTransformerProgress/ETL/FibonacciExtractor.cs @@ -0,0 +1,82 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example4b_WithTransformerProgress.ETL +{ + internal class FibonacciExtractor : IExtractAsync, IExtractWithProgressAsync + { + private int _progressInterval = 1_000; + + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync(IProgress progress) + { + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + count = Interlocked.Increment(ref count); + + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example4b-WithTransformerProgress/ETL/IntToStringTransformer.cs b/examples/Net8.0/Example4b-WithTransformerProgress/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..6a932dd --- /dev/null +++ b/examples/Net8.0/Example4b-WithTransformerProgress/ETL/IntToStringTransformer.cs @@ -0,0 +1,73 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example4b_WithTransformerProgress.ETL +{ + internal class IntToStringTransformer + : ITransformAsync, ITransformWithProgressAsync + { + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items, IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example4b-WithTransformerProgress/Example4b-WithTransformerProgress.csproj b/examples/Net8.0/Example4b-WithTransformerProgress/Example4b-WithTransformerProgress.csproj new file mode 100644 index 0000000..c965d78 --- /dev/null +++ b/examples/Net8.0/Example4b-WithTransformerProgress/Example4b-WithTransformerProgress.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + Example4b_WithTransformerProgress + enable + enable + 1.0.0 + + + + + + + + diff --git a/examples/Net8.0/Example4b-WithTransformerProgress/Program.cs b/examples/Net8.0/Example4b-WithTransformerProgress/Program.cs new file mode 100644 index 0000000..48de2e6 --- /dev/null +++ b/examples/Net8.0/Example4b-WithTransformerProgress/Program.cs @@ -0,0 +1,53 @@ +using Example4b_WithTransformerProgress.ETL; + +namespace Example4b_WithTransformerProgress +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + var progress = new Progress(p => + { + Console.WriteLine($"Transformed {ConsoleColors.Cyan}{p.CurrentCount}{ConsoleColors.Reset} items."); + }); + + // Best practice is to only use one progress reporter per ETL process. Using multiple progress reporters + // can lead to confusion and inconsistent reporting. + var sourceItems = extractor.ExtractAsync(); + var transformedItems = transformer.TransformAsync(sourceItems, progress); + await loader.LoadAsync(transformedItems); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + internal record EtlProgress(int CurrentCount); + + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + public const string Cyan = "\u001b[36m"; + } +} diff --git a/examples/Net8.0/Example4c-WithLoaderProgress/ETL/ConsoleLoader.cs b/examples/Net8.0/Example4c-WithLoaderProgress/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..5226ccc --- /dev/null +++ b/examples/Net8.0/Example4c-WithLoaderProgress/ETL/ConsoleLoader.cs @@ -0,0 +1,72 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example4c_WithLoaderProgress.ETL +{ + internal class ConsoleLoader : ILoadAsync, ILoadWithProgressAsync + { + + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async Task LoadAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example4c-WithLoaderProgress/ETL/FibonacciExtractor.cs b/examples/Net8.0/Example4c-WithLoaderProgress/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..49e041d --- /dev/null +++ b/examples/Net8.0/Example4c-WithLoaderProgress/ETL/FibonacciExtractor.cs @@ -0,0 +1,82 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example4c_WithLoaderProgress.ETL +{ + internal class FibonacciExtractor : IExtractAsync, IExtractWithProgressAsync + { + private int _progressInterval = 1_000; + + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync(IProgress progress) + { + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + count = Interlocked.Increment(ref count); + + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example4c-WithLoaderProgress/ETL/IntToStringTransformer.cs b/examples/Net8.0/Example4c-WithLoaderProgress/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..40a7344 --- /dev/null +++ b/examples/Net8.0/Example4c-WithLoaderProgress/ETL/IntToStringTransformer.cs @@ -0,0 +1,73 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example4c_WithLoaderProgress.ETL +{ + internal class IntToStringTransformer + : ITransformAsync, ITransformWithProgressAsync + { + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync(IAsyncEnumerable items, IProgress progress) + { + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example4c-WithLoaderProgress/Example4c-WithLoaderProgress.csproj b/examples/Net8.0/Example4c-WithLoaderProgress/Example4c-WithLoaderProgress.csproj new file mode 100644 index 0000000..27474bb --- /dev/null +++ b/examples/Net8.0/Example4c-WithLoaderProgress/Example4c-WithLoaderProgress.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + Example4c_WithLoaderProgress + enable + enable + 1.0.0 + + + + + + + + diff --git a/examples/Net8.0/Example4c-WithLoaderProgress/Program.cs b/examples/Net8.0/Example4c-WithLoaderProgress/Program.cs new file mode 100644 index 0000000..bdc4bae --- /dev/null +++ b/examples/Net8.0/Example4c-WithLoaderProgress/Program.cs @@ -0,0 +1,54 @@ +using Example4c_WithLoaderProgress.ETL; + +namespace Example4c_WithLoaderProgress +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + var progress = new Progress(p => + { + Console.WriteLine($"Loaded {ConsoleColors.Cyan}{p.CurrentCount}{ConsoleColors.Reset} items."); + }); + + // Best practice is to only use one progress reporter per ETL process. Using multiple progress reporters + // can lead to confusion and inconsistent reporting. + var sourceItems = extractor.ExtractAsync(); + var transformedItems = transformer.TransformAsync(sourceItems); + await loader.LoadAsync(transformedItems, progress); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + + internal record EtlProgress(int CurrentCount); + + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + public const string Cyan = "\u001b[36m"; + } +} diff --git a/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/ETL/ConsoleLoader.cs b/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..b80de1c --- /dev/null +++ b/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/ETL/ConsoleLoader.cs @@ -0,0 +1,141 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example5a_ExtractorWithProgressAndCancellation.ETL +{ + internal class ConsoleLoader : ILoadWithProgressAndCancellationAsync + { + + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async Task LoadAsync(IAsyncEnumerable items) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, CancellationToken token) + { + + ArgumentNullException.ThrowIfNull(items, nameof(items)); + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + await foreach (var item in items.WithCancellation(token)) + { + // You can either throw an exception if cancellation is requested + // token.ThrowIfCancellationRequested(); + + // or gracefully handle it. + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Loading cancelled{ConsoleColors.Reset}."); + return; + } + + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + } + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, IProgress progress) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + + public async Task LoadAsync(IAsyncEnumerable items, IProgress progress, CancellationToken token) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + } +} diff --git a/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/ETL/FibonacciExtractor.cs b/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..fad0f09 --- /dev/null +++ b/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/ETL/FibonacciExtractor.cs @@ -0,0 +1,155 @@ +using System.Runtime.CompilerServices; +using Wolfgang.Etl.Abstractions; + +namespace Example5a_ExtractorWithProgressAndCancellation.ETL +{ + internal class FibonacciExtractor : IExtractWithProgressAndCancellationAsync + { + private int _progressInterval = 1_000; + + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async IAsyncEnumerable ExtractAsync() + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync([EnumeratorCancellation] CancellationToken token) + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + // Throw exception if cancellation is requested + // See Example3-ExtractorWithGracefulCancellation for a more graceful cancellation approach + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync(IProgress progress) + { + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + count = Interlocked.Increment(ref count); + + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable ExtractAsync(IProgress progress, [EnumeratorCancellation] CancellationToken token) + { + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + // You can either throw an exception if cancellation is requested + // token.ThrowIfCancellationRequested(); + + // or gracefully handle it. + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Extraction cancelled{ConsoleColors.Reset}."); + yield break; + } + + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + count = Interlocked.Increment(ref count); + + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/ETL/IntToStringTransformer.cs b/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..6b32431 --- /dev/null +++ b/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/ETL/IntToStringTransformer.cs @@ -0,0 +1,159 @@ +using System.Runtime.CompilerServices; +using Wolfgang.Etl.Abstractions; + +namespace Example5a_ExtractorWithProgressAndCancellation.ETL +{ + internal class IntToStringTransformer : ITransformWithProgressAndCancellationAsync + { + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public async IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items + ) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + [EnumeratorCancellation] CancellationToken token + ) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + await foreach (var item in items.WithCancellation(token)) + { + // Throw exception if cancellation is requested + // See Example3-ExtractorWithGracefulCancellation for a more graceful cancellation approach + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + } + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + IProgress progress + ) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + + + + public async IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + IProgress progress, + [EnumeratorCancellation] CancellationToken token + ) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items) + { + // You can either throw an exception if cancellation is requested + // token.ThrowIfCancellationRequested(); + + // or gracefully handle it. + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Transformation cancelled{ConsoleColors.Reset}."); + yield break; + } + + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + count = Interlocked.Increment(ref count); + + } + + progress.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/Example5a-ExtractorWithProgressAndCancellation.csproj b/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/Example5a-ExtractorWithProgressAndCancellation.csproj new file mode 100644 index 0000000..192e917 --- /dev/null +++ b/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/Example5a-ExtractorWithProgressAndCancellation.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + Example5a_ExtractorWithProgressAndCancellation + enable + enable + 1.0.0 + + + + + + + + diff --git a/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/Program.cs b/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/Program.cs new file mode 100644 index 0000000..0b999aa --- /dev/null +++ b/examples/Net8.0/Example5a-ExtractorWithProgressAndCancellation/Program.cs @@ -0,0 +1,58 @@ +using Example5a_ExtractorWithProgressAndCancellation.ETL; + +namespace Example5a_ExtractorWithProgressAndCancellation +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process...{ConsoleColors.Reset}\n\n"); + + var progress = new Progress(p => + { + Console.WriteLine($"Extracted {ConsoleColors.Cyan}{p.CurrentCount}{ConsoleColors.Reset} items."); + }); + + // Set a cancellation token to cancel the extraction after 1 second + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var token = cts.Token; + + // Best practice is to only use one progress reporter per ETL process. Using multiple progress reporters + // can lead to confusion and inconsistent reporting. This example passes the progress reporter to + // the extractor, but you could also pass it to the transformer or loader depending on your needs. + var sourceItems = extractor.ExtractAsync(progress, token); + var transformedItems = transformer.TransformAsync(sourceItems, token); + await loader.LoadAsync(transformedItems, token); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + } + } + + + internal record EtlProgress(int CurrentCount); + + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + public const string Cyan = "\u001b[36m"; + } +} diff --git a/examples/Net8.0/Example6-ReducingDuplicateCode/ETL/ConsoleLoader.cs b/examples/Net8.0/Example6-ReducingDuplicateCode/ETL/ConsoleLoader.cs new file mode 100644 index 0000000..bdc98e8 --- /dev/null +++ b/examples/Net8.0/Example6-ReducingDuplicateCode/ETL/ConsoleLoader.cs @@ -0,0 +1,107 @@ +using Wolfgang.Etl.Abstractions; + +namespace Example6_ReducingDuplicateCode.ETL +{ + internal class ConsoleLoader : ILoadWithProgressAndCancellationAsync + { + + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public Task LoadAsync(IAsyncEnumerable items) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + + return WorkerAsync(items, null, CancellationToken.None); + } + + + + public Task LoadAsync(IAsyncEnumerable items, CancellationToken token) + { + + ArgumentNullException.ThrowIfNull(items, nameof(items)); + + return WorkerAsync(items, null, token); + } + + + + public Task LoadAsync(IAsyncEnumerable items, IProgress progress) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + return WorkerAsync(items, progress, CancellationToken.None); + } + + + + public Task LoadAsync(IAsyncEnumerable items, IProgress progress, CancellationToken token) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + return WorkerAsync(items, progress, token); + } + + + + private async Task WorkerAsync + ( + IAsyncEnumerable items, + IProgress? progress, + CancellationToken token + ) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} data to console asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress?.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items.WithCancellation(token)) + { + token.ThrowIfCancellationRequested(); + + Console.WriteLine($"Loading item: {item}\n"); + await Task.Delay(50); // Simulate some delay for loading + count = Interlocked.Increment(ref count); + + } + + progress?.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + + Console.WriteLine($"{ConsoleColors.Green}Loading{ConsoleColors.Reset} completed.\n"); + } + + + } +} diff --git a/examples/Net8.0/Example6-ReducingDuplicateCode/ETL/FibonacciExtractor.cs b/examples/Net8.0/Example6-ReducingDuplicateCode/ETL/FibonacciExtractor.cs new file mode 100644 index 0000000..9550590 --- /dev/null +++ b/examples/Net8.0/Example6-ReducingDuplicateCode/ETL/FibonacciExtractor.cs @@ -0,0 +1,111 @@ +using System.Runtime.CompilerServices; +using Wolfgang.Etl.Abstractions; + +namespace Example6_ReducingDuplicateCode.ETL +{ + internal class FibonacciExtractor : IExtractWithProgressAndCancellationAsync + { + private int _progressInterval = 1_000; + + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public IAsyncEnumerable ExtractAsync() + { + return WorkerAsync(null, CancellationToken.None); + } + + + + public IAsyncEnumerable ExtractAsync(CancellationToken token) + { + return WorkerAsync(null, token); + } + + + + public IAsyncEnumerable ExtractAsync(IProgress progress) + { + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + return WorkerAsync(progress, CancellationToken.None); + } + + + + public IAsyncEnumerable ExtractAsync + ( + IProgress progress, + CancellationToken token + ) + { + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + return WorkerAsync(progress, token); + } + + + + private async IAsyncEnumerable WorkerAsync + ( + IProgress? progress, + [EnumeratorCancellation] CancellationToken token + ) + { + Console.WriteLine($"{ConsoleColors.Green}Extracting{ConsoleColors.Reset} Fibonacci numbers asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress?.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + // You can either throw an exception if cancellation is requested + // token.ThrowIfCancellationRequested(); + + // or gracefully handle it. + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Extraction cancelled{ConsoleColors.Reset}."); + yield break; + } + + Console.WriteLine($"Extracting Fibonacci number {x + 1}: {current}"); + yield return current; + count = Interlocked.Increment(ref count); + + var temp = current; + current += previous; + previous = temp; + await Task.Delay(100); // Simulate asynchronous operation + } + + progress?.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Extraction{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example6-ReducingDuplicateCode/ETL/IntToStringTransformer.cs b/examples/Net8.0/Example6-ReducingDuplicateCode/ETL/IntToStringTransformer.cs new file mode 100644 index 0000000..24c7f7c --- /dev/null +++ b/examples/Net8.0/Example6-ReducingDuplicateCode/ETL/IntToStringTransformer.cs @@ -0,0 +1,129 @@ +using System.Runtime.CompilerServices; +using Wolfgang.Etl.Abstractions; + +namespace Example6_ReducingDuplicateCode.ETL +{ + internal class IntToStringTransformer : ITransformWithProgressAndCancellationAsync + { + + private int _progressInterval = 1_000; + + /// + /// The number of milliseconds between progress updates. + /// + public int ProgressInterval + { + get => _progressInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Progress interval must be greater than 0."); + } + _progressInterval = value; + } + } + + + + public IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items + ) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + + return WorkerAsync(items, null, CancellationToken.None); + } + + + + public IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + CancellationToken token + ) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + + return WorkerAsync(items, null, token); + } + + + + public IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + IProgress progress + ) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + return WorkerAsync(items, progress, CancellationToken.None); + } + + + + public IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + IProgress progress, + CancellationToken token + ) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + ArgumentNullException.ThrowIfNull(progress, nameof(progress)); + + return WorkerAsync(items, progress, token); + + } + + + + private async IAsyncEnumerable WorkerAsync + ( + IAsyncEnumerable items, + IProgress? progress, + [EnumeratorCancellation] CancellationToken token + ) + { + ArgumentNullException.ThrowIfNull(items, nameof(items)); + + Console.WriteLine($"{ConsoleColors.Green}Transforming{ConsoleColors.Reset} integers to strings asynchronously...\n"); + + var count = 0; + await using var timer = new Timer + ( + _ => progress?.Report(new EtlProgress(Volatile.Read(ref count))), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(_progressInterval) // Use the configured progress interval + ); + + + await foreach (var item in items.WithCancellation(token)) + { + // You can either throw an exception if cancellation is requested + // token.ThrowIfCancellationRequested(); + + // or gracefully handle it. + if (token.IsCancellationRequested) + { + Console.WriteLine($"{ConsoleColors.Red}Transformation cancelled{ConsoleColors.Reset}."); + yield break; + } + + Console.WriteLine($"Transforming integer {item} to string."); + await Task.Delay(50); // Simulate some delay for transformation + yield return item.ToString(); + count = Interlocked.Increment(ref count); + + } + + progress?.Report(new EtlProgress(Volatile.Read(ref count))); // Report final count + + Console.WriteLine($"{ConsoleColors.Green}Transformation{ConsoleColors.Reset} completed.\n"); + } + } +} diff --git a/examples/Net8.0/Example6-ReducingDuplicateCode/Example6-ReducingDuplicateCode.csproj b/examples/Net8.0/Example6-ReducingDuplicateCode/Example6-ReducingDuplicateCode.csproj new file mode 100644 index 0000000..86f1c34 --- /dev/null +++ b/examples/Net8.0/Example6-ReducingDuplicateCode/Example6-ReducingDuplicateCode.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + Example6_ReducingDuplicateCode + enable + enable + 1.0.0 + + + + + + + + diff --git a/examples/Net8.0/Example6-ReducingDuplicateCode/Program.cs b/examples/Net8.0/Example6-ReducingDuplicateCode/Program.cs new file mode 100644 index 0000000..f5f931d --- /dev/null +++ b/examples/Net8.0/Example6-ReducingDuplicateCode/Program.cs @@ -0,0 +1,150 @@ +using Example6_ReducingDuplicateCode.ETL; + +namespace Example6_ReducingDuplicateCode +{ + internal class Program + { + private static async Task Main() + { + // Print assembly version + var assembly = typeof(Program).Assembly; + var assemblyVersion = assembly.GetName().Version; + Console.WriteLine($"{ConsoleColors.Green}Assembly Version: {assemblyVersion}{ConsoleColors.Reset}\n"); + + // Print .NET Framework version + var frameworkVersion = Environment.Version; + Console.WriteLine($"{ConsoleColors.Green}.NET Version: {frameworkVersion}{ConsoleColors.Reset}\n"); + + + await EtlWithNoProgressOrCancellation(); + + + await EtlWithCancellationToken(); + + + await EtlWithProgress(); + + + await EtlWithProgressAndCancellationToken(); + + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}All ETLs completed.{ConsoleColors.Reset}"); + } + + + + private static async Task EtlWithNoProgressOrCancellation() + { + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process with no progress or cancellation...{ConsoleColors.Reset}\n\n"); + + var sourceItems = extractor.ExtractAsync(); + var transformedItems = transformer.TransformAsync(sourceItems); + await loader.LoadAsync(transformedItems); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + + } + + + + + private static async Task EtlWithCancellationToken() + { + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process wit cancellation...{ConsoleColors.Reset}\n\n"); + + // Set a cancellation token to cancel the extraction after 1 second + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var token = cts.Token; + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + + var sourceItems = extractor.ExtractAsync(token); + var transformedItems = transformer.TransformAsync(sourceItems, token); + await loader.LoadAsync(transformedItems, token); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + + } + + + + + private static async Task EtlWithProgress() + { + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process wit cancellation...{ConsoleColors.Reset}\n\n"); + + var progress = new Progress(p => + { + Console.WriteLine($"Loaded {ConsoleColors.Cyan}{p.CurrentCount}{ConsoleColors.Reset} items."); + }); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + // Best practice is to only use one progress reporter per ETL process. Using multiple progress reporters + // can lead to confusion and inconsistent reporting. This example passes the progress reporter to + // the loader, but you could also pass it to the extractor or transformer depending on your needs. + var sourceItems = extractor.ExtractAsync(); + var transformedItems = transformer.TransformAsync(sourceItems); + await loader.LoadAsync(transformedItems, progress); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + + } + + + + private static async Task EtlWithProgressAndCancellationToken() + { + Console.WriteLine($"{ConsoleColors.Yellow} Starting ETL process wit cancellation...{ConsoleColors.Reset}\n\n"); + + // Set a cancellation token to cancel the extraction after 1 second + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var token = cts.Token; + + var progress = new Progress(p => + { + Console.WriteLine($"Loaded {ConsoleColors.Cyan}{p.CurrentCount}{ConsoleColors.Reset} items."); + }); + + + var extractor = new FibonacciExtractor(); + var transformer = new IntToStringTransformer(); + var loader = new ConsoleLoader(); + + // Best practice is to only use one progress reporter per ETL process. Using multiple progress reporters + // can lead to confusion and inconsistent reporting. This example passes the progress reporter to + // the loader, but you could also pass it to the extractor or transformer depending on your needs. + var sourceItems = extractor.ExtractAsync(token); + var transformedItems = transformer.TransformAsync(sourceItems, token); + await loader.LoadAsync(transformedItems, progress, token); + + Console.WriteLine($"\n\n{ConsoleColors.Yellow}ETL process completed.{ConsoleColors.Reset}"); + + } + } + + + + internal record EtlProgress(int CurrentCount); + + + + internal class ConsoleColors + { + public const string Green = "\u001b[32m"; + public const string Yellow = "\u001b[33m"; + public const string Reset = "\u001b[0m"; + public const string Red = "\u001b[31m"; + public const string Cyan = "\u001b[36m"; + } +} diff --git a/src/Wolfgang.Etl.Abstractions.sln b/src/Wolfgang.Etl.Abstractions.sln deleted file mode 100644 index cd85e4f..0000000 --- a/src/Wolfgang.Etl.Abstractions.sln +++ /dev/null @@ -1,22 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.31903.59 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wolfgang.Etl.Abstractions", "Wolfgang.Etl.Abstractions\Wolfgang.Etl.Abstractions.csproj", "{3F8DCAF2-8420-47D0-8F3B-DD7F81B72ED0}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {3F8DCAF2-8420-47D0-8F3B-DD7F81B72ED0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3F8DCAF2-8420-47D0-8F3B-DD7F81B72ED0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3F8DCAF2-8420-47D0-8F3B-DD7F81B72ED0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3F8DCAF2-8420-47D0-8F3B-DD7F81B72ED0}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/src/Wolfgang.Etl.Abstractions/Class1.cs b/src/Wolfgang.Etl.Abstractions/Class1.cs deleted file mode 100644 index 438b407..0000000 --- a/src/Wolfgang.Etl.Abstractions/Class1.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Wolfgang.Etl.Abstractions -{ - internal class Class1 - { - public void Update(string author) - { - // Testing CodeQL - var commandText = "SELECT * FROM books WHERE author LIKE '%" + author + "%'"; - } - } -} diff --git a/src/Wolfgang.Etl.Abstractions/ExtractorBase.cs b/src/Wolfgang.Etl.Abstractions/ExtractorBase.cs new file mode 100644 index 0000000..3ef9a6a --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/ExtractorBase.cs @@ -0,0 +1,321 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Threading; + + + +namespace Wolfgang.Etl.Abstractions +{ + public abstract class ExtractorBase + : IExtractWithProgressAndCancellationAsync + { + + private int _reportingInterval = 1_000; + private int _maximumItemCount = int.MaxValue; + private int _skippedItemCount; + private int _currentItemCount; + private int _currentSkippedItemCount; + + + /// + /// The number of milliseconds between progress updates. + /// + /// Value cannot be less than 1 + public int ReportingInterval + { + get => _reportingInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Reporting interval must be greater than 0."); + } + _reportingInterval = value; + } + } + + + /// + /// The current number of items extracted so far. + /// + /// + /// It is the responsibility of the derived class to keep this value up to date as the + /// base class will have no way of knowing the correct value + /// + + [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] + public int CurrentItemCount + { + get => _currentItemCount; + protected set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value)); + } + _currentItemCount = value; + } + } + + + + /// + /// Gets the current number of records skipped + /// + public int CurrentSkippedItemCount + { + get => _currentSkippedItemCount; + protected set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), "value cannot be less than 0."); + } + } + } + + + + /// + /// The maximum number of items to extract. Once the extractor has reached this limit, + /// it should stop extracting and signal the end of the sequence. + /// + /// + /// This is useful for partially extracting data from a source, especially when the source is large + /// or infinite or during development. + /// + /// The specified value is less than 0 + /// + /// + /// var count = 0; + /// using (var reader = new StreamReader(filePath)) + /// { + /// while (!reader.EndOfStream) + /// { + /// yield return await reader.ReadLineAsync(); + /// count++; + /// if (count >= MaximumItemCount) + /// { + /// Console.WriteLine("Maximum item count reached. Stopping extraction."); + /// break; // Stop extracting if the maximum item count is reached + /// } + /// } + /// } + /// + /// + + [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] + public int MaximumItemCount + { + get => _maximumItemCount; + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), "Maximum item count cannot be less than 0."); + } + _maximumItemCount = value; + } + } + + + + /// + /// The number of items to skip before extracting. + /// The extractor should skip the specified number of items before starting to yield results. + /// + /// + /// This is useful for partially extracting data from a source during development, or to skip + /// items that were already processed or are not relevant for the current extraction. + /// + /// The specified value is less than 0 + /// + /// + /// using (var reader = new StreamReader(filePath)) + /// { + /// // Skip the specified number of items before starting to yield results + /// + /// var skipCount = 0; + /// while (!reader.EndOfStream && skipCount < SkipItemCount) + /// { + /// await reader.ReadLineAsync(); + /// skipCount++; + /// } + /// + /// + /// // Now start yielding results + /// + /// var count++; + /// while (!reader.EndOfStream) + /// { + /// yield return await reader.ReadLineAsync(); + /// count++; + /// } + /// } + /// + /// + + [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] + public int SkipItemCount + { + get => _skippedItemCount; + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), "Skip item count cannot be less than 0."); + } + _skippedItemCount = value; + } + } + + + + /// + /// Asynchronously extracts data of type TSource from a source. + /// + /// + /// IAsyncEnumerable<T> + /// The result may be an empty sequence if no data is available or if the extraction fails. + /// + public virtual IAsyncEnumerable ExtractAsync() + { + return ExtractWorkerAsync(CancellationToken.None); + } + + + + /// + /// Asynchronously extracts data of type TSource from a source. + /// + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// IAsyncEnumerable<T> + /// The result may be an empty sequence if no data is available or if the extraction fails. + /// + /// + /// The extractor should be able to handle cancellation requests gracefully. + /// If the caller doesn't plan on cancelling the extraction, CancellationToken.None should be passed in. + /// + public virtual IAsyncEnumerable ExtractAsync(CancellationToken token) + { + return ExtractWorkerAsync(token); + } + + + + /// + /// Asynchronously extracts data of type TSource from a source. + /// + /// A provider for progress updates. + /// + /// IAsyncEnumerable<T> + /// The result may be an empty sequence if no data is available or if the extraction fails. + /// + /// The value of progress is null + public virtual IAsyncEnumerable ExtractAsync(IProgress progress) + { + if (progress == null) + { + throw new ArgumentNullException(nameof(progress), "Progress cannot be null."); + } + + using var timer = new Timer + ( + _ => progress.Report(CreateProgressReport()), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(ReportingInterval) + ); + + return ExtractWorkerAsync(CancellationToken.None); + } + + + + /// + /// Asynchronously extracts data of type TSource from a source. + /// + /// A provider for progress updates. + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// IAsyncEnumerable<T> + /// The result may be an empty sequence if no data is available or if the extraction fails. + /// + /// + /// The extractor should be able to handle cancellation requests gracefully. + /// If the caller doesn't plan on cancelling the extraction, CancellationToken.None should be passed in. + /// + /// The value of progress is null + public virtual IAsyncEnumerable ExtractAsync(IProgress progress, CancellationToken token) + { + if (progress == null) + { + throw new ArgumentNullException(nameof(progress), "Progress cannot be null."); + } + + using var timer = new Timer + ( + _ => progress.Report(CreateProgressReport()), + null, + TimeSpan.Zero, + + TimeSpan.FromMilliseconds(ReportingInterval) + ); + + + return ExtractWorkerAsync(token); + } + + + + /// + /// This method is the core implementation of the extraction logic and should be + /// overridden by derived classes. + /// + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// IAsyncEnumerable<T> + /// The result may be an empty sequence if no data is available or if the extraction fails. + /// + protected abstract IAsyncEnumerable ExtractWorkerAsync(CancellationToken token); + + + + /// + /// Creates a progress report of type TProgress. This gives the derived class the opportunity to + /// implement a custom progress report that is specific to the extraction process. + /// + /// Progress of type TProgress + protected abstract TProgress CreateProgressReport(); + + + + /// + /// Increments the CurrentItemCount in a thread safe manner. + /// + /// + /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not + /// thread safe. This method ensures that CurrentItemCount is incremented safely + /// + protected void IncrementCurrentItemCount() + { + Interlocked.Increment(ref _currentItemCount); + } + + + + /// + /// Increments the CurrentItemCount in a thread safe manner. + /// + /// + /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not + /// thread safe. This method ensures that CurrentItemCount is incremented safely + /// + protected void IncrementCurrentSkippedItemCount() + { + Interlocked.Increment(ref _currentSkippedItemCount); + } + } +} diff --git a/src/Wolfgang.Etl.Abstractions/IExtractAsync.cs b/src/Wolfgang.Etl.Abstractions/IExtractAsync.cs new file mode 100644 index 0000000..a4ca924 --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/IExtractAsync.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; + +namespace Wolfgang.Etl.Abstractions +{ + /// + /// Defines an asynchronous extractor interface for extracting data of type T. + /// A class implementing this interface is intended to be the first step in an + /// ETL (Extract, Transform, Load) process. + /// + /// Represents a single item from the source of the ETL + /// + /// The extractor is responsible for pulling data from a source, which could be a + /// database, file, API, web service or any other data source. It is also responsible for + /// handling any exceptions that may occur during the extraction process, including + /// but not limited to network issues, data format errors, or source unavailability. + /// The extractor should be resilient and capable of retrying operations in case of + /// transient failures. The extracted data is returned as an asynchronous stream of + /// type T, allowing for efficient processing by the transformer and loader components + /// of the ETL pipeline. Ideally, the extractor should NOT do any transformation of the + /// data; its sole responsibility is to pull data from the source and provide it in its + /// raw form to the next step in the ETL process. However, occasionally, some minimal + /// transformation may be necessary to ensure the data is in a suitable format for further + /// processing. For example, if the data is being extracted from a CSV file the same + /// library that parses the CSV file may also bind the data to a specific type T. In this case + /// the pragmatic approach is to allow the extractor to perform this minimal transformation + /// + public interface IExtractAsync + where TSource : notnull + + { + /// + /// Asynchronously extracts data of type TSource from a source. + /// + /// IAsyncEnumerable<T> + /// The result may be an empty sequence if no data is available or if the extraction fails. + IAsyncEnumerable ExtractAsync(); + } +} diff --git a/src/Wolfgang.Etl.Abstractions/IExtractWithCancellationAsync.cs b/src/Wolfgang.Etl.Abstractions/IExtractWithCancellationAsync.cs new file mode 100644 index 0000000..0adcb9b --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/IExtractWithCancellationAsync.cs @@ -0,0 +1,45 @@ +using System.Collections.Generic; +using System.Threading; + +namespace Wolfgang.Etl.Abstractions +{ + /// + /// Defines an asynchronous extractor interface for extracting data of type T. + /// A class implementing this interface is intended to be the first step in an + /// ETL (Extract, Transform, Load) process. + /// + /// Represents a single item from the source of the ETL + /// + /// The extractor is responsible for pulling data from a source, which could be a + /// database, file, API, web service or any other data source. It is also responsible for + /// handling any exceptions that may occur during the extraction process, including + /// but not limited to network issues, data format errors, or source unavailability. + /// The extractor should be resilient and capable of retrying operations in case of + /// transient failures. The extracted data is returned as an asynchronous stream of + /// type T, allowing for efficient processing by the transformer and loader components + /// of the ETL pipeline. Ideally, the extractor should NOT do any transformation of the + /// data; its sole responsibility is to pull data from the source and provide it in its + /// raw form to the next step in the ETL process. However, occasionally, some minimal + /// transformation may be necessary to ensure the data is in a suitable format for further + /// processing. For example, if the data is being extracted from a CSV file the same + /// library that parses the CSV file may also bind the data to a specific type T. In this case + /// the pragmatic approach is to allow the extractor to perform this minimal transformation + /// + public interface IExtractWithCancellationAsync : + IExtractAsync + where TSource : notnull + { + /// + /// Asynchronously extracts data of type TSource from a source. + /// + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// IAsyncEnumerable<T> - The result may be an empty sequence if no data is available or if the extraction fails. + /// + /// + /// The extractor should be able to handle cancellation requests gracefully. + /// If the caller doesn't plan on cancelling the extraction, they can pass CancellationToken.None. + /// + IAsyncEnumerable ExtractAsync(CancellationToken token); + } +} diff --git a/src/Wolfgang.Etl.Abstractions/IExtractWithProgressAndCancellationAsync.cs b/src/Wolfgang.Etl.Abstractions/IExtractWithProgressAndCancellationAsync.cs new file mode 100644 index 0000000..b41c186 --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/IExtractWithProgressAndCancellationAsync.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Threading; + + +namespace Wolfgang.Etl.Abstractions +{ + /// + /// Defines an asynchronous extractor interface for extracting data of type T. + /// A class implementing this interface is intended to be the first step in an + /// ETL (Extract, Transform, Load) process. + /// + /// Represents a single item from the source of the ETL + /// The value of the updated progress + /// + /// The extractor is responsible for pulling data from a source, which could be a + /// database, file, API, web service or any other data source. It is also responsible for + /// handling any exceptions that may occur during the extraction process, including + /// but not limited to network issues, data format errors, or source unavailability. + /// The extractor should be resilient and capable of retrying operations in case of + /// transient failures. The extracted data is returned as an asynchronous stream of + /// type T, allowing for efficient processing by the transformer and loader components + /// of the ETL pipeline. Ideally, the extractor should NOT do any transformation of the + /// data; its sole responsibility is to pull data from the source and provide it in its + /// raw form to the next step in the ETL process. However, occasionally, some minimal + /// transformation may be necessary to ensure the data is in a suitable format for further + /// processing. For example, if the data is being extracted from a CSV file the same + /// library that parses the CSV file may also bind the data to a specific type T. In this case + /// the pragmatic approach is to allow the extractor to perform this minimal transformation + /// + public interface IExtractWithProgressAndCancellationAsync : + IExtractWithCancellationAsync , + IExtractWithProgressAsync + where TSource : notnull where TProgress : notnull + { + /// + /// Asynchronously extracts data of type TSource from a source. + /// + /// A provider for progress updates. + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// IAsyncEnumerable<T> The result may be an empty sequence if no data is available or if the extraction fails. + /// + /// + /// The extractor should be able to handle cancellation requests gracefully. + /// If the caller doesn't plan on cancelling the extraction, they can pass CancellationToken.None. + /// + /// The value of progress is null + IAsyncEnumerable ExtractAsync + ( + IProgress progress, + CancellationToken token + ); + } +} diff --git a/src/Wolfgang.Etl.Abstractions/IExtractWithProgressAsync.cs b/src/Wolfgang.Etl.Abstractions/IExtractWithProgressAsync.cs new file mode 100644 index 0000000..cecf532 --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/IExtractWithProgressAsync.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; + + +namespace Wolfgang.Etl.Abstractions +{ + /// + /// Defines an asynchronous extractor interface for extracting data of type T. + /// A class implementing this interface is intended to be the first step in an + /// ETL (Extract, Transform, Load) process. + /// + /// Represents a single item from the source of the ETL + /// The value of the updated progress + /// + /// The extractor is responsible for pulling data from a source, which could be a + /// database, file, API, web service or any other data source. It is also responsible for + /// handling any exceptions that may occur during the extraction process, including + /// but not limited to network issues, data format errors, or source unavailability. + /// The extractor should be resilient and capable of retrying operations in case of + /// transient failures. The extracted data is returned as an asynchronous stream of + /// type T, allowing for efficient processing by the transformer and loader components + /// of the ETL pipeline. Ideally, the extractor should NOT do any transformation of the + /// data; its sole responsibility is to pull data from the source and provide it in its + /// raw form to the next step in the ETL process. However, occasionally, some minimal + /// transformation may be necessary to ensure the data is in a suitable format for further + /// processing. For example, if the data is being extracted from a CSV file the same + /// library that parses the CSV file may also bind the data to a specific type T. In this case + /// the pragmatic approach is to allow the extractor to perform this minimal transformation + /// + public interface IExtractWithProgressAsync : + IExtractAsync + where TSource : notnull + { + /// + /// Asynchronously extracts data of type TSource from a source. + /// + /// A provider for progress updates. + /// IAsyncEnumerable<T> The result may be an empty sequence if no data is available or if the extraction fails. + /// + /// The value of progress is null + IAsyncEnumerable ExtractAsync + ( + IProgress progress + ); + } +} diff --git a/src/Wolfgang.Etl.Abstractions/ILoadAsync.cs b/src/Wolfgang.Etl.Abstractions/ILoadAsync.cs new file mode 100644 index 0000000..49b909a --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/ILoadAsync.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + + +namespace Wolfgang.Etl.Abstractions +{ + /// + /// Defines an asynchronous loader interface for loading data of type T. + /// A class implementing this interface is intended to be the last step in an + /// ETL (Extract, Transform, Load) process. + /// + /// Represents a single item to be sent to the destination of the ETL + /// + /// The loader is the last step in the ETL process and is responsible for saving + /// data to the destination after processing. The loader is responsible for handling + /// any extractions that may occur during the loading process, including but not + /// limited to communication errors, authentication errors, permission errors. The + /// loader should be resilient and capable of retrying operations in case of + /// transient failures. The loaders should be resilient and capable of retrying + /// operations in case of transient failures. The loaded data is provided as an + /// asynchronous stream of type TDestination, allowing for efficient writing of + /// the data to the destination. Ideally, the loader should NOT do any transformation + /// of the data; its sole responsibility is to save the data to the destination. + /// However, occasionally, some minimal transformation may be necessary to + /// ensure the data is in a suitable format for further processing. For example, if the + /// data is being written to a CSV file, the same library that writes the CSV file + /// may handle the conversion of objects of type T to strings (rows). In this case, + /// the pragmatic approach is to allow the loader to perform this minimal transformation. + /// + public interface ILoadAsync + where TDestination : notnull + + { + /// + /// Loads the data asynchronously. + /// + /// The items to be loaded to the destination. + /// A task representing the asynchronous operation. + /// The value of items is null + Task LoadAsync + ( + IAsyncEnumerable items + ); + } +} \ No newline at end of file diff --git a/src/Wolfgang.Etl.Abstractions/ILoadWithCancellationAsync.cs b/src/Wolfgang.Etl.Abstractions/ILoadWithCancellationAsync.cs new file mode 100644 index 0000000..c27e8a9 --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/ILoadWithCancellationAsync.cs @@ -0,0 +1,51 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + + +namespace Wolfgang.Etl.Abstractions +{ + /// + /// Defines an asynchronous loader interface for loading data of type T. + /// A class implementing this interface is intended to be the last step in an + /// ETL (Extract, Transform, Load) process. + /// + /// Represents a single item to be sent to the destination of the ETL + /// + /// The loader is the last step in the ETL process and is responsible for saving + /// data to the destination after processing. The loader is responsible for handling + /// any extractions that may occur during the loading process, including but not + /// limited to communication errors, authentication errors, permission errors. The + /// loader should be resilient and capable of retrying operations in case of + /// transient failures. The loaders should be resilient and capable of retrying + /// operations in case of transient failures. The loaded data is provided as an + /// asynchronous stream of type TDestination, allowing for efficient writing of + /// the data to the destination. Ideally, the loader should NOT do any transformation + /// of the data; its sole responsibility is to save the data to the destination. + /// However, occasionally, some minimal transformation may be necessary to + /// ensure the data is in a suitable format for further processing. For example, if the + /// data is being written to a CSV file, the same library that writes the CSV file + /// may handle the conversion of objects of type T to strings (rows). In this case, + /// the pragmatic approach is to allow the loader to perform this minimal transformation. + /// + public interface ILoadWithCancellationAsync : + ILoadAsync + where TDestination : notnull + { + /// + /// Loads the data asynchronously. + /// + /// The items to be loaded to the destination. + /// A CancellationToken to observe while waiting for the task to complete. + /// A task representing the asynchronous operation. + /// + /// The extractor should be able to handle cancellation requests gracefully. + /// If the caller doesn't plan on cancelling the extraction, they can pass CancellationToken.None. + /// + Task LoadAsync + ( + IAsyncEnumerable items, + CancellationToken token + ); + } +} \ No newline at end of file diff --git a/src/Wolfgang.Etl.Abstractions/ILoadWithProgressAndCancellationAsync.cs b/src/Wolfgang.Etl.Abstractions/ILoadWithProgressAndCancellationAsync.cs new file mode 100644 index 0000000..658dd17 --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/ILoadWithProgressAndCancellationAsync.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + + +namespace Wolfgang.Etl.Abstractions +{ + /// + /// Defines an asynchronous loader interface for loading data of type T. + /// A class implementing this interface is intended to be the last step in an + /// ETL (Extract, Transform, Load) process. + /// + /// Represents a single item to be sent to the destination of the ETL + /// + /// + /// The loader is the last step in the ETL process and is responsible for saving + /// data to the destination after processing. The loader is responsible for handling + /// any extractions that may occur during the loading process, including but not + /// limited to communication errors, authentication errors, permission errors. The + /// loader should be resilient and capable of retrying operations in case of + /// transient failures. The loaders should be resilient and capable of retrying + /// operations in case of transient failures. The loaded data is provided as an + /// asynchronous stream of type TDestination, allowing for efficient writing of + /// the data to the destination. Ideally, the loader should NOT do any transformation + /// of the data; its sole responsibility is to save the data to the destination. + /// However, occasionally, some minimal transformation may be necessary to + /// ensure the data is in a suitable format for further processing. For example, if the + /// data is being written to a CSV file, the same library that writes the CSV file + /// may handle the conversion of objects of type T to strings (rows). In this case, + /// the pragmatic approach is to allow the loader to perform this minimal transformation. + /// + public interface ILoadWithProgressAndCancellationAsync : + ILoadWithProgressAsync, + ILoadWithCancellationAsync + where TDestination : notnull where TProgress : notnull + { + /// + /// Loads the data asynchronously. + /// + /// The items to be loaded to the destination. + /// A provider for progress updates. + /// A CancellationToken to observe while waiting for the task to complete. + /// A task representing the asynchronous operation. + /// + /// The extractor should be able to handle cancellation requests gracefully. + /// If the caller doesn't plan on cancelling the extraction, they can pass CancellationToken.None. + /// + /// The value of items or progress is null + Task LoadAsync + ( + IAsyncEnumerable items, + IProgress progress, + CancellationToken token + ); + } +} \ No newline at end of file diff --git a/src/Wolfgang.Etl.Abstractions/ILoadWithProgressAsync.cs b/src/Wolfgang.Etl.Abstractions/ILoadWithProgressAsync.cs new file mode 100644 index 0000000..8b6076d --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/ILoadWithProgressAsync.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + + +namespace Wolfgang.Etl.Abstractions +{ + /// + /// Defines an asynchronous loader interface for loading data of type T. + /// A class implementing this interface is intended to be the last step in an + /// ETL (Extract, Transform, Load) process. + /// + /// Represents a single item to be sent to the destination of the ETL + /// + /// + /// The loader is the last step in the ETL process and is responsible for saving + /// data to the destination after processing. The loader is responsible for handling + /// any extractions that may occur during the loading process, including but not + /// limited to communication errors, authentication errors, permission errors. The + /// loader should be resilient and capable of retrying operations in case of + /// transient failures. The loaders should be resilient and capable of retrying + /// operations in case of transient failures. The loaded data is provided as an + /// asynchronous stream of type TDestination, allowing for efficient writing of + /// the data to the destination. Ideally, the loader should NOT do any transformation + /// of the data; its sole responsibility is to save the data to the destination. + /// However, occasionally, some minimal transformation may be necessary to + /// ensure the data is in a suitable format for further processing. For example, if the + /// data is being written to a CSV file, the same library that writes the CSV file + /// may handle the conversion of objects of type T to strings (rows). In this case, + /// the pragmatic approach is to allow the loader to perform this minimal transformation. + /// + public interface ILoadWithProgressAsync : + ILoadAsync + where TDestination : notnull where TProgress : notnull + { + /// + /// Loads the data asynchronously. + /// + /// The items to be loaded to the destination. + /// A provider for progress updates. + /// A task representing the asynchronous operation. + /// The value of items or progress is null + Task LoadAsync + ( + IAsyncEnumerable items, + IProgress progress + ); + } +} \ No newline at end of file diff --git a/src/Wolfgang.Etl.Abstractions/ITransformAsync.cs b/src/Wolfgang.Etl.Abstractions/ITransformAsync.cs new file mode 100644 index 0000000..65fbe9c --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/ITransformAsync.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; + + +namespace Wolfgang.Etl.Abstractions +{ + + /// + /// Defines an asynchronous transformer interface for transforming data of type T to TResult. + /// A class implementing this interface is intended to be the second step in an + /// ETL (Extract, Transform, Load) process. + /// + /// Represents a single item from the source of the ETL + /// Represents a single item to be sent to the destination of the ETL + /// + /// The transformer is responsible for transforming data from source type to the destination type. + /// The transformation may involve converting data types, filtering, aggregating, and mapping data. + /// The transformer should be resilient and capable of handling any exceptions that may occur. The + /// data is provided as an asynchronous stream of type TSource, and returned as an asynchronous + /// stream of type TDestination allowing for efficient processing by the loader component. + /// Ideally, the transformer should do all the transformation of the data, including not limited to + /// deserializing a string of JSON from web service call and binding it to a specific type T or + /// serializing an object of type T to a string of JSON to before passing it to the loader to. + /// + public interface ITransformAsync + where TSource : notnull where TDestination : notnull + + { + /// + /// Asynchronously transforms data of type TSource to TDestination. + /// + /// Asynchronous list of TSource + /// Asynchronous<T> + /// The value of items is null + IAsyncEnumerableTransformAsync + ( + IAsyncEnumerable items + ); + } +} \ No newline at end of file diff --git a/src/Wolfgang.Etl.Abstractions/ITransformWithCancellationAsync.cs b/src/Wolfgang.Etl.Abstractions/ITransformWithCancellationAsync.cs new file mode 100644 index 0000000..ddc4a99 --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/ITransformWithCancellationAsync.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; +using System.Threading; + + +namespace Wolfgang.Etl.Abstractions +{ + /// + /// Defines an asynchronous transformer interface for transforming data of type T to TResult. + /// A class implementing this interface is intended to be the second step in an + /// ETL (Extract, Transform, Load) process. + /// + /// Represents a single item from the source of the ETL + /// Represents a single item to be sent to the destination of the ETL + /// + /// The transformer is responsible for transforming data from source type to the destination type. + /// The transformation may involve converting data types, filtering, aggregating, and mapping data. + /// The transformer should be resilient and capable of handling any exceptions that may occur. The + /// data is provided as an asynchronous stream of type TSource, and returned as an asynchronous + /// stream of type TDestination allowing for efficient processing by the loader component. + /// Ideally, the transformer should do all the transformation of the data, including not limited to + /// deserializing a string of JSON from web service call and binding it to a specific type T or + /// serializing an object of type T to a string of JSON to before passing it to the loader to. + /// + public interface ITransformWithCancellationAsync : + ITransformAsync + where TSource : notnull where TDestination : notnull + { + /// + /// Asynchronously transforms data of type TSource to TDestination. + /// + /// Asynchronous list of TSource + /// A CancellationToken to observe while waiting for the task to complete. + /// Asynchronous<T> + /// + /// The extractor should be able to handle cancellation requests gracefully. + /// If the caller doesn't plan on cancelling the extraction, they can pass CancellationToken.None. + /// + IAsyncEnumerableTransformAsync + ( + IAsyncEnumerable items, + CancellationToken token + ); + } +} diff --git a/src/Wolfgang.Etl.Abstractions/ITransformWithProgressAndCancellationAsync.cs b/src/Wolfgang.Etl.Abstractions/ITransformWithProgressAndCancellationAsync.cs new file mode 100644 index 0000000..f492b64 --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/ITransformWithProgressAndCancellationAsync.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Threading; + + +namespace Wolfgang.Etl.Abstractions +{ + /// + /// Defines an asynchronous transformer interface for transforming data of type T to TResult. + /// A class implementing this interface is intended to be the second step in an + /// ETL (Extract, Transform, Load) process. + /// + /// Represents a single item from the source of the ETL + /// Represents a single item to be sent to the destination of the ETL + /// The value of the updated progress + /// + /// The transformer is responsible for transforming data from source type to the destination type. + /// The transformation may involve converting data types, filtering, aggregating, and mapping data. + /// The transformer should be resilient and capable of handling any exceptions that may occur. The + /// data is provided as an asynchronous stream of type TSource, and returned as an asynchronous + /// stream of type TDestination allowing for efficient processing by the loader component. + /// Ideally, the transformer should do all the transformation of the data, including not limited to + /// deserializing a string of JSON from web service call and binding it to a specific type T or + /// serializing an object of type T to a string of JSON to before passing it to the loader to. + /// + public interface ITransformWithProgressAndCancellationAsync : + ITransformWithCancellationAsync, + ITransformWithProgressAsync + where TSource : notnull where TProgress : notnull + + { + /// + /// Asynchronously transforms data of type TSource to TDestination. + /// + /// Asynchronous list of TSource + /// A provider for progress updates. + /// A CancellationToken to observe while waiting for the task to complete. + /// Asynchronous<T> + /// + /// The extractor should be able to handle cancellation requests gracefully. + /// If the caller doesn't plan on cancelling the extraction, they can pass CancellationToken.None. + /// + /// The value of items or progress is null + IAsyncEnumerableTransformAsync + ( + IAsyncEnumerable items, + IProgress progress, + CancellationToken token + ); + } +} diff --git a/src/Wolfgang.Etl.Abstractions/ITransformWithProgressAsync.cs b/src/Wolfgang.Etl.Abstractions/ITransformWithProgressAsync.cs new file mode 100644 index 0000000..38a71d8 --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/ITransformWithProgressAsync.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; + + +namespace Wolfgang.Etl.Abstractions +{ + /// + /// Defines an asynchronous transformer interface for transforming data of type T to TResult. + /// A class implementing this interface is intended to be the second step in an + /// ETL (Extract, Transform, Load) process. + /// + /// Represents a single item from the source of the ETL + /// Represents a single item to be sent to the destination of the ETL + /// The value of the updated progress + /// + /// The transformer is responsible for transforming data from source type to the destination type. + /// The transformation may involve converting data types, filtering, aggregating, and mapping data. + /// The transformer should be resilient and capable of handling any exceptions that may occur. The + /// data is provided as an asynchronous stream of type TSource, and returned as an asynchronous + /// stream of type TDestination allowing for efficient processing by the loader component. + /// Ideally, the transformer should do all the transformation of the data, including not limited to + /// deserializing a string of JSON from web service call and binding it to a specific type T or + /// serializing an object of type T to a string of JSON to before passing it to the loader to. + /// + public interface ITransformWithProgressAsync : + ITransformAsync + where TSource : notnull where TDestination : notnull where TProgress : notnull + { + /// + /// Asynchronously transforms data of type TSource to TDestination. + /// + /// Asynchronous list of TSource + /// A provider for progress updates. + /// Asynchronous<T> + /// The value of items or progress is null + IAsyncEnumerableTransformAsync + ( + IAsyncEnumerable items, + IProgress progress + ); + } +} diff --git a/src/Wolfgang.Etl.Abstractions/LoaderBase.cs b/src/Wolfgang.Etl.Abstractions/LoaderBase.cs new file mode 100644 index 0000000..52bccb4 --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/LoaderBase.cs @@ -0,0 +1,334 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Threading; +using System.Threading.Tasks; + + + +namespace Wolfgang.Etl.Abstractions +{ + public abstract class LoaderBase + : ILoadWithProgressAndCancellationAsync + { + + private int _reportingInterval = 1_000; + private int _maximumItemCount = int.MaxValue; + private int _skipItemCount; + private int _currentItemCount; + private int _currentSkippedItemCount; + + + /// + /// The number of milliseconds between progress updates. + /// + /// Value cannot be less than 1 + public int ReportingInterval + { + get => _reportingInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Reporting interval must be greater than 0."); + } + _reportingInterval = value; + } + } + + + + /// + /// The current number of items loaded so far. + /// + /// + /// It is the responsibility of the derived class to keep this value up to date as the + /// base class will have no way of knowing the correct value + /// + + [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] + public int CurrentItemCount + { + get => _currentItemCount; + protected set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value)); + } + _currentItemCount = value; + } + } + + + + /// + /// Gets the current number of records skipped + /// + public int CurrentSkippedItemCount + { + get => _currentSkippedItemCount; + protected set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), "value cannot be less than 0."); + } + } + } + + + + + /// + /// The maximum number of items to load. Once the loader has reached this limit, + /// it should stop loading items and exist as if it had reached the end of the list + /// + /// + /// This is useful for partially loading data from a source, especially when the source is large + /// or infinite or during development. + /// + /// The specified value is less than 1 + /// + /// + /// foreach (var item in items.Skip(SkipItemCount).Take(MaxItemCount)) + /// { + /// // Process the item + /// } + /// + /// + + public int MaximumItemCount + { + get => _maximumItemCount; + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), "Maximum item count cannot be less than 0."); + } + _maximumItemCount = value; + } + } + + + + /// + /// The number of items skipped before loading. + /// The loader should skip the specified number of items before starting to process the remaining items. + /// + /// + /// This is useful for skipping the beginning of the list during testing or because it may already be loaded + /// + /// The specified value is less than 0 + /// + /// + /// foreach (var item in items.Skip(SkipItemCount).Take(MaxItemCount)) + /// { + /// // Process the item + /// } + /// + /// + + public int SkipItemCount + { + get => _skipItemCount; + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), "Skip item count cannot be less than 0."); + } + _skipItemCount = value; + } + } + + + + /// + /// Asynchronously loads data of type TDestination into the target destination. + /// + /// The items to be loaded to the destination. + /// + /// Items may be an empty sequence if no data is available or if the extraction fails. + /// + /// Task + /// Argument items is null + public virtual Task LoadAsync + ( + IAsyncEnumerable items + ) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + return LoadWorkerAsync(items, CancellationToken.None); + } + + + + /// + /// Asynchronously loads data of type TDestination into the target destination. + /// + /// The items to be loaded to the destination. + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// Items may be an empty sequence if no data is available or if the extraction fails. + /// + /// Task + /// Argument items is null + public virtual Task LoadAsync + ( + IAsyncEnumerable items, + CancellationToken token + ) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + return LoadWorkerAsync(items, CancellationToken.None); + } + + + + /// + /// Asynchronously loads data of type TDestination into the target destination. + /// + /// The items to be loaded to the destination. + /// A provider for progress updates. + /// + /// Items may be an empty sequence if no data is available or if the extraction fails. + /// + /// Task + /// Argument items is null + /// Argument progress is null + public virtual Task LoadAsync + ( + IAsyncEnumerable items, + IProgress progress + ) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } + + using var timer = new Timer + ( + _ => progress.Report(CreateProgressReport()), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(ReportingInterval) + ); + + return LoadWorkerAsync(items, CancellationToken.None); + } + + + + /// + /// Asynchronously loads data of type TDestination into the target destination. + /// + /// The items to be loaded to the destination. + /// A CancellationToken to observe while waiting for the task to complete. + /// A provider for progress updates. + /// + /// Items may be an empty sequence if no data is available or if the extraction fails. + /// + /// Task + /// Argument items is null + /// Argument progress is null + public virtual Task LoadAsync + ( + IAsyncEnumerable items, + IProgress progress, + CancellationToken token + ) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } + + using var timer = new Timer + ( + _ => progress.Report(CreateProgressReport()), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(ReportingInterval) + ); + + + return LoadWorkerAsync(items, token); + } + + + + /// + /// This method is the core implementation of the loading logic and should be + /// overridden by derived classes. + /// + /// The items to be loaded to the destination. + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// Items may be an empty sequence if no data is available or if the extraction fails. + /// + /// Task + /// Argument items is null + protected abstract Task LoadWorkerAsync + ( + IAsyncEnumerableitems, + CancellationToken token + ); + + + + /// + /// Creates a progress report of type TProgress. This gives the derived class the opportunity to + /// implement a custom progress report that is specific to the extraction process. + /// + /// Progress of type TProgress + protected abstract TProgress CreateProgressReport(); + + + + /// + /// Increments the CurrentItemCount in a thread safe manner. + /// + /// + /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not + /// thread safe. This method ensures that CurrentItemCount is incremented safely + /// + protected void IncrementCurrentItemCount() + { + Interlocked.Increment(ref _currentItemCount); + } + + + + /// + /// Increments the CurrentItemCount in a thread safe manner. + /// + /// + /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not + /// thread safe. This method ensures that CurrentItemCount is incremented safely + /// + protected void IncrementCurrentSkippedItemCount() + { + Interlocked.Increment(ref _currentSkippedItemCount); + } + + } +} diff --git a/src/Wolfgang.Etl.Abstractions/TransformerBase.cs b/src/Wolfgang.Etl.Abstractions/TransformerBase.cs new file mode 100644 index 0000000..94e8f0a --- /dev/null +++ b/src/Wolfgang.Etl.Abstractions/TransformerBase.cs @@ -0,0 +1,315 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Threading; + + + +namespace Wolfgang.Etl.Abstractions +{ + public abstract class TransformerBase + : ITransformWithProgressAndCancellationAsync + { + + private int _reportingInterval = 1_000; + private int _maximumItemCount = int.MaxValue; + private int _skipItemCount; + private int _currentItemCount; + private int _currentSkippedItemCount; + + + /// + /// The number of milliseconds between progress updates. + /// + /// Value cannot be less than 1 + public int ReportingInterval + { + get => _reportingInterval; + set + { + if (value < 1) + { + throw new ArgumentOutOfRangeException(nameof(value), "Reporting interval must be greater than 0."); + } + _reportingInterval = value; + } + } + + + /// + /// The current number of items transformed so far. + /// + /// + /// It is the responsibility of the derived class to keep this value up to date as the + /// base class will have no way of knowing the correct value + /// + + [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] + public int CurrentItemCount + { + get => _currentItemCount; + protected set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value)); + } + _currentItemCount = value; + } + } + + + + /// + /// Gets the current number of records skipped + /// + public int CurrentSkippedItemCount + { + get => _currentSkippedItemCount; + protected set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), "value cannot be less than 0."); + } + } + } + + + + /// + /// The maximum number of items to transform. Once the transformer has reached this limit, + /// it should stop transforming and signal the end of the sequence. + /// + /// + /// This is useful for transforming a subset of data, especially when the source is large + /// or infinite or during development. + /// + /// The specified value is less than 1 + /// + /// + /// foreach (var item in items.Skip(SkipItemCount).Take(MaxItemCount)) + /// { + /// // Transformer each item and return it + /// } + /// + /// + + public int MaximumItemCount + { + get => _maximumItemCount; + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), "Maximum item count cannot be less than 0."); + } + _maximumItemCount = value; + } + } + + + + /// + /// The number of items to skip before transforming. + /// The transformer should skip the specified number of items before starting to yield results. + /// + /// + /// This is useful for transforming a subset of data, especially when the source is large + /// or infinite or during development. + /// + /// The specified value is less than 0 + /// + /// + /// foreach (var item in items.Skip(SkipItemCount).Take(MaxItemCount)) + /// { + /// // Transformer each item and return it + /// } + /// + /// + + public int SkipItemCount + { + get => _skipItemCount; + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), "Skip item count cannot be less than 0."); + } + _skipItemCount = value; + } + } + + + + /// + /// Asynchronously transforms data of type TSource to TDestination + /// + /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed + /// + /// IAsyncEnumerable<T> + /// The result may be an empty sequence if no data is available or if the transformation fails. + /// + public virtual IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items + ) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + return TransformWorkerAsync(items, CancellationToken.None); + } + + + + /// + /// Asynchronously transforms data of type TSource to TDestination + /// + /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed + /// A CancellationToken to observe while waiting for the task to complete + /// + /// IAsyncEnumerable<TDestination> - A list of 0 or more transformed items + /// + /// + /// + public virtual IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + CancellationToken token + ) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + return TransformWorkerAsync(items, token); + } + + + + /// + /// Asynchronously transforms data of type TSource to TDestination + /// + /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed + /// A provider for progress updates. + /// IAsyncEnumerable<T> The result may be an empty sequence if no data is available or if the transformation fails. + /// + /// The value of progress is null + public virtual IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + IProgress progress + ) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } + + using var timer = new Timer + ( + _ => progress.Report(CreateProgressReport()), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(ReportingInterval) + ); + + return TransformWorkerAsync(items, CancellationToken.None); + } + + + + /// + /// Asynchronously transforms data of type TSource to TDestination + /// + /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed + /// A provider for progress updates. + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// IAsyncEnumerable<T> The result may be an empty sequence if no data is available or if the transformation fails. + /// + /// + /// The transformer should be able to handle cancellation requests gracefully. + /// If the caller doesn't plan on cancelling the transformation, they can pass CancellationToken.None. + /// + /// The value of progress is null + public virtual IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + IProgress progress, + CancellationToken token + ) + { + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } + + using var timer = new Timer + ( + _ => progress.Report(CreateProgressReport()), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(ReportingInterval) + ); + + + return TransformWorkerAsync(items, token); + } + + + + protected abstract IAsyncEnumerable TransformWorkerAsync + ( + IAsyncEnumerableitems, + CancellationToken token + ); + + + + protected abstract TProgress CreateProgressReport(); + + + + /// + /// Increments the CurrentItemCount in a thread safe manner. + /// + /// + /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not + /// thread safe. This method ensures that CurrentItemCount is incremented safely + /// + protected void IncrementCurrentItemCount() + { + Interlocked.Increment(ref _currentItemCount); + } + + + + /// + /// Increments the CurrentItemCount in a thread safe manner. + /// + /// + /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not + /// thread safe. This method ensures that CurrentItemCount is incremented safely + /// + protected void IncrementCurrentSkippedItemCount() + { + Interlocked.Increment(ref _currentSkippedItemCount); + } + + } +} diff --git a/src/Wolfgang.Etl.Abstractions/Wolfgang.Etl.Abstractions.csproj b/src/Wolfgang.Etl.Abstractions/Wolfgang.Etl.Abstractions.csproj index ad5a308..6978c9d 100644 --- a/src/Wolfgang.Etl.Abstractions/Wolfgang.Etl.Abstractions.csproj +++ b/src/Wolfgang.Etl.Abstractions/Wolfgang.Etl.Abstractions.csproj @@ -1,8 +1,41 @@  - 0.1.0 - netstandard2.0 + + net462;net472;net48;net481; + netstandard2.0;netstandard2.1; + net8.0;net9.0 + + latest + 0.4.0 + False + $(AssemblyName) + Chris Wolfgang + Contains interfaces and base classes used to build ETL applications + 2025 + https://github.com/Chris-Wolfgang/ETL-Abstractions + README.md + https://github.com/Chris-Wolfgang/ETL-Abstractions + 1.0.0 + MIT + True + ETL.png + + + True + \ + + + True + \ + + + + + + + + diff --git a/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/ExtractorBaseTests.cs b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/ExtractorBaseTests.cs new file mode 100644 index 0000000..1a3c686 --- /dev/null +++ b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/ExtractorBaseTests.cs @@ -0,0 +1,328 @@ +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Wolfgang.Etl.Abstractions.Tests.Unit.Models; +using Xunit.Abstractions; + +namespace Wolfgang.Etl.Abstractions.Tests.Unit.BaseClassTests +{ + public class ExtractorBaseTests(ITestOutputHelper testOutputHelper) + { + + [Fact] + public async Task ExtractorBase_works_with_specified_versions_of_dotnet() + { + var sut = new FibonacciExtractorFromExtractorBase(); + + Assert.Equal(10, await sut.ExtractAsync().Take(10).CountAsync()); + } + + + + + [Fact] + public async Task ExtractAsync_returns_expected_results() + { + var expectedResults = new[] { 1, 1, 2, 3, 5, 8, 13, 21, 34, 55 }; + + var sut = new FibonacciExtractorFromExtractorBase(); + + var actualResults = await sut.ExtractAsync().ToListAsync(); + + Assert.Equal(expectedResults, actualResults); + } + + + + [Fact] + public async Task ExtractWithCancellationAsync_returns_expected_results() + { + var expectedResults = new[] { 1, 1, 2, 3, 5, 8, 13, 21, 34, 55 }; + + var sut = new FibonacciExtractorFromExtractorBase(); + + var actualResults = await sut.ExtractAsync(CancellationToken.None).ToListAsync(); + + Assert.Equal(expectedResults, actualResults); + } + + + + [Fact] + public async Task ExtractWithCancellationAsync_throws_exception_when_cancellation_is_requested() + { + var sut = new FibonacciExtractorFromExtractorBase(5); + + var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(5)); + + await Assert.ThrowsAsync(async () => await sut.ExtractAsync(cts.Token).ToListAsync(CancellationToken.None)); + } + + + + [Fact] + public async Task ExtractWithProgressAsync_when_passed_null_progress_throws_ArgumentNullException() + { + var sut = new FibonacciExtractorFromExtractorBase(); + + await Assert.ThrowsAsync(async () => await sut.ExtractAsync(null!).ToListAsync()); + } + + + + [Fact] + public async Task ExtractWithProgressAsync_returns_expected_results() + { + var expectedResults = new[] { 1, 1, 2, 3, 5, 8, 13, 21, 34, 55 }; + + var sut = new FibonacciExtractorFromExtractorBase(); + + var progress = new Progress(_ => { }); + + var actualResults = await sut.ExtractAsync(progress).ToListAsync(); + + Assert.Equal(expectedResults, actualResults); + } + + + + + [Fact(Skip = "Need to investigate why this test fails most of the time but occasionally passes")] + public async Task ExtractWithProgressAsync_reports_progress_expected_results() + { + var sut = new FibonacciExtractorFromExtractorBase(250) + { + ReportingInterval = 100 + }; + + var progressReportCount = 0; + var progress = new Progress(_ => + { + testOutputHelper.WriteLine("Progress reported."); + Interlocked.Increment(ref progressReportCount); + }); + + await sut.ExtractAsync(progress).ToListAsync(); + Assert.True(progressReportCount > 0, $"Value was expected to be greater than 0 but was {progressReportCount}"); + } + + + + [Fact] + public async Task ExtractWithProgressAsync_throws_exception_when_cancellation_is_requested() + { + var sut = new FibonacciExtractorFromExtractorBase(5); + + var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(5)); + + await Assert.ThrowsAsync(async () => await sut.ExtractAsync(cts.Token).ToListAsync(CancellationToken.None)); + } + + + + [Fact] + public async Task ExtractWithProgressAndCancellationAsync_throws_exception_when_cancellation_is_requested() + { + var sut = new FibonacciExtractorFromExtractorBase(5); + + var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(5)); + var progress = new Progress(_ => { }); + + await Assert.ThrowsAsync(async () => await sut.ExtractAsync(progress, cts.Token).ToListAsync(CancellationToken.None)); + } + + + + [Fact] + public async Task ExtractWithProgressAndCancellationAsync_when_passed_null_progress_throws_ArgumentNullException() + { + var sut = new FibonacciExtractorFromExtractorBase(); + + var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(5)); + + await Assert.ThrowsAsync(async () => await sut.ExtractAsync(null!, cts.Token).ToListAsync(CancellationToken.None)); + } + + + + [Fact] + public async Task ExtractWithProgressAndCancellationAsync_returns_expected_results() + { + var expectedResults = new[] { 1, 1, 2, 3, 5, 8, 13, 21, 34, 55 }; + + var sut = new FibonacciExtractorFromExtractorBase(); + + var progress = new Progress(_ => { }); + + var actualResults = await sut.ExtractAsync(progress, CancellationToken.None).ToListAsync(); + + Assert.Equal(expectedResults, actualResults); + } + + + + [Fact(Skip = "Need to investigate why this test fails most of the time but occasionally passes")] + public async Task ExtractWithProgressAndCancellationAsync_reports_progress_expected_results() + { + var sut = new FibonacciExtractorFromExtractorBase(250) + { + ReportingInterval = 100 + }; + + var progressReportCount = 0; + var progress = new Progress(_ => + { + testOutputHelper.WriteLine("Progress reported."); + Interlocked.Increment(ref progressReportCount); + }); + + await sut.ExtractAsync(progress, CancellationToken.None).ToListAsync(); + + Assert.True(progressReportCount > 0, $"Value was expected to be greater than 0 but was {progressReportCount}"); + } + + + + [Fact] + public void CurrentItemCount_when_assigned_a_value_less_than_0_throws_ArgumentOutOfRangeException() + { + + var sut = new FibonacciExtractorFromExtractorBase(); + + Assert.Throws(() => sut.TestSettingCurrentItemCount( -1)); + } + + + + [Fact] + public void CurrentItemCount_when_assigned_a_valid_value_stores_the_value() + { + + var sut = new FibonacciExtractorFromExtractorBase(); + + sut.TestSettingCurrentItemCount(10); + + Assert.Equal(10, sut.CurrentItemCount); + } + + + + [Fact] + public void ReportingInterval_when_assigned_a_value_less_than_0_throws_ArgumentOutOfRangeException() + { + + var sut = new FibonacciExtractorFromExtractorBase(); + Assert.Throws(() => sut.ReportingInterval = -1); + } + + + [Fact] + public void ReportingInterval_when_assigned_a_valid_value_stores_the_value() + { + + var sut = new FibonacciExtractorFromExtractorBase() + { + ReportingInterval = 10 + }; + Assert.Equal(10, sut.ReportingInterval); + } + + + + [Fact] + public void MaximumItemCount_when_assigned_a_value_less_than_0_throws_ArgumentOutOfRangeException() + { + + var sut = new FibonacciExtractorFromExtractorBase(); + Assert.Throws(() => sut.MaximumItemCount = -1); + } + + + [Fact] + public void MaximumItemCount_when_assigned_a_valid_value_stores_the_value() + { + + var sut = new FibonacciExtractorFromExtractorBase() + { + MaximumItemCount = 10 + }; + Assert.Equal(10, sut.MaximumItemCount); + } + + + [Fact] + public void SkipItemCount_when_assigned_a_value_less_than_0_throws_ArgumentOutOfRangeException() + { + + var sut = new FibonacciExtractorFromExtractorBase(); + Assert.Throws(() => sut.SkipItemCount = -1); + } + + + [Fact] + public void SkipItemCount_when_assigned_a_valid_value_stores_the_value() + { + + var sut = new FibonacciExtractorFromExtractorBase() + { + SkipItemCount = 10 + }; + Assert.Equal(10, sut.SkipItemCount); + } + } + + + + + [ExcludeFromCodeCoverage] + internal class FibonacciExtractorFromExtractorBase(int delay) : ExtractorBase + { + public FibonacciExtractorFromExtractorBase() : this(0) { } + + + protected override async IAsyncEnumerable ExtractWorkerAsync + ( + [EnumeratorCancellation] CancellationToken token + ) + { + var stopwatch = Stopwatch.StartNew(); + + var current = 1; + var previous = 0; + for (var x = 0; x < 10; ++x) + { + await Task.Delay(delay, token); // Simulate asynchronous operation + if (token.IsCancellationRequested) + { + throw new TaskCanceledException("Extraction was cancelled."); + } + yield return current; + ++CurrentItemCount; + + var temp = current; + current += previous; + previous = temp; + } + + stopwatch.Stop(); + } + + + + protected override EtlProgress CreateProgressReport() + { + return new EtlProgress(CurrentItemCount); + } + + + + /// + /// Used for testing purposes to set the CurrentItemCount property. + /// + /// + public void TestSettingCurrentItemCount(int value) + { + CurrentItemCount = value; + } + } +} diff --git a/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/LoaderBaseTests.cs b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/LoaderBaseTests.cs new file mode 100644 index 0000000..b78d043 --- /dev/null +++ b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/LoaderBaseTests.cs @@ -0,0 +1,419 @@ +using System.Diagnostics.CodeAnalysis; +using Wolfgang.Etl.Abstractions.Tests.Unit.Models; +using Xunit.Abstractions; + +namespace Wolfgang.Etl.Abstractions.Tests.Unit.BaseClassTests +{ + public class LoaderBaseTests(ITestOutputHelper testOutputHelper) + { + + + [Fact] + public async Task LoaderBase_works_with_specified_versions_of_dotnet() + { + var actualItems = new List(); + var sut = new ConsoleLoaderFromBase(actualItems); + + var expected = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + var items = expected.ToAsyncEnumerable(); + + await sut.LoadAsync(items); + + Assert.Equal(expected, actualItems.ToArray()); + } + + + + [Fact] + public async Task LoadAsync_when_passed_null_items_throws_ArgumentNullException() + { + var actualResults = new List(); + + var sut = new ConsoleLoaderFromBase(actualResults); + + await Assert.ThrowsAsync(() => sut.LoadAsync(null!)); + } + + + + [Fact] + public async Task LoadAsync_returns_expected_results() + { + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + + var actualResults = new List(); + + var sut = new ConsoleLoaderFromBase(actualResults); + + await sut.LoadAsync(expectedResults.ToAsyncEnumerable()); + + Assert.Equal(expectedResults, actualResults); + } + + + + [Fact] + public async Task LoadWithCancellationAsync_when_passed_null_items_throws_ArgumentNullException() + { + var actualResults = new List(); + var sut = new ConsoleLoaderFromBase(actualResults); + + await Assert.ThrowsAsync(() => sut.LoadAsync(null!, CancellationToken.None)); + } + + + + [Fact] + public async Task LoadWithCancellationAsync_returns_expected_results() + { + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + + var actualResults = new List(); + var sut = new ConsoleLoaderFromBase(actualResults); + + await sut.LoadAsync(expectedResults.ToAsyncEnumerable(), CancellationToken.None); + + Assert.Equal(expectedResults, actualResults); + } + + + + [Fact(Skip = "Need to determine why cancellation request isn't being honored.")] + public async Task LoadWithCancellationAsync_throws_exception_when_cancellation_is_requested() + { + var buffer = new List(); + const int delay = 100; // Delay in milliseconds + var sut = new ConsoleLoaderFromBase(buffer, delay); + + using var cts = new CancellationTokenSource(); + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + + // Start the load operation + var task = sut.LoadAsync(expectedResults.ToAsyncEnumerable(), cts.Token); + + try + { + cts.Cancel(); + + await task; + Assert.Fail("Expected OperationCanceledException was not thrown."); + } + catch (OperationCanceledException) + { + // Expected exception + } + } + + + + [Fact] + public async Task LoadWithProgressAsync_when_passed_null_items_throws_ArgumentNullException() + { + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + + var actualResults = new List(); + + var sut = new ConsoleLoaderFromBase(actualResults); + + var progress = new Progress(_ => { }); + + await Assert.ThrowsAsync(async () => + await sut.LoadAsync(null!, progress)); + } + + + + [Fact] + public async Task LoadWithProgressAsync_when_passed_null_progress_throws_ArgumentNullException() + { + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + + var actualResults = new List(); + + var sut = new ConsoleLoaderFromBase(actualResults); + await Assert.ThrowsAsync(async () => + await sut.LoadAsync(expectedResults.ToAsyncEnumerable(), null!)); + } + + + + [Fact] + public async Task LoadWithProgressAsync_returns_expected_results() + { + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + + var actualResults = new List(); + + var sut = new ConsoleLoaderFromBase(actualResults); + var progress = new Progress(_ => { }); + + await sut.LoadAsync(expectedResults.ToAsyncEnumerable(), progress); + + Assert.Equal(expectedResults, actualResults); + } + + + + [Fact(Skip = "Need to investigate why this test fails most of the time but occasionally passes")] + public async Task LoadWithProgressAsync_reports_progress_expected_results() + { + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + + var actualResults = new List(); + + var sut = new ConsoleLoaderFromBase(actualResults) + { + ReportingInterval = 100 + }; + + var progressReportCount = 0; + var progress = new Progress(_ => + { + testOutputHelper.WriteLine("Progress reported."); + Interlocked.Increment(ref progressReportCount); + }); + + await sut.LoadAsync(expectedResults.ToAsyncEnumerable(), progress); + Assert.True(progressReportCount > 0, $"Value was expected to be greater than 0 but was {progressReportCount}"); + } + + + + [Fact(Skip = "Need to determine why cancellation request isn't being honored.")] + public async Task LoadWithProgressAsync_throws_exception_when_cancellation_is_requested() + { + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + var actualResults = new List(); + + var sut = new ConsoleLoaderFromBase(actualResults, 1000); + + var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + + await Assert.ThrowsAsync(async () => + await sut.LoadAsync(expectedResults.ToAsyncEnumerable(), cts.Token)); + } + + + + [Fact] + public async Task LoadWithProgressAndCancellationAsync_when_passed_null_items_throws_ArgumentNullException() + { + var actualResults = new List(); + + var sut = new ConsoleLoaderFromBase(actualResults); + + var progressReportCount = 0; + var progress = new Progress(_ => + { + testOutputHelper.WriteLine("Progress reported."); + Interlocked.Increment(ref progressReportCount); + }); + + await Assert.ThrowsAsync(async () => + await sut.LoadAsync(null!, progress, CancellationToken.None)); + } + + + + [Fact] + public async Task LoadWithProgressAndCancellationAsync_when_passed_null_progress_throws_ArgumentNullException() + { + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + var actualResults = new List(); + + var sut = new ConsoleLoaderFromBase(actualResults); + + await Assert.ThrowsAsync(async () => + await sut.LoadAsync(expectedResults.ToAsyncEnumerable(), null!, CancellationToken.None)); + } + + + + [Fact] + public async Task LoadWithProgressAndCancellationAsync_returns_expected_results() + { + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + var actualResults = new List(); + + var sut = new ConsoleLoaderFromBase(actualResults); + var progress = new Progress(_ => { }); + + await sut.LoadAsync(expectedResults.ToAsyncEnumerable(), progress, CancellationToken.None); + + Assert.Equal(expectedResults, actualResults); + } + + + + [Fact(Skip = "Need to investigate why this test fails most of the time but occasionally passes")] + public async Task LoadWithProgressAndCancellationAsync_reports_progress_expected_results() + { + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + var actualResults = new List(); + + var sut = new ConsoleLoaderFromBase(actualResults, 1000) + { + ReportingInterval = 100 + }; + + var progressReportCount = 0; + var progress = new Progress(_ => + { + testOutputHelper.WriteLine("Progress reported."); + Interlocked.Increment(ref progressReportCount); + }); + + await sut.LoadAsync(expectedResults.ToAsyncEnumerable(), progress, CancellationToken.None); + + Assert.True(progressReportCount > 0, $"Value was expected to be greater than 0 but was {progressReportCount}"); + } + + + + [Fact] + public void CurrentItemCount_when_assigned_a_value_less_than_0_throws_ArgumentOutOfRangeException() + { + + var sut = new ConsoleLoaderFromBase(new List()); + Assert.Throws(() => sut.TestSettingCurrentItemCount(-1)); + } + + + + [Fact] + public void CurrentItemCount_when_assigned_a_valid_value_stores_the_value() + { + + var sut = new ConsoleLoaderFromBase(new List()); + sut.TestSettingCurrentItemCount(10); + + Assert.Equal(10, sut.CurrentItemCount); + } + + + + [Fact] + public void ReportingInterval_when_assigned_a_value_less_than_0_throws_ArgumentOutOfRangeException() + { + + var sut = new ConsoleLoaderFromBase(new List()); + Assert.Throws(() => sut.ReportingInterval = -1); + } + + + + [Fact] + public void ReportingInterval_when_assigned_a_valid_value_stores_the_value() + { + + var sut = new ConsoleLoaderFromBase(new List()) + { + ReportingInterval = 10 + }; + Assert.Equal(10, sut.ReportingInterval); + } + + + + [Fact] + public void MaximumItemCount_when_assigned_a_value_less_than_1_throws_ArgumentOutOfRangeException() + { + + var sut = new ConsoleLoaderFromBase(new List()); + + Assert.Throws(() => sut.MaximumItemCount = -1); + } + + + + [Fact] + public void MaximumItemCount_when_assigned_a_valid_value_stores_the_value() + { + + var sut = new ConsoleLoaderFromBase(new List()) + { + MaximumItemCount = 10 + }; + Assert.Equal(10, sut.MaximumItemCount); + } + + + + [Fact] + public void SkipItemCount_when_assigned_a_value_less_than_0_throws_ArgumentOutOfRangeException() + { + + var sut = new ConsoleLoaderFromBase(new List()); + Assert.Throws(() => sut.SkipItemCount = -1); + } + + + + [Fact] + public void SkipItemCount_when_assigned_a_valid_value_stores_the_value() + { + + var sut = new ConsoleLoaderFromBase(new List()) + { + SkipItemCount = 10 + }; + Assert.Equal(10, sut.SkipItemCount); + } + } + + + + [ExcludeFromCodeCoverage] + internal class ConsoleLoaderFromBase : LoaderBase + { + + private readonly List _buffer; + private readonly int _delay; + + public ConsoleLoaderFromBase(List buffer, int delay = 0) + { + _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); + if (delay < 0) + throw new ArgumentOutOfRangeException(nameof(delay), "Delay must be non-negative."); + _delay = delay; + } + + + protected override async Task LoadWorkerAsync(IAsyncEnumerable items, CancellationToken token) + { + Console.WriteLine($"Delay {_delay}"); + + await foreach (var item in items) + { + Console.WriteLine($"Waiting {_delay}ms"); + if (token.IsCancellationRequested) + { + Console.WriteLine("Operation was cancelled."); + throw new TaskCanceledException("The load operation was cancelled."); + } + Console.WriteLine($"Loading item: {item}"); + await Task.Delay(_delay, token); + _buffer.Add(item); + } + } + + + + protected override EtlProgress CreateProgressReport() + { + return new EtlProgress(CurrentItemCount); + } + + + + /// + /// Used for testing purposes to set the CurrentItemCount property. + /// + /// + public void TestSettingCurrentItemCount(int value) + { + CurrentItemCount = value; + } + + } +} diff --git a/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/TransformerBaseTests.cs b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/TransformerBaseTests.cs new file mode 100644 index 0000000..1eff34c --- /dev/null +++ b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/TransformerBaseTests.cs @@ -0,0 +1,411 @@ +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Wolfgang.Etl.Abstractions.Tests.Unit.Models; + +namespace Wolfgang.Etl.Abstractions.Tests.Unit.BaseClassTests +{ + public class TransformerBaseTests + { + + + [Fact] + public async Task TransformerBase_works_with_specified_versions_of_dotnet() + { + var numbers = new[] { 1, 1, 2, 3, 5, 8, 13, 21, 34, 55 }; + var sut = new IntToStringTransformerFromTransformerBase(); + + var actual = await sut.TransformAsync(numbers.ToAsyncEnumerable()).ToListAsync(); + var expected = numbers.Select(n => n.ToString()).ToList(); + + Assert.Equal(expected, actual); + } + + + + // Transform tests + + [Fact] + public async Task TransformAsync_when_passed_null_items_throws_ArgumentNullException() + { + var sut = new IntToStringTransformerFromTransformerBase(); + + await Assert.ThrowsAsync (async ()=> await sut.TransformAsync(null!).ToListAsync()); + } + + + + [Fact] + public async Task TransformAsync_returns_expected_values() + { + var sut = new IntToStringTransformerFromTransformerBase(); + + var items = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }.ToAsyncEnumerable(); + + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + var actualResults = await sut.TransformAsync(items).ToListAsync(); + + Assert.Equal(expectedResults, actualResults); + } + + + + // Transform With Cancellation tests + + + [Fact] + public async Task TransformWithCancellationAsync_when_passed_null_items_throws_ArgumentNullException() + { + var sut = new IntToStringTransformerFromTransformerBase(); + + await Assert.ThrowsAsync(async () => + await sut.TransformAsync(null!, CancellationToken.None).ToListAsync()); + } + + + + [Fact] + public async Task TransformWithCancellationAsync_returns_expected_values() + { + var sut = new IntToStringTransformerFromTransformerBase(); + + var items = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }.ToAsyncEnumerable(); + + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + var actualResults = await sut.TransformAsync(items, CancellationToken.None).ToListAsync(); + + Assert.Equal(expectedResults, actualResults); + } + + + + [Fact] + public async Task TransformWithCancellationAsync_when_cancelled_throws_OperationCancelledException() + { + var sut = new IntToStringTransformerFromTransformerBase(1000); + + var items = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }.ToAsyncEnumerable(); + var cts = new CancellationTokenSource(); + + var task = sut.TransformAsync(items, cts.Token); + + try + { + cts.Cancel(); + await task.ToListAsync(); + Assert.Fail("OperationCanceledException was expected but not thrown"); + } + catch (OperationCanceledException) + { + // Expected exception was thrown + } + } + + + + + // Transform With Progress tests + + + [Fact] + public async Task TransformWithProgressAsync_when_passed_null_items_throws_ArgumentNullException() + { + var sut = new IntToStringTransformerFromTransformerBase(); + + var progress = new Progress(_ => { }); + + var ex = await Assert.ThrowsAsync(async () => + await sut.TransformAsync(null!, progress).ToListAsync()); + Assert.Equal("items", ex.ParamName); + } + + + + [Fact] + public async Task TransformWithProgressAsync_when_passed_null_progress_throws_ArgumentNullException() + { + var sut = new IntToStringTransformerFromTransformerBase(); + + var items = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }.ToAsyncEnumerable(); + + var ex = await Assert.ThrowsAsync(async () => + await sut.TransformAsync(items, null!).ToListAsync()); + Assert.Equal("progress", ex.ParamName); + } + + + + [Fact] + public async Task TransformWithProgressAsync_returns_expected_values() + { + var sut = new IntToStringTransformerFromTransformerBase(); + + var items = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }.ToAsyncEnumerable(); + var progress = new Progress(_ => { }); + + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + var actualResults = await sut.TransformAsync(items, progress).ToListAsync(); + + Assert.Equal(expectedResults, actualResults); + } + + + + [Fact(Skip = "Test succeeds on its own, but fails when run with other tests. Need to determine what is going on and get it working")] + public async Task TransformWithProgressAsync_reports_progress() + { + var sut = new IntToStringTransformerFromTransformerBase(100) + { + ReportingInterval = 100 // Report progress every second + }; + + var progressReported = false; + + var items = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }.ToAsyncEnumerable(); + var progress = new Progress(_ => progressReported = true); + + await sut.TransformAsync(items, progress).ToListAsync(); + + Assert.True(progressReported, "Progress was not reported"); + } + + + + // Transform With Progress And Cancellation tests + + + [Fact] + public async Task TransformWithProgressAndCancellationAsync_when_passed_null_items_throws_ArgumentNullException() + { + var sut = new IntToStringTransformerFromTransformerBase(); + + var progress = new Progress(_ => { }); + + var ex = await Assert.ThrowsAsync(async () => + await sut.TransformAsync(null!, progress, CancellationToken.None).ToListAsync()); + Assert.Equal("items", ex.ParamName); + } + + + + [Fact] + public async Task TransformWithProgressAndCancellationAsync_when_passed_null_progress_throws_ArgumentNullException() + { + var sut = new IntToStringTransformerFromTransformerBase(); + + var items = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }.ToAsyncEnumerable(); + + var ex = await Assert.ThrowsAsync(async () => + await sut.TransformAsync(items, null!, CancellationToken.None).ToListAsync()); + Assert.Equal("progress", ex.ParamName); + } + + + [Fact] + public async Task TransformWithProgressAndCancellationAsync_returns_expected_values() + { + var sut = new IntToStringTransformerFromTransformerBase(); + + var items = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }.ToAsyncEnumerable(); + var progress = new Progress(_ => { }); + + var expectedResults = new[] { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" }; + var actualResults = await sut.TransformAsync(items, progress, CancellationToken.None).ToListAsync(); + + Assert.Equal(expectedResults, actualResults); + } + + + + [Fact(Skip = "Test succeeds on its own, but fails when run with other tests. Need to determine what is going on and get it working")] + public async Task TransformWithProgressAndCancellationAsync_reports_progress() + { + var sut = new IntToStringTransformerFromTransformerBase + { + ReportingInterval = 1000 // Report progress every second + }; + + var progressReported = false; + + var items = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }.ToAsyncEnumerable(); + var progress = new Progress(_ => + { + progressReported = true; + }); + + await sut.TransformAsync(items, progress, CancellationToken.None).ToListAsync(); + + Assert.True(progressReported, "Progress was not reported"); + } + + + + + [Fact] + public async Task TransformWithProgressAndCancellationAsync_when_cancelled_throws_OperationCancelledException() + { + var sut = new IntToStringTransformerFromTransformerBase(100); + + var items = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }.ToAsyncEnumerable(); + var cts = new CancellationTokenSource(); + + var progress = new Progress(); + var task = sut.TransformAsync(items, progress, cts.Token); + + try + { + cts.Cancel(); + await task.ToListAsync(); + Assert.Fail("OperationCanceledException was expected but not thrown"); + } + catch (OperationCanceledException) + { + // Expected exception was thrown + } + } + + + + [Fact] + public void CurrentItemCount_when_assigned_a_value_less_than_0_throws_ArgumentOutOfRangeException() + { + + var sut = new IntToStringTransformerFromTransformerBase(); + Assert.Throws(() => sut.TestSettingCurrentItemCount(-1)); + } + + + + [Fact] + public void CurrentItemCount_when_assigned_a_valid_value_stores_the_value() + { + + var sut = new IntToStringTransformerFromTransformerBase(); + sut.TestSettingCurrentItemCount(10); + + Assert.Equal(10, sut.CurrentItemCount); + } + + + + [Fact] + public void ReportingInterval_when_assigned_a_value_less_than_0_throws_ArgumentOutOfRangeException() + { + + var sut = new IntToStringTransformerFromTransformerBase(); + Assert.Throws(() => sut.ReportingInterval = -1); + } + + + + [Fact] + public void ReportingInterval_when_assigned_a_valid_value_stores_the_value() + { + + var sut = new IntToStringTransformerFromTransformerBase + { + ReportingInterval = 10 + }; + Assert.Equal(10, sut.ReportingInterval); + } + + + + [Fact] + public void MaximumItemCount_when_assigned_a_value_less_than_1_throws_ArgumentOutOfRangeException() + { + + var sut = new IntToStringTransformerFromTransformerBase(); + + Assert.Throws(() => sut.MaximumItemCount = -1); + } + + + + [Fact] + public void MaximumItemCount_when_assigned_a_valid_value_stores_the_value() + { + + var sut = new IntToStringTransformerFromTransformerBase + { + MaximumItemCount = 10 + }; + Assert.Equal(10, sut.MaximumItemCount); + } + + + + [Fact] + public void SkipItemCount_when_assigned_a_value_less_than_0_throws_ArgumentOutOfRangeException() + { + + var sut = new IntToStringTransformerFromTransformerBase(); + Assert.Throws(() => sut.SkipItemCount = -1); + } + + + + [Fact] + public void SkipItemCount_when_assigned_a_valid_value_stores_the_value() + { + + var sut = new IntToStringTransformerFromTransformerBase + { + SkipItemCount = 10 + }; + Assert.Equal(10, sut.SkipItemCount); + } + + + + [ExcludeFromCodeCoverage] + internal class IntToStringTransformerFromTransformerBase : TransformerBase + { + private readonly int _delay; + + public IntToStringTransformerFromTransformerBase(int delay = 0) + { + if (delay < 0) throw new ArgumentOutOfRangeException(nameof(delay)); + _delay = delay; + } + + + + protected override async IAsyncEnumerable TransformWorkerAsync + ( + IAsyncEnumerable items, + [EnumeratorCancellation] CancellationToken token + ) + { + await foreach (var item in items.WithCancellation(token)) + { + await Task.Delay(_delay, token); // Simulate some delay in processing + if (token.IsCancellationRequested) + { + throw new OperationCanceledException(token); + } + yield return item.ToString(); + ++CurrentItemCount; + } + } + + + + protected override EtlProgress CreateProgressReport() + { + return new EtlProgress(CurrentItemCount); + } + + + + /// + /// Used for testing purposes to set the CurrentItemCount property. + /// + /// + public void TestSettingCurrentItemCount(int value) + { + CurrentItemCount = value; + } + + } + } +} diff --git a/tests/Wolfgang.Etl.Abstractions.Tests.Unit/Models/EtlProgress.cs b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/Models/EtlProgress.cs new file mode 100644 index 0000000..d4ba6d5 --- /dev/null +++ b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/Models/EtlProgress.cs @@ -0,0 +1,19 @@ +namespace Wolfgang.Etl.Abstractions.Tests.Unit.Models; + +internal record EtlProgress +{ + public EtlProgress(int currentCount) + { + if (currentCount < 0) + { + throw new ArgumentOutOfRangeException(nameof(currentCount), "Current count cannot be less than 0."); + } + + CurrentCount = currentCount; + } + + + + public int CurrentCount { get; } + +} \ No newline at end of file diff --git a/tests/Wolfgang.Etl.Abstractions.Tests.Unit/Wolfgang.Etl.Abstractions.Tests.Unit.csproj b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/Wolfgang.Etl.Abstractions.Tests.Unit.csproj new file mode 100644 index 0000000..11fb908 --- /dev/null +++ b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/Wolfgang.Etl.Abstractions.Tests.Unit.csproj @@ -0,0 +1,45 @@ + + + + + net462;net472;net48;net481; + net50;net6.0;net7.0; + net8.0;net9.0 + + 1.0.0 + latest + + enable + enable + + false + true + + + + + [17.13.0] + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + [2.8.2] + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + +