diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..9ffe9ab --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,17 @@ +# This CODEOWNERS file designates code owners for different parts of the repository. +# Update usernames or teams as appropriate for your project. + +# Set the default owner for everything in the repository +* @Chris-Wolfgang + +# Example: Assign a team or user to a specific directory +# /src/ @Chris-Wolfgang + +# Example: Assign a different owner to documentation files +# /docs/ @Chris-Wolfgang + +# Example: Assign an owner to GitHub Actions workflows +# /.github/workflows/ @Chris-Wolfgang + +# Example: Assign an owner to the .github folder to protect the CODEOWNERS file +/.github/ @Chris-Wolfgang diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.yaml b/.github/ISSUE_TEMPLATE/BUG_REPORT.yaml new file mode 100644 index 0000000..2076bc8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.yaml @@ -0,0 +1,79 @@ +name: "🐞 Bug report" +description: "File a bug report to help us improve" +title: "[Bug]: " +labels: [bug, needs-triage] +assignees: [] +body: + - type: markdown + attributes: + value: | + ## Thanks for reporting a bug! + + Please fill out the information below to help us resolve the issue as quickly as possible. + + - type: textarea + id: description + attributes: + label: "Describe the bug" + description: "A clear and concise description of what the bug is." + placeholder: "Bug details go here..." + validations: + required: true + + - type: textarea + id: steps-to-reproduce + attributes: + label: "Steps to reproduce" + description: "How can we reproduce the behavior?" + placeholder: | + 1. Go to '...' + 2. Click on '....' + 3. Scroll down to '....' + 4. See error + validations: + required: true + + - type: textarea + id: expected-behavior + attributes: + label: "Expected behavior" + description: "What did you expect to happen?" + placeholder: "It should..." + validations: + required: true + + - type: textarea + id: actual-behavior + attributes: + label: "Actual behavior" + description: "What actually happened?" + placeholder: "Instead, it..." + validations: + required: true + + - type: input + id: environment + attributes: + label: "Environment" + description: "Please provide information about your environment (OS, browser, version, etc.)" + placeholder: "e.g. Windows 11, Chrome 124.0.1" + validations: + required: false + + - type: textarea + id: screenshots + attributes: + label: "Screenshots" + description: "If applicable, add screenshots to help explain your problem." + placeholder: "Drag & drop images or paste them here." + validations: + required: false + + - type: textarea + id: additional-context + attributes: + label: "Additional context" + description: "Add any other context about the problem here." + placeholder: "Anything else?" + validations: + required: false diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9418e6b..885018f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,14 +1,46 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file - -version: 2 -updates: version: 2 updates: + - package-ecosystem: "nuget" + directory: "/" # Root - for solution-level dependencies + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "dotnet" + - package-ecosystem: "nuget" directory: "/src" schedule: interval: "weekly" - open-pull-requests-limit: 5 + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "dotnet" + + - package-ecosystem: "nuget" + directory: "/tests" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "dotnet" + + - package-ecosystem: "nuget" + directory: "/benchmarks" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "dotnet" + + - package-ecosystem: "nuget" + directory: "/examples" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "dotnet" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..7f07299 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,40 @@ +## Description + + + +Fixes/Complete # (issue) + +## Type of change + +Please delete options that are not relevant. + +- [ ] Bug fix +- [ ] New feature +- [ ] Breaking change +- [ ] Documentation update +- [ ] Refactor + +## How Has This Been Tested? + + + +- [ ] Test A +- [ ] Test B + +## Checklist + +- [ ] My code follows the style guidelines of this project +- [ ] I have performed a self-review of my own code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] My changes generate no new warnings +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes + +## Screenshots (if applicable) + + + +## Additional context + + diff --git a/.github/workflows/create-labels.yaml b/.github/workflows/create-labels.yaml new file mode 100644 index 0000000..ce661f5 --- /dev/null +++ b/.github/workflows/create-labels.yaml @@ -0,0 +1,86 @@ +name: Create Dependabot Security and Dependencies Labels +on: + workflow_dispatch: + +jobs: + create-labels: + permissions: + issues: write + runs-on: ubuntu-latest + steps: + - name: Create "dependabot - security" label + uses: actions/github-script@v6 + with: + script: | + try { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: "dependabot - security", + color: "b60205" + }); + } catch (error) { + if (error.status === 422 && error.response?.data?.errors?.[0]?.code === 'already_exists') { + console.log('Label "dependabot - security" already exists, skipping creation'); + } else { + console.error('Failed to create label "dependabot - security":', error.message); + throw error; + } + } + - name: Create "dependabot-dependencies" label + uses: actions/github-script@v6 + with: + script: | + try { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: "dependabot-dependencies", + color: "d93f0b" + }); + } catch (error) { + if (error.status === 422 && error.response?.data?.errors?.[0]?.code === 'already_exists') { + console.log('Label "dependabot-dependencies" already exists, skipping creation'); + } else { + console.error('Failed to create label "dependabot-dependencies":', error.message); + throw error; + } + } + - name: Create "dependencies" label + uses: actions/github-script@v6 + with: + script: | + try { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: "dependencies", + color: "0366d6" + }); + } catch (error) { + if (error.status === 422 && error.response?.data?.errors?.[0]?.code === 'already_exists') { + console.log('Label "dependencies" already exists, skipping creation'); + } else { + console.error('Failed to create label "dependencies":', error.message); + throw error; + } + } + - name: Create "dotnet" label + uses: actions/github-script@v6 + with: + script: | + try { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: "dotnet", + color: "512bd4" + }); + } catch (error) { + if (error.status === 422 && error.response?.data?.errors?.[0]?.code === 'already_exists') { + console.log('Label "dotnet" already exists, skipping creation'); + } else { + console.error('Failed to create label "dotnet":', error.message); + throw error; + } + } diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 0453213..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Build and Publish .NET Package - -permissions: - contents: read - pull-requests: write - -on: - push: - branches: - - main - #tags: - #- 'v*' # e.g., v1.0.0 - -jobs: - build-and-publish: - runs-on: windows-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up .NET - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '9.0.x' - - - name: Restore dependencies - shell: pwsh - run: | - Get-ChildItem -Path "src" -Recurse -Filter *.csproj | ForEach-Object { - dotnet restore $_.FullName - } - - - name: Build projects - shell: pwsh - run: | - Get-ChildItem -Path "src" -Recurse -Filter *.csproj | ForEach-Object { - dotnet build $_.FullName --configuration Release --no-restore - } - - - name: Run unit tests - shell: pwsh - run: | - Get-ChildItem -Path "src" -Recurse -Filter *Tests.Unit.csproj | ForEach-Object { - dotnet test $_.FullName --configuration Release --no-build --verbosity normal - } - - - name: Pack projects - shell: pwsh - run: | - mkdir ./nupkg - Get-ChildItem -Path "src" -Recurse -Filter *.csproj | Where-Object { $_.Name -notlike "*Tests.csproj" } | ForEach-Object { - dotnet pack $_.FullName --configuration Release --no-build --output ./nupkg - } - - - name: Push packages to NuGet - shell: pwsh - run: | - Get-ChildItem -Path "./nupkg" -Filter *.nupkg | ForEach-Object { - dotnet nuget push $_.FullName --api-key ${{ secrets.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json - } diff --git a/.github/workflows/docfx.yaml b/.github/workflows/docfx.yaml new file mode 100644 index 0000000..a935947 --- /dev/null +++ b/.github/workflows/docfx.yaml @@ -0,0 +1,54 @@ +name: Deploy DocFX Pages + +on: + push: + branches: + - main # Your primary branch + +jobs: + build: + runs-on: ubuntu-latest + + permissions: + contents: read # Allow read access for checkout + pages: write # Allow write access for Pages deployment + id-token: write # Allow writing of ID tokens for deployment + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + + - name: Install DocFX + run: dotnet tool update docfx --global + + - name: Build DocFx Metadata + run: docfx metadata + working-directory: docfx_project + + - name: Build Docs + run: docfx build + working-directory: docfx_project + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docfx_project/_site # The path to the folder to upload + + deploy: + needs: build + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 0000000..a5d9e1c --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,199 @@ +# This workflow runs on pull requests to validate code quality, run tests, and perform security scans +# before merging into main or other protected branches + +name: PR Checks + +permissions: + contents: read + +on: + pull_request: + branches: + - main + +jobs: + build-and-test: + runs-on: ${{ matrix.os }} + if: github.repository != 'Chris-Wolfgang/repo-template' # Only run in child repos otherwise this will fail because the template does not have any projects + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 5.0.x + 6.0.x + 7.0.x + 8.0.x + 9.0.x + + - name: Restore dependencies + run: dotnet restore + + # Linux/macOS: Build non-.NET Framework targets only + - name: Build projects - Linux/macOS + if: runner.os != 'Windows' + shell: bash + run: | + # Build main library (excluding .NET Framework) + for fw in netstandard2.0 netstandard2.1 netcoreapp3.1 net5.0 net6.0 net7.0 net8.0 net9.0; do + echo "Building Wolfgang.Etl.Abstractions for $fw" + dotnet build src/Wolfgang.Etl.Abstractions/Wolfgang.Etl.Abstractions.csproj \ + --no-restore \ + --configuration Release \ + --framework "$fw" + done + + # Build test project (excluding .NET Framework) + for fw in netcoreapp3.1 net5.0 net6.0 net7.0 net8.0 net9.0 net10.0; do + echo "Building tests for $fw" + dotnet build tests/Wolfgang.Etl.Abstractions.Tests.Unit/Wolfgang.Etl.Abstractions.Tests.Unit.csproj \ + --no-restore \ + --configuration Release \ + --framework "$fw" + done + + # Build .NET 8.0 examples + for proj in examples/Net8.0/*/*.csproj; do + [ -f "$proj" ] && dotnet build "$proj" --no-restore --configuration Release + done + + # Windows: Build all target frameworks including .NET Framework + - name: Build solution - Windows + if: runner.os == 'Windows' + run: dotnet build --no-restore --configuration Release + + # Linux/macOS: Run tests without coverage (faster) + - name: Run tests - Linux/macOS + if: runner.os != 'Windows' + shell: bash + run: | + dotnet test tests/Wolfgang.Etl.Abstractions.Tests.Unit/Wolfgang.Etl.Abstractions.Tests.Unit.csproj \ + --configuration Release \ + --framework net8.0 \ + --no-restore \ + --logger "console;verbosity=normal" + + # Windows: Run tests with coverage collection + - name: Run tests with coverage - Windows + if: runner.os == 'Windows' + continue-on-error: true + run: | + dotnet test tests/Wolfgang.Etl.Abstractions.Tests.Unit/Wolfgang.Etl.Abstractions.Tests.Unit.csproj ` + --no-build ` + --configuration Release ` + --collect:"XPlat Code Coverage" ` + --results-directory "./TestResults" ` + --logger "console;verbosity=normal" ` + --logger "trx;LogFileName=testresults.trx" + + # Verify tests passed by checking TRX file + if (Test-Path "./TestResults/*.trx") { + $trxContent = Get-Content "./TestResults/*.trx" -Raw + if ($trxContent -match 'outcome="Failed"') { + Write-Error "Tests failed" + exit 1 + } + } + + # Coverage reporting (Windows only) + - name: Install ReportGenerator + if: runner.os == 'Windows' && always() + run: dotnet tool install -g dotnet-reportgenerator-globaltool + + - name: Generate coverage report + if: runner.os == 'Windows' && always() + continue-on-error: true + run: | + reportgenerator ` + -reports:"TestResults/**/coverage.cobertura.xml" ` + -targetdir:"CoverageReport" ` + -reporttypes:"Html;TextSummary;MarkdownSummaryGithub;CsvSummary" + + - name: Check coverage thresholds + if: runner.os == 'Windows' && hashFiles('CoverageReport/Summary.txt') != '' + shell: pwsh + run: | + if (-not (Test-Path "CoverageReport/Summary.txt")) { + Write-Warning "Coverage report not generated - skipping threshold check" + exit 0 + } + + $failedProjects = @() + $threshold = 80 + + Get-Content "CoverageReport/Summary.txt" | ForEach-Object { + if ($_ -match '^[^ ].*[0-9]+%$' -and $_ -notmatch '^Summary') { + $parts = $_ -split '\s+' + $module = $parts[0] + $percent = [int]($parts[-1] -replace '%','') + + Write-Host "Module '$module': $percent%" + + if ($percent -lt $threshold) { + Write-Host " ❌ Below threshold ($threshold%)" -ForegroundColor Red + $failedProjects += "$module ($percent%)" + } else { + Write-Host " ✅ Meets threshold" -ForegroundColor Green + } + } + } + + if ($failedProjects.Count -gt 0) { + Write-Error "Projects below $threshold% coverage: $($failedProjects -join ', ')" + exit 1 + } + + - name: Upload coverage artifacts + if: runner.os == 'Windows' && always() + uses: actions/upload-artifact@v4 + with: + name: coverage-results-and-report + path: | + TestResults/ + CoverageReport/ + + # Security scanning (all platforms) + - name: Install DevSkim CLI + run: dotnet tool install --global Microsoft.CST.DevSkim.CLI + + - name: Run security scan + shell: bash + run: | + devskim analyze \ + --source-code . \ + --file-format text \ + -E \ + --ignore-rule-ids DS176209 \ + --ignore-globs "**/api/**,**/CoverageReport/**,**/TestResults/**" \ + > devskim-results.txt 2>&1 || true + + - name: Check security scan results + if: always() + shell: bash + run: | + if [ -f devskim-results.txt ]; then + cat devskim-results.txt + + if grep -qi "found" devskim-results.txt; then + echo "❌ Security issues detected!" + exit 1 + else + echo "✅ No security issues found" + fi + else + echo "⚠️ DevSkim results file not found" + fi + + - name: Upload security scan results + if: always() + uses: actions/upload-artifact@v4 + with: + name: devskim-results-${{ matrix.os }} + path: devskim-results.txt diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..cc533f4 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,87 @@ +name: Release on Version Tag + +on: + push: + tags: + - 'v*.*.*' + +permissions: + contents: write + +jobs: + build-and-test: + name: Build and Test on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 5.0.x + 6.0.x + 7.0.x + 8.0.x + 9.0.x + + - name: Restore dependencies + run: dotnet restore + + - name: Build Solution (Release) + run: dotnet build --no-restore --configuration Release + + - name: Run tests for all test projects + shell: bash + run: | + find ./tests -type f -name '*Test*.csproj' | while read proj; do + echo "Running tests for $proj" + dotnet test "$proj" --no-build --configuration Release --logger "trx" --results-directory "./TestResults" + done + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-${{ matrix.os }} + path: 'TestResults/**/*.trx' + + publish: + name: Pack and Publish NuGet + needs: build-and-test + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 5.0.x + 6.0.x + 7.0.x + 8.0.x + 9.0.x + - name: Restore dependencies + run: dotnet restore + + - name: Build Solution (Release) + run: dotnet build --no-restore --configuration Release + + - name: Pack NuGet Package for Each Project in Src (Release) + shell: bash + run: | + dotnet pack src/Wolfgang.Etl.Abstractions.csproj --no-build --configuration Release --output ./nuget-packages + + - name: Publish NuGet Package + env: + NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} # Ensure this secret is set in repository settings + run: | + for pkg in ./nuget-packages/*.nupkg; do + dotnet nuget push "$pkg" --api-key "$NUGET_API_KEY" --source https://api.nuget.org/v3/index.json + done diff --git a/CODEOWNERS b/CODEOWNERS deleted file mode 100644 index c127892..0000000 --- a/CODEOWNERS +++ /dev/null @@ -1,2 +0,0 @@ -# Core functionality - changes here should be rare! -* @Chris-Wolfgang diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..cd2f245 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,133 @@ +# Contributor Covenant Code of Conduct + +## TL;DR + +`Be excellent to one another` + + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/Wolfgang.Etl.Abstractions.sln b/Wolfgang.Etl.Abstractions.sln index b9d04c5..a80b831 100644 --- a/Wolfgang.Etl.Abstractions.sln +++ b/Wolfgang.Etl.Abstractions.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.31903.59 +# Visual Studio Version 18 +VisualStudioVersion = 18.0.11222.15 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wolfgang.Etl.Abstractions", "src\Wolfgang.Etl.Abstractions\Wolfgang.Etl.Abstractions.csproj", "{C4987BAD-4513-955F-B3C1-7563D0C1A7A3}" EndProject @@ -57,6 +57,30 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Docs", "Docs", "{8EC462FD-D docs\readme.md = docs\readme.md EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".github", ".github", "{9B9A162C-C5B8-495C-A6D0-8C3135E283B9}" + ProjectSection(SolutionItems) = preProject + .github\CODEOWNERS = .github\CODEOWNERS + .github\dependabot.yml = .github\dependabot.yml + .github\pull_request_template.md = .github\pull_request_template.md + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ISSUE_TEMPLATE", "ISSUE_TEMPLATE", "{AF971B90-A335-49AF-8AB6-F387CAED12E4}" + ProjectSection(SolutionItems) = preProject + .github\ISSUE_TEMPLATE\bug_report.md = .github\ISSUE_TEMPLATE\bug_report.md + .github\ISSUE_TEMPLATE\BUG_REPORT.yaml = .github\ISSUE_TEMPLATE\BUG_REPORT.yaml + .github\ISSUE_TEMPLATE\feature_request.md = .github\ISSUE_TEMPLATE\feature_request.md + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{2D19706F-4199-46BD-B047-C4ED3AEDD90A}" + ProjectSection(SolutionItems) = preProject + .github\workflows\create-labels.yaml = .github\workflows\create-labels.yaml + .github\workflows\docfx.yaml = .github\workflows\docfx.yaml + .github\workflows\pr.yaml = .github\workflows\pr.yaml + .github\workflows\release.yaml = .github\workflows\release.yaml + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "benchmarks", "benchmarks", "{AE206253-B766-4B6A-8C08-9E70605A2B27}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -160,6 +184,8 @@ Global {861EA36D-970E-4CFE-9E72-D3D12F0BBB60} = {336D72A1-8E5E-49DE-83D9-DF6BE458BA24} {80E49C71-1073-4208-B48F-E0F399946B3B} = {3C48157B-5E90-489E-9444-E01F51D59F86} {85A15A15-D528-4542-A546-63BE0EAED986} = {336D72A1-8E5E-49DE-83D9-DF6BE458BA24} + {AF971B90-A335-49AF-8AB6-F387CAED12E4} = {9B9A162C-C5B8-495C-A6D0-8C3135E283B9} + {2D19706F-4199-46BD-B047-C4ED3AEDD90A} = {9B9A162C-C5B8-495C-A6D0-8C3135E283B9} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {F673635D-58CE-48A5-9AE4-31F4484BED9E} diff --git a/benchmarks/.gitkeep b/benchmarks/.gitkeep new file mode 100644 index 0000000..680d565 --- /dev/null +++ b/benchmarks/.gitkeep @@ -0,0 +1 @@ +# Benchmarks diff --git a/benchmarks/placeholder.txt b/benchmarks/placeholder.txt new file mode 100644 index 0000000..e69de29 diff --git a/examples/Net4.8/Example1-BasicETL/Example1-BasicETL.csproj b/examples/Net4.8/Example1-BasicETL/Example1-BasicETL.csproj index 9524fba..5ac79a1 100644 --- a/examples/Net4.8/Example1-BasicETL/Example1-BasicETL.csproj +++ b/examples/Net4.8/Example1-BasicETL/Example1-BasicETL.csproj @@ -34,16 +34,16 @@ 4 - - ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.6\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.10.0.1\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll - - ..\..\..\packages\System.Runtime.CompilerServices.Unsafe.4.5.3\lib\net461\System.Runtime.CompilerServices.Unsafe.dll + + ..\..\..\packages\System.Runtime.CompilerServices.Unsafe.6.1.2\lib\net462\System.Runtime.CompilerServices.Unsafe.dll - - ..\..\..\packages\System.Threading.Tasks.Extensions.4.5.4\lib\net461\System.Threading.Tasks.Extensions.dll + + ..\..\..\packages\System.Threading.Tasks.Extensions.4.6.3\lib\net462\System.Threading.Tasks.Extensions.dll diff --git a/examples/Net4.8/Example1-BasicETL/packages.config b/examples/Net4.8/Example1-BasicETL/packages.config index 7b7c193..193ca78 100644 --- a/examples/Net4.8/Example1-BasicETL/packages.config +++ b/examples/Net4.8/Example1-BasicETL/packages.config @@ -1,6 +1,6 @@  - - - + + + \ No newline at end of file diff --git a/examples/Net4.8/Example2-WithCancellationToken/Example2-WithCancellationToken.csproj b/examples/Net4.8/Example2-WithCancellationToken/Example2-WithCancellationToken.csproj index 06d7c1e..06eb4db 100644 --- a/examples/Net4.8/Example2-WithCancellationToken/Example2-WithCancellationToken.csproj +++ b/examples/Net4.8/Example2-WithCancellationToken/Example2-WithCancellationToken.csproj @@ -34,8 +34,8 @@ 4 - - ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.10.0.1\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll diff --git a/examples/Net4.8/Example2-WithCancellationToken/packages.config b/examples/Net4.8/Example2-WithCancellationToken/packages.config index 27ab3ab..193ca78 100644 --- a/examples/Net4.8/Example2-WithCancellationToken/packages.config +++ b/examples/Net4.8/Example2-WithCancellationToken/packages.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/examples/Net4.8/Example3-WithGracefulCancellation/Example3-WithGracefulCancellation.csproj b/examples/Net4.8/Example3-WithGracefulCancellation/Example3-WithGracefulCancellation.csproj index 9d54856..54b7d64 100644 --- a/examples/Net4.8/Example3-WithGracefulCancellation/Example3-WithGracefulCancellation.csproj +++ b/examples/Net4.8/Example3-WithGracefulCancellation/Example3-WithGracefulCancellation.csproj @@ -34,8 +34,8 @@ 4 - - ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.10.0.1\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll diff --git a/examples/Net4.8/Example3-WithGracefulCancellation/packages.config b/examples/Net4.8/Example3-WithGracefulCancellation/packages.config index 27ab3ab..193ca78 100644 --- a/examples/Net4.8/Example3-WithGracefulCancellation/packages.config +++ b/examples/Net4.8/Example3-WithGracefulCancellation/packages.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/examples/Net4.8/Example4a-WithExtractorProgress/Example4a-WithExtractorProgress.csproj b/examples/Net4.8/Example4a-WithExtractorProgress/Example4a-WithExtractorProgress.csproj index efd9a36..ffd346b 100644 --- a/examples/Net4.8/Example4a-WithExtractorProgress/Example4a-WithExtractorProgress.csproj +++ b/examples/Net4.8/Example4a-WithExtractorProgress/Example4a-WithExtractorProgress.csproj @@ -35,8 +35,8 @@ latest - - ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.10.0.1\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll diff --git a/examples/Net4.8/Example4a-WithExtractorProgress/packages.config b/examples/Net4.8/Example4a-WithExtractorProgress/packages.config index 6c61924..f8a655f 100644 --- a/examples/Net4.8/Example4a-WithExtractorProgress/packages.config +++ b/examples/Net4.8/Example4a-WithExtractorProgress/packages.config @@ -1,6 +1,6 @@  - + diff --git a/examples/Net4.8/Example4b-WithTransformerProgress/Example4b-WithTransformerProgress.csproj b/examples/Net4.8/Example4b-WithTransformerProgress/Example4b-WithTransformerProgress.csproj index 609a437..f733bb9 100644 --- a/examples/Net4.8/Example4b-WithTransformerProgress/Example4b-WithTransformerProgress.csproj +++ b/examples/Net4.8/Example4b-WithTransformerProgress/Example4b-WithTransformerProgress.csproj @@ -35,8 +35,8 @@ latest - - ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.10.0.1\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll diff --git a/examples/Net4.8/Example4b-WithTransformerProgress/packages.config b/examples/Net4.8/Example4b-WithTransformerProgress/packages.config index 27ab3ab..193ca78 100644 --- a/examples/Net4.8/Example4b-WithTransformerProgress/packages.config +++ b/examples/Net4.8/Example4b-WithTransformerProgress/packages.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/examples/Net4.8/Example4c-WithLoaderProgress/Example4c-WithLoaderProgress.csproj b/examples/Net4.8/Example4c-WithLoaderProgress/Example4c-WithLoaderProgress.csproj index e35d699..2b1ed28 100644 --- a/examples/Net4.8/Example4c-WithLoaderProgress/Example4c-WithLoaderProgress.csproj +++ b/examples/Net4.8/Example4c-WithLoaderProgress/Example4c-WithLoaderProgress.csproj @@ -35,8 +35,8 @@ latest - - ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.10.0.1\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll diff --git a/examples/Net4.8/Example4c-WithLoaderProgress/packages.config b/examples/Net4.8/Example4c-WithLoaderProgress/packages.config index 27ab3ab..193ca78 100644 --- a/examples/Net4.8/Example4c-WithLoaderProgress/packages.config +++ b/examples/Net4.8/Example4c-WithLoaderProgress/packages.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/Example5a-ExtractorWithProgressAndCancellation.csproj b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/Example5a-ExtractorWithProgressAndCancellation.csproj index 003cce7..1a9a2cb 100644 --- a/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/Example5a-ExtractorWithProgressAndCancellation.csproj +++ b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/Example5a-ExtractorWithProgressAndCancellation.csproj @@ -35,8 +35,8 @@ latest - - ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.10.0.1\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll diff --git a/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/packages.config b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/packages.config index 27ab3ab..193ca78 100644 --- a/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/packages.config +++ b/examples/Net4.8/Example5a-ExtractorWithProgressAndCancellation/packages.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/examples/Net4.8/Example6-ReducingDuplicateCode/Example6-ReducingDuplicateCode.csproj b/examples/Net4.8/Example6-ReducingDuplicateCode/Example6-ReducingDuplicateCode.csproj index 0023993..ebe3305 100644 --- a/examples/Net4.8/Example6-ReducingDuplicateCode/Example6-ReducingDuplicateCode.csproj +++ b/examples/Net4.8/Example6-ReducingDuplicateCode/Example6-ReducingDuplicateCode.csproj @@ -35,8 +35,8 @@ 12 - - ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.9.0.7\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + ..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.10.0.1\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll diff --git a/examples/Net4.8/Example6-ReducingDuplicateCode/packages.config b/examples/Net4.8/Example6-ReducingDuplicateCode/packages.config index 27ab3ab..193ca78 100644 --- a/examples/Net4.8/Example6-ReducingDuplicateCode/packages.config +++ b/examples/Net4.8/Example6-ReducingDuplicateCode/packages.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/src/Wolfgang.Etl.Abstractions/ExtractorBase.cs b/src/Wolfgang.Etl.Abstractions/ExtractorBase.cs index 3ef9a6a..66982c1 100644 --- a/src/Wolfgang.Etl.Abstractions/ExtractorBase.cs +++ b/src/Wolfgang.Etl.Abstractions/ExtractorBase.cs @@ -5,317 +5,323 @@ -namespace Wolfgang.Etl.Abstractions +namespace Wolfgang.Etl.Abstractions; + +/// +/// Provides a basic implementation for data extractor that extracts data of type TSource +/// Library authors can use this base class to create custom extractors by inheriting from it and implementing +/// ExtractWorkerAsync and CreateProgressReport methods. +/// +/// The type of the object being extracted +/// The type of the progress object +public abstract class ExtractorBase + : IExtractWithProgressAndCancellationAsync { - public abstract class ExtractorBase - : IExtractWithProgressAndCancellationAsync - { - private int _reportingInterval = 1_000; - private int _maximumItemCount = int.MaxValue; - private int _skippedItemCount; - private int _currentItemCount; - private int _currentSkippedItemCount; + private int _reportingInterval = 1_000; + private int _maximumItemCount = int.MaxValue; + private int _skippedItemCount; + private int _currentItemCount; + private int _currentSkippedItemCount; - /// - /// The number of milliseconds between progress updates. - /// - /// Value cannot be less than 1 - public int ReportingInterval + /// + /// The number of milliseconds between progress updates. + /// + /// Value cannot be less than 1 + public int ReportingInterval + { + get => _reportingInterval; + set { - get => _reportingInterval; - set + if (value < 1) { - if (value < 1) - { - throw new ArgumentOutOfRangeException(nameof(value), "Reporting interval must be greater than 0."); - } - _reportingInterval = value; + throw new ArgumentOutOfRangeException(nameof(value), "Reporting interval must be greater than 0."); } + _reportingInterval = value; } + } - /// - /// The current number of items extracted so far. - /// - /// - /// It is the responsibility of the derived class to keep this value up to date as the - /// base class will have no way of knowing the correct value - /// + /// + /// The current number of items extracted so far. + /// + /// + /// It is the responsibility of the derived class to keep this value up to date as the + /// base class will have no way of knowing the correct value + /// - [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] - public int CurrentItemCount + [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] + public int CurrentItemCount + { + get => _currentItemCount; + protected set { - get => _currentItemCount; - protected set + if (value < 0) { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value)); - } - _currentItemCount = value; + throw new ArgumentOutOfRangeException(nameof(value)); } + _currentItemCount = value; } + } - /// - /// Gets the current number of records skipped - /// - public int CurrentSkippedItemCount + /// + /// Gets the current number of records skipped + /// + public int CurrentSkippedItemCount + { + get => _currentSkippedItemCount; + protected set { - get => _currentSkippedItemCount; - protected set + if (value < 0) { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "value cannot be less than 0."); - } + throw new ArgumentOutOfRangeException(nameof(value), "value cannot be less than 0."); } } + } - /// - /// The maximum number of items to extract. Once the extractor has reached this limit, - /// it should stop extracting and signal the end of the sequence. - /// - /// - /// This is useful for partially extracting data from a source, especially when the source is large - /// or infinite or during development. - /// - /// The specified value is less than 0 - /// - /// - /// var count = 0; - /// using (var reader = new StreamReader(filePath)) - /// { - /// while (!reader.EndOfStream) - /// { - /// yield return await reader.ReadLineAsync(); - /// count++; - /// if (count >= MaximumItemCount) - /// { - /// Console.WriteLine("Maximum item count reached. Stopping extraction."); - /// break; // Stop extracting if the maximum item count is reached - /// } - /// } - /// } - /// - /// - - [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] - public int MaximumItemCount + /// + /// The maximum number of items to extract. Once the extractor has reached this limit, + /// it should stop extracting and signal the end of the sequence. + /// + /// + /// This is useful for partially extracting data from a source, especially when the source is large + /// or infinite or during development. + /// + /// The specified value is less than 0 + /// + /// + /// var count = 0; + /// using (var reader = new StreamReader(filePath)) + /// { + /// while (!reader.EndOfStream) + /// { + /// yield return await reader.ReadLineAsync(); + /// count++; + /// if (count >= MaximumItemCount) + /// { + /// Console.WriteLine("Maximum item count reached. Stopping extraction."); + /// break; // Stop extracting if the maximum item count is reached + /// } + /// } + /// } + /// + /// + + [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] + public int MaximumItemCount + { + get => _maximumItemCount; + set { - get => _maximumItemCount; - set + if (value < 0) { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Maximum item count cannot be less than 0."); - } - _maximumItemCount = value; + throw new ArgumentOutOfRangeException(nameof(value), "Maximum item count cannot be less than 0."); } + _maximumItemCount = value; } + } - /// - /// The number of items to skip before extracting. - /// The extractor should skip the specified number of items before starting to yield results. - /// - /// - /// This is useful for partially extracting data from a source during development, or to skip - /// items that were already processed or are not relevant for the current extraction. - /// - /// The specified value is less than 0 - /// - /// - /// using (var reader = new StreamReader(filePath)) - /// { - /// // Skip the specified number of items before starting to yield results - /// - /// var skipCount = 0; - /// while (!reader.EndOfStream && skipCount < SkipItemCount) - /// { - /// await reader.ReadLineAsync(); - /// skipCount++; - /// } - /// - /// - /// // Now start yielding results - /// - /// var count++; - /// while (!reader.EndOfStream) - /// { - /// yield return await reader.ReadLineAsync(); - /// count++; - /// } - /// } - /// - /// - - [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] - public int SkipItemCount + /// + /// The number of items to skip before extracting. + /// The extractor should skip the specified number of items before starting to yield results. + /// + /// + /// This is useful for partially extracting data from a source during development, or to skip + /// items that were already processed or are not relevant for the current extraction. + /// + /// The specified value is less than 0 + /// + /// + /// using (var reader = new StreamReader(filePath)) + /// { + /// // Skip the specified number of items before starting to yield results + /// + /// var skipCount = 0; + /// while (!reader.EndOfStream && skipCount < SkipItemCount) + /// { + /// await reader.ReadLineAsync(); + /// skipCount++; + /// } + /// + /// + /// // Now start yielding results + /// + /// var count++; + /// while (!reader.EndOfStream) + /// { + /// yield return await reader.ReadLineAsync(); + /// count++; + /// } + /// } + /// + /// + + [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] + public int SkipItemCount + { + get => _skippedItemCount; + set { - get => _skippedItemCount; - set + if (value < 0) { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Skip item count cannot be less than 0."); - } - _skippedItemCount = value; + throw new ArgumentOutOfRangeException(nameof(value), "Skip item count cannot be less than 0."); } + _skippedItemCount = value; } + } - /// - /// Asynchronously extracts data of type TSource from a source. - /// - /// - /// IAsyncEnumerable<T> - /// The result may be an empty sequence if no data is available or if the extraction fails. - /// - public virtual IAsyncEnumerable ExtractAsync() - { - return ExtractWorkerAsync(CancellationToken.None); - } + /// + /// Asynchronously extracts data of type TSource from a source. + /// + /// + /// IAsyncEnumerable<T> + /// The result may be an empty sequence if no data is available or if the extraction fails. + /// + public virtual IAsyncEnumerable ExtractAsync() + { + return ExtractWorkerAsync(CancellationToken.None); + } - /// - /// Asynchronously extracts data of type TSource from a source. - /// - /// A CancellationToken to observe while waiting for the task to complete. - /// - /// IAsyncEnumerable<T> - /// The result may be an empty sequence if no data is available or if the extraction fails. - /// - /// - /// The extractor should be able to handle cancellation requests gracefully. - /// If the caller doesn't plan on cancelling the extraction, CancellationToken.None should be passed in. - /// - public virtual IAsyncEnumerable ExtractAsync(CancellationToken token) - { - return ExtractWorkerAsync(token); - } + /// + /// Asynchronously extracts data of type TSource from a source. + /// + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// IAsyncEnumerable<T> + /// The result may be an empty sequence if no data is available or if the extraction fails. + /// + /// + /// The extractor should be able to handle cancellation requests gracefully. + /// If the caller doesn't plan on cancelling the extraction, CancellationToken.None should be passed in. + /// + public virtual IAsyncEnumerable ExtractAsync(CancellationToken token) + { + return ExtractWorkerAsync(token); + } - /// - /// Asynchronously extracts data of type TSource from a source. - /// - /// A provider for progress updates. - /// - /// IAsyncEnumerable<T> - /// The result may be an empty sequence if no data is available or if the extraction fails. - /// - /// The value of progress is null - public virtual IAsyncEnumerable ExtractAsync(IProgress progress) + /// + /// Asynchronously extracts data of type TSource from a source. + /// + /// A provider for progress updates. + /// + /// IAsyncEnumerable<T> + /// The result may be an empty sequence if no data is available or if the extraction fails. + /// + /// The value of progress is null + public virtual IAsyncEnumerable ExtractAsync(IProgress progress) + { + if (progress == null) { - if (progress == null) - { - throw new ArgumentNullException(nameof(progress), "Progress cannot be null."); - } + throw new ArgumentNullException(nameof(progress), "Progress cannot be null."); + } - using var timer = new Timer - ( - _ => progress.Report(CreateProgressReport()), - null, - TimeSpan.Zero, - TimeSpan.FromMilliseconds(ReportingInterval) - ); + using var timer = new Timer + ( + _ => progress.Report(CreateProgressReport()), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(ReportingInterval) + ); - return ExtractWorkerAsync(CancellationToken.None); - } + return ExtractWorkerAsync(CancellationToken.None); + } - /// - /// Asynchronously extracts data of type TSource from a source. - /// - /// A provider for progress updates. - /// A CancellationToken to observe while waiting for the task to complete. - /// - /// IAsyncEnumerable<T> - /// The result may be an empty sequence if no data is available or if the extraction fails. - /// - /// - /// The extractor should be able to handle cancellation requests gracefully. - /// If the caller doesn't plan on cancelling the extraction, CancellationToken.None should be passed in. - /// - /// The value of progress is null - public virtual IAsyncEnumerable ExtractAsync(IProgress progress, CancellationToken token) + /// + /// Asynchronously extracts data of type TSource from a source. + /// + /// A provider for progress updates. + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// IAsyncEnumerable<T> + /// The result may be an empty sequence if no data is available or if the extraction fails. + /// + /// + /// The extractor should be able to handle cancellation requests gracefully. + /// If the caller doesn't plan on cancelling the extraction, CancellationToken.None should be passed in. + /// + /// The value of progress is null + public virtual IAsyncEnumerable ExtractAsync(IProgress progress, CancellationToken token) + { + if (progress == null) { - if (progress == null) - { - throw new ArgumentNullException(nameof(progress), "Progress cannot be null."); - } + throw new ArgumentNullException(nameof(progress), "Progress cannot be null."); + } - using var timer = new Timer - ( - _ => progress.Report(CreateProgressReport()), - null, - TimeSpan.Zero, + using var timer = new Timer + ( + _ => progress.Report(CreateProgressReport()), + null, + TimeSpan.Zero, - TimeSpan.FromMilliseconds(ReportingInterval) - ); + TimeSpan.FromMilliseconds(ReportingInterval) + ); - return ExtractWorkerAsync(token); - } + return ExtractWorkerAsync(token); + } - /// - /// This method is the core implementation of the extraction logic and should be - /// overridden by derived classes. - /// - /// A CancellationToken to observe while waiting for the task to complete. - /// - /// IAsyncEnumerable<T> - /// The result may be an empty sequence if no data is available or if the extraction fails. - /// - protected abstract IAsyncEnumerable ExtractWorkerAsync(CancellationToken token); + /// + /// This method is the core implementation of the extraction logic and should be + /// overridden by derived classes. + /// + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// IAsyncEnumerable<T> + /// The result may be an empty sequence if no data is available or if the extraction fails. + /// + protected abstract IAsyncEnumerable ExtractWorkerAsync(CancellationToken token); - /// - /// Creates a progress report of type TProgress. This gives the derived class the opportunity to - /// implement a custom progress report that is specific to the extraction process. - /// - /// Progress of type TProgress - protected abstract TProgress CreateProgressReport(); + /// + /// Creates a progress report of type TProgress. This gives the derived class the opportunity to + /// implement a custom progress report that is specific to the extraction process. + /// + /// Progress of type TProgress + protected abstract TProgress CreateProgressReport(); - /// - /// Increments the CurrentItemCount in a thread safe manner. - /// - /// - /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not - /// thread safe. This method ensures that CurrentItemCount is incremented safely - /// - protected void IncrementCurrentItemCount() - { - Interlocked.Increment(ref _currentItemCount); - } + /// + /// Increments the CurrentItemCount in a thread safe manner. + /// + /// + /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not + /// thread safe. This method ensures that CurrentItemCount is incremented safely + /// + protected void IncrementCurrentItemCount() + { + Interlocked.Increment(ref _currentItemCount); + } - /// - /// Increments the CurrentItemCount in a thread safe manner. - /// - /// - /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not - /// thread safe. This method ensures that CurrentItemCount is incremented safely - /// - protected void IncrementCurrentSkippedItemCount() - { - Interlocked.Increment(ref _currentSkippedItemCount); - } + /// + /// Increments the CurrentItemCount in a thread safe manner. + /// + /// + /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not + /// thread safe. This method ensures that CurrentItemCount is incremented safely + /// + protected void IncrementCurrentSkippedItemCount() + { + Interlocked.Increment(ref _currentSkippedItemCount); } -} +} \ No newline at end of file diff --git a/src/Wolfgang.Etl.Abstractions/LoaderBase.cs b/src/Wolfgang.Etl.Abstractions/LoaderBase.cs index faadf79..2b09152 100644 --- a/src/Wolfgang.Etl.Abstractions/LoaderBase.cs +++ b/src/Wolfgang.Etl.Abstractions/LoaderBase.cs @@ -6,328 +6,334 @@ -namespace Wolfgang.Etl.Abstractions +namespace Wolfgang.Etl.Abstractions; + +/// +/// Provides a basic implementation for data loaders the write data of type TDestination to a target destination. +/// Library authors can use this base class to create custom loaders by inheriting from it and implementing +/// LoadWorkerAsync and CreateProgressReport methods. +/// +/// The type of the destination object being written +/// The type of the progress object +public abstract class LoaderBase + : ILoadWithProgressAndCancellationAsync { - public abstract class LoaderBase - : ILoadWithProgressAndCancellationAsync - { - private int _reportingInterval = 1_000; - private int _maximumItemCount = int.MaxValue; - private int _skipItemCount; - private int _currentItemCount; - private int _currentSkippedItemCount; + private int _reportingInterval = 1_000; + private int _maximumItemCount = int.MaxValue; + private int _skipItemCount; + private int _currentItemCount; + private int _currentSkippedItemCount; - /// - /// The number of milliseconds between progress updates. - /// - /// Value cannot be less than 1 - public int ReportingInterval + /// + /// The number of milliseconds between progress updates. + /// + /// Value cannot be less than 1 + public int ReportingInterval + { + get => _reportingInterval; + set { - get => _reportingInterval; - set + if (value < 1) { - if (value < 1) - { - throw new ArgumentOutOfRangeException(nameof(value), "Reporting interval must be greater than 0."); - } - _reportingInterval = value; + throw new ArgumentOutOfRangeException(nameof(value), "Reporting interval must be greater than 0."); } + _reportingInterval = value; } + } - /// - /// The current number of items loaded so far. - /// - /// - /// It is the responsibility of the derived class to keep this value up to date as the - /// base class will have no way of knowing the correct value - /// + /// + /// The current number of items loaded so far. + /// + /// + /// It is the responsibility of the derived class to keep this value up to date as the + /// base class will have no way of knowing the correct value + /// - [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] - public int CurrentItemCount + [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] + public int CurrentItemCount + { + get => _currentItemCount; + protected set { - get => _currentItemCount; - protected set + if (value < 0) { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value)); - } - _currentItemCount = value; + throw new ArgumentOutOfRangeException(nameof(value)); } + _currentItemCount = value; } + } - /// - /// Gets the current number of records skipped - /// - public int CurrentSkippedItemCount + /// + /// Gets the current number of records skipped + /// + public int CurrentSkippedItemCount + { + get => _currentSkippedItemCount; + protected set { - get => _currentSkippedItemCount; - protected set + if (value < 0) { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "value cannot be less than 0."); - } + throw new ArgumentOutOfRangeException(nameof(value), "value cannot be less than 0."); } } + } - /// - /// The maximum number of items to load. Once the loader has reached this limit, - /// it should stop loading items and exist as if it had reached the end of the list - /// - /// - /// This is useful for partially loading data from a source, especially when the source is large - /// or infinite or during development. - /// - /// The specified value is less than 1 - /// - /// - /// foreach (var item in items.Skip(SkipItemCount).Take(MaxItemCount)) - /// { - /// // Process the item - /// } - /// - /// - - public int MaximumItemCount + /// + /// The maximum number of items to load. Once the loader has reached this limit, + /// it should stop loading items and exist as if it had reached the end of the list + /// + /// + /// This is useful for partially loading data from a source, especially when the source is large + /// or infinite or during development. + /// + /// The specified value is less than 1 + /// + /// + /// foreach (var item in items.Skip(SkipItemCount).Take(MaxItemCount)) + /// { + /// // Process the item + /// } + /// + /// + + public int MaximumItemCount + { + get => _maximumItemCount; + set { - get => _maximumItemCount; - set + if (value < 0) { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Maximum item count cannot be less than 0."); - } - _maximumItemCount = value; + throw new ArgumentOutOfRangeException(nameof(value), "Maximum item count cannot be less than 0."); } + _maximumItemCount = value; } + } - /// - /// The number of items skipped before loading. - /// The loader should skip the specified number of items before starting to process the remaining items. - /// - /// - /// This is useful for skipping the beginning of the list during testing or because it may already be loaded - /// - /// The specified value is less than 0 - /// - /// - /// foreach (var item in items.Skip(SkipItemCount).Take(MaxItemCount)) - /// { - /// // Process the item - /// } - /// - /// - - public int SkipItemCount + /// + /// The number of items skipped before loading. + /// The loader should skip the specified number of items before starting to process the remaining items. + /// + /// + /// This is useful for skipping the beginning of the list during testing or because it may already be loaded + /// + /// The specified value is less than 0 + /// + /// + /// foreach (var item in items.Skip(SkipItemCount).Take(MaxItemCount)) + /// { + /// // Process the item + /// } + /// + /// + + public int SkipItemCount + { + get => _skipItemCount; + set { - get => _skipItemCount; - set + if (value < 0) { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Skip item count cannot be less than 0."); - } - _skipItemCount = value; + throw new ArgumentOutOfRangeException(nameof(value), "Skip item count cannot be less than 0."); } + _skipItemCount = value; } + } - /// - /// Asynchronously loads data of type TDestination into the target destination. - /// - /// The items to be loaded to the destination. - /// - /// Items may be an empty sequence if no data is available or if the extraction fails. - /// - /// Task - /// Argument items is null - public virtual Task LoadAsync - ( - IAsyncEnumerable items - ) + /// + /// Asynchronously loads data of type TDestination into the target destination. + /// + /// The items to be loaded to the destination. + /// + /// Items may be an empty sequence if no data is available or if the extraction fails. + /// + /// Task + /// Argument items is null + public virtual Task LoadAsync + ( + IAsyncEnumerable items + ) + { + if (items == null) { - if (items == null) - { - throw new ArgumentNullException(nameof(items)); - } - - return LoadWorkerAsync(items, CancellationToken.None); + throw new ArgumentNullException(nameof(items)); } + return LoadWorkerAsync(items, CancellationToken.None); + } + - /// - /// Asynchronously loads data of type TDestination into the target destination. - /// - /// The items to be loaded to the destination. - /// A CancellationToken to observe while waiting for the task to complete. - /// - /// Items may be an empty sequence if no data is available or if the extraction fails. - /// - /// Task - /// Argument items is null - public virtual Task LoadAsync - ( - IAsyncEnumerable items, - CancellationToken token - ) + /// + /// Asynchronously loads data of type TDestination into the target destination. + /// + /// The items to be loaded to the destination. + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// Items may be an empty sequence if no data is available or if the extraction fails. + /// + /// Task + /// Argument items is null + public virtual Task LoadAsync + ( + IAsyncEnumerable items, + CancellationToken token + ) + { + if (items == null) { - if (items == null) - { - throw new ArgumentNullException(nameof(items)); - } - return LoadWorkerAsync(items, CancellationToken.None); + throw new ArgumentNullException(nameof(items)); } + return LoadWorkerAsync(items, CancellationToken.None); + } - /// - /// Asynchronously loads data of type TDestination into the target destination. - /// - /// The items to be loaded to the destination. - /// A provider for progress updates. - /// - /// Items may be an empty sequence if no data is available or if the extraction fails. - /// - /// Task - /// Argument items is null - /// Argument progress is null - public virtual Task LoadAsync - ( - IAsyncEnumerable items, - IProgress progress - ) + /// + /// Asynchronously loads data of type TDestination into the target destination. + /// + /// The items to be loaded to the destination. + /// A provider for progress updates. + /// + /// Items may be an empty sequence if no data is available or if the extraction fails. + /// + /// Task + /// Argument items is null + /// Argument progress is null + public virtual Task LoadAsync + ( + IAsyncEnumerable items, + IProgress progress + ) + { + if (items == null) { - if (items == null) - { - throw new ArgumentNullException(nameof(items)); - } - if (progress == null) - { - throw new ArgumentNullException(nameof(progress)); - } + throw new ArgumentNullException(nameof(items)); + } + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } - using var timer = new Timer - ( - _ => progress.Report(CreateProgressReport()), - null, - TimeSpan.Zero, - TimeSpan.FromMilliseconds(ReportingInterval) - ); + using var timer = new Timer + ( + _ => progress.Report(CreateProgressReport()), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(ReportingInterval) + ); - return LoadWorkerAsync(items, CancellationToken.None); - } + return LoadWorkerAsync(items, CancellationToken.None); + } - /// - /// Asynchronously loads data of type TDestination into the target destination. - /// - /// The items to be loaded to the destination. - /// A CancellationToken to observe while waiting for the task to complete. - /// A provider for progress updates. - /// - /// Items may be an empty sequence if no data is available or if the extraction fails. - /// - /// Task - /// Argument items is null - /// Argument progress is null - public virtual Task LoadAsync - ( - IAsyncEnumerable items, - IProgress progress, - CancellationToken token - ) + /// + /// Asynchronously loads data of type TDestination into the target destination. + /// + /// The items to be loaded to the destination. + /// A CancellationToken to observe while waiting for the task to complete. + /// A provider for progress updates. + /// + /// Items may be an empty sequence if no data is available or if the extraction fails. + /// + /// Task + /// Argument items is null + /// Argument progress is null + public virtual Task LoadAsync + ( + IAsyncEnumerable items, + IProgress progress, + CancellationToken token + ) + { + if (items == null) { - if (items == null) - { - throw new ArgumentNullException(nameof(items)); - } + throw new ArgumentNullException(nameof(items)); + } - if (progress == null) - { - throw new ArgumentNullException(nameof(progress)); - } + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } - using var timer = new Timer - ( - _ => progress.Report(CreateProgressReport()), - null, - TimeSpan.Zero, - TimeSpan.FromMilliseconds(ReportingInterval) - ); + using var timer = new Timer + ( + _ => progress.Report(CreateProgressReport()), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(ReportingInterval) + ); - return LoadWorkerAsync(items, token); - } + return LoadWorkerAsync(items, token); + } - /// - /// This method is the core implementation of the loading logic and should be - /// overridden by derived classes. - /// - /// The items to be loaded to the destination. - /// A CancellationToken to observe while waiting for the task to complete. - /// - /// Items may be an empty sequence if no data is available or if the extraction fails. - /// - /// Task - /// Argument items is null - protected abstract Task LoadWorkerAsync - ( - IAsyncEnumerableitems, - CancellationToken token - ); + /// + /// This method is the core implementation of the loading logic and should be + /// overridden by derived classes. + /// + /// The items to be loaded to the destination. + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// Items may be an empty sequence if no data is available or if the extraction fails. + /// + /// Task + /// Argument items is null + protected abstract Task LoadWorkerAsync + ( + IAsyncEnumerableitems, + CancellationToken token + ); - /// - /// Creates a progress report of type TProgress. This gives the derived class the opportunity to - /// implement a custom progress report that is specific to the extraction process. - /// - /// Progress of type TProgress - protected abstract TProgress CreateProgressReport(); + /// + /// Creates a progress report of type TProgress. This gives the derived class the opportunity to + /// implement a custom progress report that is specific to the extraction process. + /// + /// Progress of type TProgress + protected abstract TProgress CreateProgressReport(); - /// - /// Increments the CurrentItemCount in a thread safe manner. - /// - /// - /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not - /// thread safe. This method ensures that CurrentItemCount is incremented safely - /// - protected void IncrementCurrentItemCount() - { - Interlocked.Increment(ref _currentItemCount); - } + /// + /// Increments the CurrentItemCount in a thread safe manner. + /// + /// + /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not + /// thread safe. This method ensures that CurrentItemCount is incremented safely + /// + protected void IncrementCurrentItemCount() + { + Interlocked.Increment(ref _currentItemCount); + } - /// - /// Increments the CurrentItemCount in a thread safe manner. - /// - /// - /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not - /// thread safe. This method ensures that CurrentItemCount is incremented safely - /// - protected void IncrementCurrentSkippedItemCount() - { - Interlocked.Increment(ref _currentSkippedItemCount); - } - + /// + /// Increments the CurrentItemCount in a thread safe manner. + /// + /// + /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not + /// thread safe. This method ensures that CurrentItemCount is incremented safely + /// + protected void IncrementCurrentSkippedItemCount() + { + Interlocked.Increment(ref _currentSkippedItemCount); } -} + +} \ No newline at end of file diff --git a/src/Wolfgang.Etl.Abstractions/TransformerBase.cs b/src/Wolfgang.Etl.Abstractions/TransformerBase.cs index d2840c7..a1b4503 100644 --- a/src/Wolfgang.Etl.Abstractions/TransformerBase.cs +++ b/src/Wolfgang.Etl.Abstractions/TransformerBase.cs @@ -5,327 +5,334 @@ -namespace Wolfgang.Etl.Abstractions +namespace Wolfgang.Etl.Abstractions; + +/// +/// Provides a basic implementation for data transformers that convert data from TSource to TDestination +/// Library authors can use this base class to create custom transformers by inheriting from it and implementing +/// TransformWorkerAsync and CreateProgressReport methods. +/// +/// The type of the source object +/// The type of the destination object +/// The type of the progress object +public abstract class TransformerBase + : ITransformWithProgressAndCancellationAsync { - public abstract class TransformerBase - : ITransformWithProgressAndCancellationAsync - { - private int _reportingInterval = 1_000; - private int _maximumItemCount = int.MaxValue; - private int _skipItemCount; - private int _currentItemCount; - private int _currentSkippedItemCount; + private int _reportingInterval = 1_000; + private int _maximumItemCount = int.MaxValue; + private int _skipItemCount; + private int _currentItemCount; + private int _currentSkippedItemCount; - /// - /// The number of milliseconds between progress updates. - /// - /// Value cannot be less than 1 - public int ReportingInterval + /// + /// The number of milliseconds between progress updates. + /// + /// Value cannot be less than 1 + public int ReportingInterval + { + get => _reportingInterval; + set { - get => _reportingInterval; - set + if (value < 1) { - if (value < 1) - { - throw new ArgumentOutOfRangeException(nameof(value), "Reporting interval must be greater than 0."); - } - _reportingInterval = value; + throw new ArgumentOutOfRangeException(nameof(value), "Reporting interval must be greater than 0."); } + _reportingInterval = value; } + } - /// - /// The current number of items transformed so far. - /// - /// - /// It is the responsibility of the derived class to keep this value up to date as the - /// base class will have no way of knowing the correct value - /// + /// + /// The current number of items transformed so far. + /// + /// + /// It is the responsibility of the derived class to keep this value up to date as the + /// base class will have no way of knowing the correct value + /// - [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] - public int CurrentItemCount + [Range(0, int.MaxValue, ErrorMessage = "Current item count cannot be less than 0.")] + public int CurrentItemCount + { + get => _currentItemCount; + protected set { - get => _currentItemCount; - protected set + if (value < 0) { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value)); - } - _currentItemCount = value; + throw new ArgumentOutOfRangeException(nameof(value)); } + _currentItemCount = value; } + } - /// - /// Gets the current number of records skipped - /// - public int CurrentSkippedItemCount + /// + /// Gets the current number of records skipped + /// + public int CurrentSkippedItemCount + { + get => _currentSkippedItemCount; + protected set { - get => _currentSkippedItemCount; - protected set + if (value < 0) { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "value cannot be less than 0."); - } + throw new ArgumentOutOfRangeException(nameof(value), "value cannot be less than 0."); } } + } - /// - /// The maximum number of items to transform. Once the transformer has reached this limit, - /// it should stop transforming and signal the end of the sequence. - /// - /// - /// This is useful for transforming a subset of data, especially when the source is large - /// or infinite or during development. - /// - /// The specified value is less than 1 - /// - /// - /// foreach (var item in items.Skip(SkipItemCount).Take(MaxItemCount)) - /// { - /// // Transformer each item and return it - /// } - /// - /// - - public int MaximumItemCount + /// + /// The maximum number of items to transform. Once the transformer has reached this limit, + /// it should stop transforming and signal the end of the sequence. + /// + /// + /// This is useful for transforming a subset of data, especially when the source is large + /// or infinite or during development. + /// + /// The specified value is less than 1 + /// + /// + /// foreach (var item in items.Skip(SkipItemCount).Take(MaxItemCount)) + /// { + /// // Transformer each item and return it + /// } + /// + /// + + public int MaximumItemCount + { + get => _maximumItemCount; + set { - get => _maximumItemCount; - set + if (value < 0) { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Maximum item count cannot be less than 0."); - } - _maximumItemCount = value; + throw new ArgumentOutOfRangeException(nameof(value), "Maximum item count cannot be less than 0."); } + _maximumItemCount = value; } + } - /// - /// The number of items to skip before transforming. - /// The transformer should skip the specified number of items before starting to yield results. - /// - /// - /// This is useful for transforming a subset of data, especially when the source is large - /// or infinite or during development. - /// - /// The specified value is less than 0 - /// - /// - /// foreach (var item in items.Skip(SkipItemCount).Take(MaxItemCount)) - /// { - /// // Transformer each item and return it - /// } - /// - /// - - public int SkipItemCount + /// + /// The number of items to skip before transforming. + /// The transformer should skip the specified number of items before starting to yield results. + /// + /// + /// This is useful for transforming a subset of data, especially when the source is large + /// or infinite or during development. + /// + /// The specified value is less than 0 + /// + /// + /// foreach (var item in items.Skip(SkipItemCount).Take(MaxItemCount)) + /// { + /// // Transformer each item and return it + /// } + /// + /// + + public int SkipItemCount + { + get => _skipItemCount; + set { - get => _skipItemCount; - set + if (value < 0) { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Skip item count cannot be less than 0."); - } - _skipItemCount = value; + throw new ArgumentOutOfRangeException(nameof(value), "Skip item count cannot be less than 0."); } + _skipItemCount = value; } + } - /// - /// Asynchronously transforms data of type TSource to TDestination - /// - /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed - /// - /// IAsyncEnumerable<T> - /// The result may be an empty sequence if no data is available or if the transformation fails. - /// - public virtual IAsyncEnumerable TransformAsync - ( - IAsyncEnumerable items - ) + /// + /// Asynchronously transforms data of type TSource to TDestination + /// + /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed + /// + /// IAsyncEnumerable<T> + /// The result may be an empty sequence if no data is available or if the transformation fails. + /// + public virtual IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items + ) + { + if (items == null) { - if (items == null) - { - throw new ArgumentNullException(nameof(items)); - } - - return TransformWorkerAsync(items, CancellationToken.None); + throw new ArgumentNullException(nameof(items)); } + return TransformWorkerAsync(items, CancellationToken.None); + } + - /// - /// Asynchronously transforms data of type TSource to TDestination - /// - /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed - /// A CancellationToken to observe while waiting for the task to complete - /// - /// IAsyncEnumerable<TDestination> - A list of 0 or more transformed items - /// - /// - /// - public virtual IAsyncEnumerable TransformAsync - ( - IAsyncEnumerable items, - CancellationToken token - ) + /// + /// Asynchronously transforms data of type TSource to TDestination + /// + /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed + /// A CancellationToken to observe while waiting for the task to complete + /// + /// IAsyncEnumerable<TDestination> - A list of 0 or more transformed items + /// + /// + /// + public virtual IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + CancellationToken token + ) + { + if (items == null) { - if (items == null) - { - throw new ArgumentNullException(nameof(items)); - } - return TransformWorkerAsync(items, token); + throw new ArgumentNullException(nameof(items)); } + return TransformWorkerAsync(items, token); + } - /// - /// Asynchronously transforms data of type TSource to TDestination - /// - /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed - /// A provider for progress updates. - /// IAsyncEnumerable<T> The result may be an empty sequence if no data is available or if the transformation fails. - /// - /// The value of progress is null - public virtual IAsyncEnumerable TransformAsync - ( - IAsyncEnumerable items, - IProgress progress - ) + /// + /// Asynchronously transforms data of type TSource to TDestination + /// + /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed + /// A provider for progress updates. + /// IAsyncEnumerable<T> The result may be an empty sequence if no data is available or if the transformation fails. + /// + /// The value of progress is null + public virtual IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + IProgress progress + ) + { + if (items == null) { - if (items == null) - { - throw new ArgumentNullException(nameof(items)); - } + throw new ArgumentNullException(nameof(items)); + } - if (progress == null) - { - throw new ArgumentNullException(nameof(progress)); - } + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } - using var timer = new Timer - ( - _ => progress.Report(CreateProgressReport()), - null, - TimeSpan.Zero, - TimeSpan.FromMilliseconds(ReportingInterval) - ); + using var timer = new Timer + ( + _ => progress.Report(CreateProgressReport()), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(ReportingInterval) + ); - return TransformWorkerAsync(items, CancellationToken.None); - } + return TransformWorkerAsync(items, CancellationToken.None); + } - /// - /// Asynchronously transforms data of type TSource to TDestination - /// - /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed - /// A provider for progress updates. - /// A CancellationToken to observe while waiting for the task to complete. - /// - /// IAsyncEnumerable<T> The result may be an empty sequence if no data is available or if the transformation fails. - /// - /// - /// The transformer should be able to handle cancellation requests gracefully. - /// If the caller doesn't plan on cancelling the transformation, they can pass CancellationToken.None. - /// - /// The value of progress is null - public virtual IAsyncEnumerable TransformAsync - ( - IAsyncEnumerable items, - IProgress progress, - CancellationToken token - ) + /// + /// Asynchronously transforms data of type TSource to TDestination + /// + /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed + /// A provider for progress updates. + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// IAsyncEnumerable<T> The result may be an empty sequence if no data is available or if the transformation fails. + /// + /// + /// The transformer should be able to handle cancellation requests gracefully. + /// If the caller doesn't plan on cancelling the transformation, they can pass CancellationToken.None. + /// + /// The value of progress is null + public virtual IAsyncEnumerable TransformAsync + ( + IAsyncEnumerable items, + IProgress progress, + CancellationToken token + ) + { + if (items == null) { - if (items == null) - { - throw new ArgumentNullException(nameof(items)); - } + throw new ArgumentNullException(nameof(items)); + } - if (progress == null) - { - throw new ArgumentNullException(nameof(progress)); - } + if (progress == null) + { + throw new ArgumentNullException(nameof(progress)); + } - using var timer = new Timer - ( - _ => progress.Report(CreateProgressReport()), - null, - TimeSpan.Zero, - TimeSpan.FromMilliseconds(ReportingInterval) - ); + using var timer = new Timer + ( + _ => progress.Report(CreateProgressReport()), + null, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(ReportingInterval) + ); - return TransformWorkerAsync(items, token); - } + return TransformWorkerAsync(items, token); + } - /// - /// The worker method that performs the actual transformation. - /// - /// - /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed - /// - /// - /// A CancellationToken to observe while waiting for the task to complete. - /// - /// - protected abstract IAsyncEnumerable TransformWorkerAsync - ( - IAsyncEnumerableitems, - CancellationToken token - ); + /// + /// The worker method that performs the actual transformation. + /// + /// + /// IAsyncEnumerable<TSource> - A list of 0 or more items to be transformed + /// + /// + /// A CancellationToken to observe while waiting for the task to complete. + /// + /// + protected abstract IAsyncEnumerable TransformWorkerAsync + ( + IAsyncEnumerableitems, + CancellationToken token + ); - /// - /// Creates a progress report object of type TProgress. - /// - /// - /// TProgress - A new instance of the progress report object. - /// - protected abstract TProgress CreateProgressReport(); + /// + /// Creates a progress report object of type TProgress. + /// + /// + /// TProgress - A new instance of the progress report object. + /// + protected abstract TProgress CreateProgressReport(); - /// - /// Increments the CurrentItemCount in a thread safe manner. - /// - /// - /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not - /// thread safe. This method ensures that CurrentItemCount is incremented safely - /// - protected void IncrementCurrentItemCount() - { - Interlocked.Increment(ref _currentItemCount); - } + /// + /// Increments the CurrentItemCount in a thread safe manner. + /// + /// + /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not + /// thread safe. This method ensures that CurrentItemCount is incremented safely + /// + protected void IncrementCurrentItemCount() + { + Interlocked.Increment(ref _currentItemCount); + } - /// - /// Increments the CurrentItemCount in a thread safe manner. - /// - /// - /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not - /// thread safe. This method ensures that CurrentItemCount is incremented safely - /// - protected void IncrementCurrentSkippedItemCount() - { - Interlocked.Increment(ref _currentSkippedItemCount); - } - + /// + /// Increments the CurrentItemCount in a thread safe manner. + /// + /// + /// Simply calling CurrentItemCount++ or CurrentItemCount += 1 is not + /// thread safe. This method ensures that CurrentItemCount is incremented safely + /// + protected void IncrementCurrentSkippedItemCount() + { + Interlocked.Increment(ref _currentSkippedItemCount); } -} + +} \ No newline at end of file diff --git a/src/Wolfgang.Etl.Abstractions/Wolfgang.Etl.Abstractions.csproj b/src/Wolfgang.Etl.Abstractions/Wolfgang.Etl.Abstractions.csproj index 9138167..2eb59cf 100644 --- a/src/Wolfgang.Etl.Abstractions/Wolfgang.Etl.Abstractions.csproj +++ b/src/Wolfgang.Etl.Abstractions/Wolfgang.Etl.Abstractions.csproj @@ -2,9 +2,11 @@ -net462;net472;net48;net481; -netstandard2.0;netstandard2.1; -net8.0;net9.0 + net462;net472;net48;net481; + netstandard2.0;netstandard2.1; + netcoreapp3.1; + net5.0;net6.0;net7.0; + net8.0;net9.0 latest 0.6.0 @@ -42,7 +44,7 @@ net8.0;net9.0 - + diff --git a/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/LoaderBaseTests.cs b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/LoaderBaseTests.cs index dc3f51c..053c8a0 100644 --- a/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/LoaderBaseTests.cs +++ b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/LoaderBaseTests.cs @@ -274,7 +274,7 @@ public async Task LoadWithProgressAndCancellationAsync_reports_progress_expected public void CurrentItemCount_when_assigned_a_value_less_than_0_throws_ArgumentOutOfRangeException() { - var sut = new ConsoleLoaderFromBase(new List()); + var sut = new ConsoleLoaderFromBase([]); Assert.Throws(() => sut.TestSettingCurrentItemCount(-1)); } @@ -284,7 +284,7 @@ public void CurrentItemCount_when_assigned_a_value_less_than_0_throws_ArgumentOu public void CurrentItemCount_when_assigned_a_valid_value_stores_the_value() { - var sut = new ConsoleLoaderFromBase(new List()); + var sut = new ConsoleLoaderFromBase([]); sut.TestSettingCurrentItemCount(10); Assert.Equal(10, sut.CurrentItemCount); @@ -296,7 +296,7 @@ public void CurrentItemCount_when_assigned_a_valid_value_stores_the_value() public void ReportingInterval_when_assigned_a_value_less_than_0_throws_ArgumentOutOfRangeException() { - var sut = new ConsoleLoaderFromBase(new List()); + var sut = new ConsoleLoaderFromBase([]); Assert.Throws(() => sut.ReportingInterval = -1); } @@ -306,7 +306,7 @@ public void ReportingInterval_when_assigned_a_value_less_than_0_throws_ArgumentO public void ReportingInterval_when_assigned_a_valid_value_stores_the_value() { - var sut = new ConsoleLoaderFromBase(new List()) + var sut = new ConsoleLoaderFromBase([]) { ReportingInterval = 10 }; @@ -319,7 +319,7 @@ public void ReportingInterval_when_assigned_a_valid_value_stores_the_value() public void MaximumItemCount_when_assigned_a_value_less_than_1_throws_ArgumentOutOfRangeException() { - var sut = new ConsoleLoaderFromBase(new List()); + var sut = new ConsoleLoaderFromBase([]); Assert.Throws(() => sut.MaximumItemCount = -1); } @@ -330,7 +330,7 @@ public void MaximumItemCount_when_assigned_a_value_less_than_1_throws_ArgumentOu public void MaximumItemCount_when_assigned_a_valid_value_stores_the_value() { - var sut = new ConsoleLoaderFromBase(new List()) + var sut = new ConsoleLoaderFromBase([]) { MaximumItemCount = 10 }; @@ -343,7 +343,7 @@ public void MaximumItemCount_when_assigned_a_valid_value_stores_the_value() public void SkipItemCount_when_assigned_a_value_less_than_0_throws_ArgumentOutOfRangeException() { - var sut = new ConsoleLoaderFromBase(new List()); + var sut = new ConsoleLoaderFromBase([]); Assert.Throws(() => sut.SkipItemCount = -1); } @@ -353,7 +353,7 @@ public void SkipItemCount_when_assigned_a_value_less_than_0_throws_ArgumentOutOf public void SkipItemCount_when_assigned_a_valid_value_stores_the_value() { - var sut = new ConsoleLoaderFromBase(new List()) + var sut = new ConsoleLoaderFromBase([]) { SkipItemCount = 10 }; diff --git a/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/TransformerBaseTests.cs b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/TransformerBaseTests.cs index 1eff34c..c9e62af 100644 --- a/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/TransformerBaseTests.cs +++ b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/BaseClassTests/TransformerBaseTests.cs @@ -379,10 +379,7 @@ [EnumeratorCancellation] CancellationToken token await foreach (var item in items.WithCancellation(token)) { await Task.Delay(_delay, token); // Simulate some delay in processing - if (token.IsCancellationRequested) - { - throw new OperationCanceledException(token); - } + token.ThrowIfCancellationRequested(); yield return item.ToString(); ++CurrentItemCount; } diff --git a/tests/Wolfgang.Etl.Abstractions.Tests.Unit/Wolfgang.Etl.Abstractions.Tests.Unit.csproj b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/Wolfgang.Etl.Abstractions.Tests.Unit.csproj index 6a7d6ee..f7b90e5 100644 --- a/tests/Wolfgang.Etl.Abstractions.Tests.Unit/Wolfgang.Etl.Abstractions.Tests.Unit.csproj +++ b/tests/Wolfgang.Etl.Abstractions.Tests.Unit/Wolfgang.Etl.Abstractions.Tests.Unit.csproj @@ -3,9 +3,9 @@ net462;net472;net48;net481; - netcoreapp3.1; - net50;net6.0;net7.0; - net8.0;net9.0 + netcoreapp3.1; + net5.0;net6.0;net7.0; + net8.0;net9.0;net10.0 1.0.0 latest @@ -15,28 +15,74 @@ false true - Copyright {copyright year} {author} + Copyright 2025 Chris Wolfgang - + [17.13.0] - - all runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + [2.8.2] - all + all runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + + +