diff --git a/.github/scripts/process-mock-benchmarks.js b/.github/scripts/process-mock-benchmarks.js new file mode 100644 index 0000000000..13822ba206 --- /dev/null +++ b/.github/scripts/process-mock-benchmarks.js @@ -0,0 +1,384 @@ +const fs = require('fs'); +const path = require('path'); + +const RESULTS_DIR = 'mock-benchmark-results'; +const OUTPUT_DIR = 'docs/docs/benchmarks/mocks'; +const STATIC_DIR = 'docs/static/benchmarks/mocks'; + +console.log('🚀 Processing mock benchmark results...\n'); + +// Ensure output directories exist +[OUTPUT_DIR, STATIC_DIR].forEach(dir => { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } +}); + +function findMarkdownFiles(dir) { + const files = []; + + function walk(currentPath) { + if (!fs.existsSync(currentPath)) return; + + const entries = fs.readdirSync(currentPath, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(currentPath, entry.name); + if (entry.isDirectory()) { + walk(fullPath); + } else if (entry.name.endsWith('.md')) { + files.push(fullPath); + } + } + } + + walk(dir); + return files; +} + +function parseMarkdownTable(content) { + const lines = content.split('\n'); + const tableStart = lines.findIndex(l => l.includes('| Method') || l.includes('| Description')); + if (tableStart === -1) return null; + + const headers = lines[tableStart].split('|').map(h => h.trim()).filter(Boolean); + const data = []; + + for (let i = tableStart + 2; i < lines.length; i++) { + const line = lines[i].trim(); + if (!line.startsWith('|')) break; + + const values = line.split('|').map(v => v.trim()).filter(Boolean); + if (values.length === headers.length) { + const row = {}; + headers.forEach((header, idx) => { + row[header] = values[idx]; + }); + data.push(row); + } + } + + return data; +} + +function extractEnvironmentInfo(content) { + const lines = content.split('\n'); + const envStart = lines.findIndex(l => l.includes('BenchmarkDotNet')); + if (envStart === -1) return {}; + + const info = {}; + for (let i = envStart; i < Math.min(envStart + 10, lines.length); i++) { + const line = lines[i]; + if (line.includes('BenchmarkDotNet')) info.benchmarkDotNetVersion = line.trim(); + if (line.includes('OS:')) info.os = line.split(':')[1]?.trim(); + if (line.includes('.NET SDK')) info.sdk = line.trim(); + if (line.includes('Host')) info.host = line.split(':')[1]?.trim(); + } + + return info; +} + +function parseMeanValue(meanStr) { + if (!meanStr) return 0; + const cleaned = meanStr.replace(/,/g, ''); + const match = cleaned.match(/[\d.]+/); + return match ? parseFloat(match[0]) : 0; +} + +function getUnit(meanStr) { + if (!meanStr) return 'ns'; + if (meanStr.includes(' s') && !meanStr.includes('ms') && !meanStr.includes('ns') && !meanStr.includes('Ξs')) return 's'; + if (meanStr.includes('ms')) return 'ms'; + if (meanStr.includes('Ξs') || meanStr.includes('us')) return 'Ξs'; + return 'ns'; +} + +// Map from benchmark class names to friendly category names +const categoryMap = { + 'MockCreationBenchmarks': 'MockCreation', + 'SetupBenchmarks': 'Setup', + 'InvocationBenchmarks': 'Invocation', + 'VerificationBenchmarks': 'Verification', + 'CallbackBenchmarks': 'Callback', + 'CombinedWorkflowBenchmarks': 'CombinedWorkflow' +}; + +const categoryDescriptions = { + 'MockCreation': 'Mock instance creation performance', + 'Setup': 'Mock behavior configuration (returns, matchers)', + 'Invocation': 'Calling methods on mock objects', + 'Verification': 'Verifying mock method calls', + 'Callback': 'Callback registration and execution', + 'CombinedWorkflow': 'Full workflow: create → setup → invoke → verify' +}; + +// Process results +const categories = {}; +let environmentInfo = {}; + +console.log('📊 Processing mock benchmark results...'); +const allFiles = findMarkdownFiles(RESULTS_DIR); +console.log(` Found ${allFiles.length} markdown files`); + +if (allFiles.length > 0) { + console.log(' Sample paths:'); + allFiles.slice(0, 3).forEach(f => console.log(` ${f}`)); +} + +allFiles.forEach(file => { + const content = fs.readFileSync(file, 'utf8'); + const data = parseMarkdownTable(content); + + if (!environmentInfo.os) { + environmentInfo = extractEnvironmentInfo(content); + } + + if (data && data.length > 0) { + // Extract category from artifact directory path + // Path structure: mock-benchmark-results/mock_benchmark_/.../*.md + let category = null; + for (const [className, catName] of Object.entries(categoryMap)) { + if (file.includes(className) || file.includes(`mock_benchmark_${className}`)) { + category = catName; + break; + } + } + + if (category) { + // Merge results if we already have some for this category + if (categories[category]) { + categories[category] = categories[category].concat(data); + } else { + categories[category] = data; + } + console.log(` ✓ Processed ${category}: ${data.length} entries`); + } else { + console.warn(` ⚠ïļ Could not extract category from file path: ${file}`); + } + } +}); + +const timestamp = new Date().toISOString().split('T')[0]; + +// Generate individual benchmark pages for each category +console.log('\n📝 Generating documentation...'); + +Object.entries(categories).forEach(([category, data], index) => { + const description = categoryDescriptions[category] || category; + const unit = getUnit(data[0]?.Mean); + const maxMean = Math.max(...data.map(d => parseMeanValue(d.Mean))); + + const benchmarkPage = `--- +title: "Mock Benchmark: ${category}" +description: "${description} — TUnit.Mocks vs Moq vs NSubstitute vs FakeItEasy" +sidebar_position: ${index + 2} +--- + +# ${category} Benchmark + +:::info Last Updated +This benchmark was automatically generated on **${timestamp}** from the latest CI run. + +**Environment:** ${environmentInfo.os || 'Ubuntu Latest'} â€Ē ${environmentInfo.sdk || '.NET 10'} +::: + +## 📊 Results + +${description}: + +| Method | Mean | Error | StdDev | Allocated | +|--------|------|-------|--------|-----------| +${data.map(row => { + const name = (row.Description || row.Method || '').includes('TUnit') ? `**${row.Description || row.Method}**` : (row.Description || row.Method); + return `| ${name} | ${row.Mean || 'N/A'} | ${row.Error || 'N/A'} | ${row.StdDev || 'N/A'} | ${row.Allocated || 'N/A'} |`; +}).join('\n')} + +## 📈 Visual Comparison + +\`\`\`mermaid +%%{init: { + 'theme':'base', + 'themeVariables': { + 'primaryColor': '#2563eb', + 'primaryTextColor': '#1f2937', + 'primaryBorderColor': '#1e40af', + 'lineColor': '#6b7280', + 'secondaryColor': '#7c3aed', + 'tertiaryColor': '#dc2626', + 'background': '#ffffff', + 'pie1': '#2563eb', + 'pie2': '#7c3aed', + 'pie3': '#dc2626', + 'pie4': '#f59e0b', + 'pie5': '#10b981', + 'pie6': '#06b6d4', + 'pie7': '#ec4899', + 'pie8': '#6366f1' + } +}}%% +xychart-beta + title "${category} Performance Comparison" + x-axis [${data.map(d => `"${(d.Description || d.Method || '').replace(/"/g, "'")}"` ).join(', ')}] + y-axis "Time (${unit})" 0 --> ${Math.ceil(maxMean * 1.2) || 100} + bar [${data.map(d => parseMeanValue(d.Mean)).join(', ')}] +\`\`\` + +## ðŸŽŊ Key Insights + +This benchmark compares **TUnit.Mocks** (source-generated) against runtime proxy-based mocking libraries for ${description.toLowerCase()}. + +--- + +:::note Methodology +View the [mock benchmarks overview](/docs/benchmarks/mocks) for methodology details and environment information. +::: + +*Last generated: ${new Date().toISOString()}* +`; + + fs.writeFileSync(path.join(OUTPUT_DIR, `${category}.md`), benchmarkPage); + console.log(` ✓ Created ${OUTPUT_DIR}/${category}.md`); + + // Generate individual JSON file + const benchmarkJson = { + timestamp: new Date().toISOString(), + category, + description, + environment: environmentInfo, + results: data + }; + + fs.writeFileSync( + path.join(STATIC_DIR, `${category}.json`), + JSON.stringify(benchmarkJson, null, 2) + ); + console.log(` ✓ Created ${STATIC_DIR}/${category}.json`); +}); + +// Generate index/overview page +const indexPage = `--- +title: Mock Library Benchmarks +description: Performance comparisons between TUnit.Mocks, Moq, NSubstitute, and FakeItEasy +sidebar_position: 1 +--- + +# Mock Library Benchmarks + +:::info Last Updated +These benchmarks were automatically generated on **${timestamp}** from the latest CI run. + +**Environment:** ${environmentInfo.os || 'Ubuntu Latest'} â€Ē ${environmentInfo.sdk || '.NET 10'} +::: + +## 🚀 Overview + +These benchmarks compare **TUnit.Mocks** (source-generated, AOT-compatible) against the most popular .NET mocking libraries that use runtime proxy generation: + +| Library | Approach | AOT Compatible | +|---------|----------|----------------| +| **TUnit.Mocks** | Source-generated at compile time | ✅ Yes | +| **Moq** | Runtime proxy via Castle.DynamicProxy | ❌ No | +| **NSubstitute** | Runtime proxy via Castle.DynamicProxy | ❌ No | +| **FakeItEasy** | Runtime proxy via Castle.DynamicProxy | ❌ No | + +## 📊 Benchmark Categories + +Click on any benchmark to view detailed results: + +${Object.keys(categories).map(category => + `- [${category}](${category}) - ${categoryDescriptions[category] || category}` +).join('\n')} + +## 📈 What's Measured + +Each benchmark category tests a specific aspect of mocking library usage: + +- **MockCreation** — How fast can each library create a mock instance? +- **Setup** — How fast can you configure return values and argument matchers? +- **Invocation** — Once set up, how fast are method calls on the mock? +- **Verification** — How fast can you verify that methods were called correctly? +- **Callback** — How fast are callbacks triggered during mock invocations? +- **CombinedWorkflow** — The full real-world pattern: create → setup → invoke → verify + +## 🔧 Methodology + +- **Tool**: ${environmentInfo.benchmarkDotNetVersion || 'BenchmarkDotNet'} +- **OS**: ${environmentInfo.os || 'Ubuntu Latest (GitHub Actions)'} +- **Runtime**: ${environmentInfo.host || '.NET 10'} +- **Statistical Rigor**: Multiple iterations with warm-up and outlier detection +- **Memory**: Allocation tracking enabled via \`[MemoryDiagnoser]\` + +### Why Source-Generated Mocks? + +TUnit.Mocks generates mock implementations at compile time, eliminating: +- Runtime proxy generation overhead +- Dynamic assembly emission +- Reflection-based method dispatch + +This makes TUnit.Mocks compatible with **Native AOT** and **IL trimming**, while also providing performance benefits for standard .NET execution. + +### Source Code + +All benchmark source code is available in the [\`TUnit.Mocks.Benchmarks\`](https://github.com/thomhurst/TUnit/tree/main/TUnit.Mocks.Benchmarks) directory. + +--- + +:::note Continuous Benchmarking +These benchmarks run automatically daily via [GitHub Actions](https://github.com/thomhurst/TUnit/actions/workflows/mock-benchmarks.yml). + +Each benchmark runs multiple iterations with statistical analysis to ensure accuracy. Results may vary based on hardware and test characteristics. +::: + +*Last generated: ${new Date().toISOString()}* +`; + +fs.writeFileSync(path.join(OUTPUT_DIR, 'index.md'), indexPage); +console.log(` ✓ Created ${OUTPUT_DIR}/index.md`); + +// Generate latest.json +const benchmarkData = { + timestamp: new Date().toISOString(), + environment: environmentInfo, + categories, + stats: { + categoryCount: Object.keys(categories).length, + totalBenchmarks: Object.values(categories).reduce((sum, arr) => sum + arr.length, 0), + lastUpdated: new Date().toISOString() + } +}; + +fs.writeFileSync( + path.join(STATIC_DIR, 'latest.json'), + JSON.stringify(benchmarkData, null, 2) +); +console.log(` ✓ Created ${STATIC_DIR}/latest.json`); + +// Generate summary.json +const summary = { + categories: Object.keys(categories), + timestamp, + environment: `${environmentInfo.os || 'Ubuntu Latest'} â€Ē ${environmentInfo.sdk || '.NET 10'}`, + libraries: ['TUnit.Mocks', 'Moq', 'NSubstitute', 'FakeItEasy'] +}; + +fs.writeFileSync( + path.join(STATIC_DIR, 'summary.json'), + JSON.stringify(summary, null, 2) +); +console.log(` ✓ Created ${STATIC_DIR}/summary.json`); + +// Summary +console.log('\n✅ Mock benchmark processing complete!\n'); +console.log(`Summary:`); +console.log(` - Categories: ${Object.keys(categories).length}`); +console.log(` - Total entries: ${Object.values(categories).reduce((sum, arr) => sum + arr.length, 0)}`); +console.log(` - Markdown pages generated: ${Object.keys(categories).length + 1}`); +console.log(` - JSON files generated: ${Object.keys(categories).length + 2}`); + +console.log(`\n📊 Mock benchmarks produced:`); +Object.keys(categories).forEach(cat => console.log(` - ${cat} (${categories[cat].length} entries)`)); + +if (Object.keys(categories).length === 0) { + console.warn('\n⚠ïļ WARNING: No mock benchmark categories were found!'); + console.warn('This likely means the artifact directory structure is not as expected.'); + console.warn(`Expected structure: ${RESULTS_DIR}/mock_benchmark_/`); +} diff --git a/.github/workflows/mock-benchmarks.yml b/.github/workflows/mock-benchmarks.yml new file mode 100644 index 0000000000..9a502fb7dc --- /dev/null +++ b/.github/workflows/mock-benchmarks.yml @@ -0,0 +1,213 @@ +name: Mock Benchmarks + +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + run-mock-benchmarks: + environment: ${{ github.ref == 'refs/heads/main' && 'Production' || 'Pull Requests' }} + strategy: + matrix: + benchmark: [MockCreationBenchmarks, SetupBenchmarks, InvocationBenchmarks, VerificationBenchmarks, CallbackBenchmarks, CombinedWorkflowBenchmarks] + fail-fast: false + runs-on: ubuntu-latest + concurrency: + group: "mock-benchmarks-${{ matrix.benchmark }}" + cancel-in-progress: true + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.x + + - name: Run Benchmark + run: dotnet run -c Release -- --filter "*${{ matrix.benchmark }}*" --exporters json github + working-directory: "TUnit.Mocks.Benchmarks" + + - name: Upload Results + uses: actions/upload-artifact@v7 + if: always() + with: + name: mock_benchmark_${{ matrix.benchmark }} + path: | + **/BenchmarkDotNet.Artifacts/** + + process-and-upload-mock-benchmarks: + needs: [run-mock-benchmarks] + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + permissions: + contents: write + pull-requests: write + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + token: ${{ secrets.ADMIN_TOKEN }} + + - name: Download All Mock Benchmark Artifacts + uses: actions/download-artifact@v8 + with: + path: mock-benchmark-results/ + pattern: mock_benchmark_* + merge-multiple: false + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: '24' + + - name: Process Mock Benchmark Results + run: | + node .github/scripts/process-mock-benchmarks.js + + - name: Upload MockCreation Benchmark + uses: actions/upload-artifact@v7 + if: always() + with: + name: mock-benchmark-MockCreation + path: | + docs/docs/benchmarks/mocks/MockCreation.md + docs/static/benchmarks/mocks/MockCreation.json + retention-days: 90 + + - name: Upload Setup Benchmark + uses: actions/upload-artifact@v7 + if: always() + with: + name: mock-benchmark-Setup + path: | + docs/docs/benchmarks/mocks/Setup.md + docs/static/benchmarks/mocks/Setup.json + retention-days: 90 + + - name: Upload Invocation Benchmark + uses: actions/upload-artifact@v7 + if: always() + with: + name: mock-benchmark-Invocation + path: | + docs/docs/benchmarks/mocks/Invocation.md + docs/static/benchmarks/mocks/Invocation.json + retention-days: 90 + + - name: Upload Verification Benchmark + uses: actions/upload-artifact@v7 + if: always() + with: + name: mock-benchmark-Verification + path: | + docs/docs/benchmarks/mocks/Verification.md + docs/static/benchmarks/mocks/Verification.json + retention-days: 90 + + - name: Upload Callback Benchmark + uses: actions/upload-artifact@v7 + if: always() + with: + name: mock-benchmark-Callback + path: | + docs/docs/benchmarks/mocks/Callback.md + docs/static/benchmarks/mocks/Callback.json + retention-days: 90 + + - name: Upload CombinedWorkflow Benchmark + uses: actions/upload-artifact@v7 + if: always() + with: + name: mock-benchmark-CombinedWorkflow + path: | + docs/docs/benchmarks/mocks/CombinedWorkflow.md + docs/static/benchmarks/mocks/CombinedWorkflow.json + retention-days: 90 + + - name: Upload Summary Files + uses: actions/upload-artifact@v7 + if: always() + with: + name: mock-benchmark-summary + path: | + docs/docs/benchmarks/mocks/index.md + docs/static/benchmarks/mocks/latest.json + docs/static/benchmarks/mocks/summary.json + retention-days: 90 + + - name: Check for Changes + id: check_changes + run: | + git add docs/docs/benchmarks/mocks/ docs/static/benchmarks/mocks/ + if git diff --quiet --staged; then + echo "has_changes=false" >> $GITHUB_OUTPUT + echo "No mock benchmark changes detected" + else + echo "has_changes=true" >> $GITHUB_OUTPUT + echo "Mock benchmark changes detected" + fi + + - name: Create Pull Request + if: steps.check_changes.outputs.has_changes == 'true' + id: create_pr + uses: peter-evans/create-pull-request@v8 + with: + token: ${{ secrets.ADMIN_TOKEN }} + commit-message: 'chore: update mock benchmark results' + branch: automated-mock-benchmarks-update + delete-branch: true + title: 'ðŸĪ– Update Mock Benchmark Results' + body: | + ## Automated Mock Benchmark Update + + This PR updates the mock benchmark documentation with the latest results from the Mock Benchmarks workflow. + + ### Benchmarks Produced + + Individual benchmark artifacts are available for download: + - `mock-benchmark-MockCreation` + - `mock-benchmark-Setup` + - `mock-benchmark-Invocation` + - `mock-benchmark-Verification` + - `mock-benchmark-Callback` + - `mock-benchmark-CombinedWorkflow` + - `mock-benchmark-summary` (aggregated overview) + + ### Libraries Compared + - **TUnit.Mocks** (source-generated, AOT-compatible) + - **Moq** (runtime proxy generation) + - **NSubstitute** (runtime proxy generation) + - **FakeItEasy** (runtime proxy generation) + + ### Changes + - Updated mock benchmark data in `docs/static/benchmarks/mocks/latest.json` + - Regenerated mock benchmark documentation in `docs/docs/benchmarks/mocks/` + - Updated mock benchmark summary in `docs/static/benchmarks/mocks/summary.json` + + ### Workflow Run + - **Run ID**: ${{ github.run_id }} + - **Triggered**: ${{ github.event_name }} + + --- + + ðŸĪ– This PR was automatically created and will be merged automatically once CI checks pass. + labels: | + automated + benchmarks + documentation + ignore-for-release + draft: false + + - name: Merge PR Immediately + if: steps.check_changes.outputs.has_changes == 'true' && steps.create_pr.outputs.pull-request-number != '' + env: + GH_TOKEN: ${{ secrets.ADMIN_TOKEN }} + run: | + # Wait a moment for PR to be fully created + sleep 5 + gh pr merge ${{ steps.create_pr.outputs.pull-request-number }} --squash --delete-branch --admin diff --git a/Directory.Packages.props b/Directory.Packages.props index ee86beb1a3..ee772db5fc 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -18,6 +18,7 @@ + @@ -54,6 +55,7 @@ + diff --git a/TUnit.CI.slnx b/TUnit.CI.slnx index d7d8bd8f79..1cb546d87b 100644 --- a/TUnit.CI.slnx +++ b/TUnit.CI.slnx @@ -79,6 +79,11 @@ + + + + + diff --git a/TUnit.Mocks.Benchmarks/CallbackBenchmarks.cs b/TUnit.Mocks.Benchmarks/CallbackBenchmarks.cs new file mode 100644 index 0000000000..88375ee2c5 --- /dev/null +++ b/TUnit.Mocks.Benchmarks/CallbackBenchmarks.cs @@ -0,0 +1,125 @@ +using BenchmarkDotNet.Attributes; +using FakeItEasy; +using Moq; +using NSubstitute; + +namespace TUnit.Mocks.Benchmarks; + +[MemoryDiagnoser] +[SimpleJob] +[JsonExporterAttribute.Full] +[MarkdownExporterAttribute.GitHub] +public class CallbackBenchmarks +{ + [Benchmark(Description = "TUnit.Mocks")] + public int TUnitMocks_Callback() + { + var count = 0; + var mock = Mock.Of(); + mock.Send(TUnitArg.Any(), TUnitArg.Any()) + .Callback(() => count++); + + var svc = mock.Object; + svc.Send("user@test.com", "Hello"); + svc.Send("user@test.com", "World"); + return count; + } + + [Benchmark(Description = "Moq")] + public int Moq_Callback() + { + var count = 0; + var mock = new Moq.Mock(); + mock.Setup(x => x.Send(It.IsAny(), It.IsAny())) + .Callback(() => count++); + + var svc = mock.Object; + svc.Send("user@test.com", "Hello"); + svc.Send("user@test.com", "World"); + return count; + } + + [Benchmark(Description = "NSubstitute")] + public int NSubstitute_Callback() + { + var count = 0; + var sub = Substitute.For(); + sub.When(x => x.Send(NSubstitute.Arg.Any(), NSubstitute.Arg.Any())) + .Do(_ => count++); + + sub.Send("user@test.com", "Hello"); + sub.Send("user@test.com", "World"); + return count; + } + + [Benchmark(Description = "FakeItEasy")] + public int FakeItEasy_Callback() + { + var count = 0; + var fake = A.Fake(); + A.CallTo(() => fake.Send(A.Ignored, A.Ignored)) + .Invokes(() => count++); + + fake.Send("user@test.com", "Hello"); + fake.Send("user@test.com", "World"); + return count; + } + + [Benchmark(Description = "TUnit.Mocks (with args)")] + public string TUnitMocks_CallbackWithArgs() + { + var lastMessage = ""; + var mock = Mock.Of(); + mock.Log(TUnitArg.Any(), TUnitArg.Any()) + .Callback(() => lastMessage = "logged"); + + var logger = mock.Object; + logger.Log("INFO", "Test message 1"); + logger.Log("WARN", "Test message 2"); + logger.Log("ERROR", "Test message 3"); + return lastMessage; + } + + [Benchmark(Description = "Moq (with args)")] + public string Moq_CallbackWithArgs() + { + var lastMessage = ""; + var mock = new Moq.Mock(); + mock.Setup(x => x.Log(It.IsAny(), It.IsAny())) + .Callback((level, msg) => lastMessage = msg); + + var logger = mock.Object; + logger.Log("INFO", "Test message 1"); + logger.Log("WARN", "Test message 2"); + logger.Log("ERROR", "Test message 3"); + return lastMessage; + } + + [Benchmark(Description = "NSubstitute (with args)")] + public string NSubstitute_CallbackWithArgs() + { + var lastMessage = ""; + var sub = Substitute.For(); + sub.When(x => x.Log(NSubstitute.Arg.Any(), NSubstitute.Arg.Any())) + .Do(callInfo => lastMessage = callInfo.ArgAt(1)); + + sub.Log("INFO", "Test message 1"); + sub.Log("WARN", "Test message 2"); + sub.Log("ERROR", "Test message 3"); + return lastMessage; + } + + [Benchmark(Description = "FakeItEasy (with args)")] + public string FakeItEasy_CallbackWithArgs() + { + var lastMessage = ""; + var fake = A.Fake(); + A.CallTo(() => fake.Log(A.Ignored, A.Ignored)) + .Invokes((string level, string msg) => lastMessage = msg); + + fake.Log("INFO", "Test message 1"); + fake.Log("WARN", "Test message 2"); + fake.Log("ERROR", "Test message 3"); + return lastMessage; + } +} diff --git a/TUnit.Mocks.Benchmarks/CombinedWorkflowBenchmarks.cs b/TUnit.Mocks.Benchmarks/CombinedWorkflowBenchmarks.cs new file mode 100644 index 0000000000..03ccb46249 --- /dev/null +++ b/TUnit.Mocks.Benchmarks/CombinedWorkflowBenchmarks.cs @@ -0,0 +1,122 @@ +using BenchmarkDotNet.Attributes; +using FakeItEasy; +using Moq; +using NSubstitute; + +namespace TUnit.Mocks.Benchmarks; + +/// +/// Simulates a real-world test scenario: create mock, setup, invoke, verify. +/// +[MemoryDiagnoser] +[SimpleJob] +[JsonExporterAttribute.Full] +[MarkdownExporterAttribute.GitHub] +public class CombinedWorkflowBenchmarks +{ + [Benchmark(Description = "TUnit.Mocks")] + public void TUnitMocks_FullWorkflow() + { + // Create + var repoMock = Mock.Of(); + var loggerMock = Mock.Of(); + + // Setup + repoMock.GetById(1).Returns(new User { Id = 1, Name = "Alice", Email = "alice@test.com" }); + repoMock.Exists(1).Returns(true); + loggerMock.IsEnabled(TUnitArg.Any()).Returns(true); + + // Invoke + var repo = repoMock.Object; + var logger = loggerMock.Object; + + var user = repo.GetById(1); + var exists = repo.Exists(1); + logger.Log("INFO", $"User {user!.Name} exists: {exists}"); + repo.Save(new User { Id = 2, Name = "Bob" }); + + // Verify + repoMock.GetById(1).WasCalled(Times.Once); + repoMock.Exists(1).WasCalled(Times.Once); + repoMock.Save(TUnitArg.Any()).WasCalled(Times.Once); + loggerMock.Log(TUnitArg.Any(), TUnitArg.Any()).WasCalled(Times.Once); + } + + [Benchmark(Description = "Moq")] + public void Moq_FullWorkflow() + { + // Create + var repoMock = new Moq.Mock(); + var loggerMock = new Moq.Mock(); + + // Setup + repoMock.Setup(x => x.GetById(1)).Returns(new User { Id = 1, Name = "Alice", Email = "alice@test.com" }); + repoMock.Setup(x => x.Exists(1)).Returns(true); + loggerMock.Setup(x => x.IsEnabled(It.IsAny())).Returns(true); + + // Invoke + var repo = repoMock.Object; + var logger = loggerMock.Object; + + var user = repo.GetById(1); + var exists = repo.Exists(1); + logger.Log("INFO", $"User {user!.Name} exists: {exists}"); + repo.Save(new User { Id = 2, Name = "Bob" }); + + // Verify + repoMock.Verify(x => x.GetById(1), Moq.Times.Once()); + repoMock.Verify(x => x.Exists(1), Moq.Times.Once()); + repoMock.Verify(x => x.Save(It.IsAny()), Moq.Times.Once()); + loggerMock.Verify(x => x.Log(It.IsAny(), It.IsAny()), Moq.Times.Once()); + } + + [Benchmark(Description = "NSubstitute")] + public void NSubstitute_FullWorkflow() + { + // Create + var repo = Substitute.For(); + var logger = Substitute.For(); + + // Setup + repo.GetById(1).Returns(new User { Id = 1, Name = "Alice", Email = "alice@test.com" }); + repo.Exists(1).Returns(true); + logger.IsEnabled(NSubstitute.Arg.Any()).Returns(true); + + // Invoke + var user = repo.GetById(1); + var exists = repo.Exists(1); + logger.Log("INFO", $"User {user!.Name} exists: {exists}"); + repo.Save(new User { Id = 2, Name = "Bob" }); + + // Verify + repo.Received(1).GetById(1); + repo.Received(1).Exists(1); + repo.Received(1).Save(NSubstitute.Arg.Any()); + logger.Received(1).Log(NSubstitute.Arg.Any(), NSubstitute.Arg.Any()); + } + + [Benchmark(Description = "FakeItEasy")] + public void FakeItEasy_FullWorkflow() + { + // Create + var repo = A.Fake(); + var logger = A.Fake(); + + // Setup + A.CallTo(() => repo.GetById(1)).Returns(new User { Id = 1, Name = "Alice", Email = "alice@test.com" }); + A.CallTo(() => repo.Exists(1)).Returns(true); + A.CallTo(() => logger.IsEnabled(A.Ignored)).Returns(true); + + // Invoke + var user = repo.GetById(1); + var exists = repo.Exists(1); + logger.Log("INFO", $"User {user!.Name} exists: {exists}"); + repo.Save(new User { Id = 2, Name = "Bob" }); + + // Verify + A.CallTo(() => repo.GetById(1)).MustHaveHappenedOnceExactly(); + A.CallTo(() => repo.Exists(1)).MustHaveHappenedOnceExactly(); + A.CallTo(() => repo.Save(A.Ignored)).MustHaveHappenedOnceExactly(); + A.CallTo(() => logger.Log(A.Ignored, A.Ignored)).MustHaveHappenedOnceExactly(); + } +} diff --git a/TUnit.Mocks.Benchmarks/GlobalUsings.cs b/TUnit.Mocks.Benchmarks/GlobalUsings.cs new file mode 100644 index 0000000000..c3970493a4 --- /dev/null +++ b/TUnit.Mocks.Benchmarks/GlobalUsings.cs @@ -0,0 +1,4 @@ +global using TUnit.Mocks; +global using TUnit.Mocks.Generated; +global using TUnitArg = TUnit.Mocks.Arguments.Arg; +global using NSubArg = NSubstitute.Arg; diff --git a/TUnit.Mocks.Benchmarks/Interfaces.cs b/TUnit.Mocks.Benchmarks/Interfaces.cs new file mode 100644 index 0000000000..12d233a26a --- /dev/null +++ b/TUnit.Mocks.Benchmarks/Interfaces.cs @@ -0,0 +1,42 @@ +namespace TUnit.Mocks.Benchmarks; + +/// +/// Common interfaces used across all mock benchmarks. +/// Each mocking library will mock these same interfaces. +/// +public interface ICalculatorService +{ + int Add(int a, int b); + double Divide(double numerator, double denominator); + string Format(int value); +} + +public interface IUserRepository +{ + User? GetById(int id); + IReadOnlyList GetAll(); + void Save(User user); + void Delete(int id); + bool Exists(int id); +} + +public interface INotificationService +{ + void Send(string recipient, string message); + Task SendAsync(string recipient, string message); + bool IsAvailable(); +} + +public interface ILogger +{ + void Log(string level, string message); + void LogError(string message, Exception exception); + bool IsEnabled(string level); +} + +public class User +{ + public int Id { get; set; } + public string Name { get; set; } = string.Empty; + public string Email { get; set; } = string.Empty; +} diff --git a/TUnit.Mocks.Benchmarks/InvocationBenchmarks.cs b/TUnit.Mocks.Benchmarks/InvocationBenchmarks.cs new file mode 100644 index 0000000000..4cf1092e69 --- /dev/null +++ b/TUnit.Mocks.Benchmarks/InvocationBenchmarks.cs @@ -0,0 +1,142 @@ +using BenchmarkDotNet.Attributes; +using FakeItEasy; +using Moq; +using NSubstitute; + +namespace TUnit.Mocks.Benchmarks; + +[MemoryDiagnoser] +[SimpleJob] +[JsonExporterAttribute.Full] +[MarkdownExporterAttribute.GitHub] +public class InvocationBenchmarks +{ + private Mock? _tunitMock; + private ICalculatorService? _tunitObject; + private Moq.Mock? _moqMock; + private ICalculatorService? _moqObject; + private ICalculatorService? _nsubObject; + private ICalculatorService? _fakeObject; + + [GlobalSetup] + public void Setup() + { + // TUnit.Mocks + _tunitMock = Mock.Of(); + _tunitMock.Add(TUnitArg.Any(), TUnitArg.Any()).Returns(42); + _tunitMock.Format(TUnitArg.Any()).Returns("formatted"); + _tunitObject = _tunitMock.Object; + + // Moq + _moqMock = new Moq.Mock(); + _moqMock.Setup(x => x.Add(It.IsAny(), It.IsAny())).Returns(42); + _moqMock.Setup(x => x.Format(It.IsAny())).Returns("formatted"); + _moqObject = _moqMock.Object; + + // NSubstitute + _nsubObject = Substitute.For(); + _nsubObject.Add(NSubstitute.Arg.Any(), NSubstitute.Arg.Any()).Returns(42); + _nsubObject.Format(NSubstitute.Arg.Any()).Returns("formatted"); + + // FakeItEasy + _fakeObject = A.Fake(); + A.CallTo(() => _fakeObject.Add(A.Ignored, A.Ignored)).Returns(42); + A.CallTo(() => _fakeObject.Format(A.Ignored)).Returns("formatted"); + } + + [Benchmark(Description = "TUnit.Mocks")] + public int TUnitMocks_Invoke() + { + return _tunitObject!.Add(1, 2); + } + + [Benchmark(Description = "Moq")] + public int Moq_Invoke() + { + return _moqObject!.Add(1, 2); + } + + [Benchmark(Description = "NSubstitute")] + public int NSubstitute_Invoke() + { + return _nsubObject!.Add(1, 2); + } + + [Benchmark(Description = "FakeItEasy")] + public int FakeItEasy_Invoke() + { + return _fakeObject!.Add(1, 2); + } + + [Benchmark(Description = "TUnit.Mocks (String)")] + public string TUnitMocks_InvokeString() + { + return _tunitObject!.Format(42); + } + + [Benchmark(Description = "Moq (String)")] + public string Moq_InvokeString() + { + return _moqObject!.Format(42); + } + + [Benchmark(Description = "NSubstitute (String)")] + public string NSubstitute_InvokeString() + { + return _nsubObject!.Format(42); + } + + [Benchmark(Description = "FakeItEasy (String)")] + public string FakeItEasy_InvokeString() + { + return _fakeObject!.Format(42); + } + + [Benchmark(Description = "TUnit.Mocks (100 calls)")] + public int TUnitMocks_ManyInvocations() + { + var sum = 0; + for (var i = 0; i < 100; i++) + { + sum += _tunitObject!.Add(i, i); + } + + return sum; + } + + [Benchmark(Description = "Moq (100 calls)")] + public int Moq_ManyInvocations() + { + var sum = 0; + for (var i = 0; i < 100; i++) + { + sum += _moqObject!.Add(i, i); + } + + return sum; + } + + [Benchmark(Description = "NSubstitute (100 calls)")] + public int NSubstitute_ManyInvocations() + { + var sum = 0; + for (var i = 0; i < 100; i++) + { + sum += _nsubObject!.Add(i, i); + } + + return sum; + } + + [Benchmark(Description = "FakeItEasy (100 calls)")] + public int FakeItEasy_ManyInvocations() + { + var sum = 0; + for (var i = 0; i < 100; i++) + { + sum += _fakeObject!.Add(i, i); + } + + return sum; + } +} diff --git a/TUnit.Mocks.Benchmarks/MockCreationBenchmarks.cs b/TUnit.Mocks.Benchmarks/MockCreationBenchmarks.cs new file mode 100644 index 0000000000..ba30d7516e --- /dev/null +++ b/TUnit.Mocks.Benchmarks/MockCreationBenchmarks.cs @@ -0,0 +1,70 @@ +using BenchmarkDotNet.Attributes; +using FakeItEasy; +using Moq; +using NSubstitute; +using TUnit.Mocks; + +namespace TUnit.Mocks.Benchmarks; + +[MemoryDiagnoser] +[SimpleJob] +[JsonExporterAttribute.Full] +[MarkdownExporterAttribute.GitHub] +public class MockCreationBenchmarks +{ + [Benchmark(Description = "TUnit.Mocks")] + public object TUnitMocks_CreateMock() + { + var mock = Mock.Of(); + return mock.Object; + } + + [Benchmark(Description = "Moq")] + public object Moq_CreateMock() + { + var mock = new Moq.Mock(); + return mock.Object; + } + + [Benchmark(Description = "NSubstitute")] + public object NSubstitute_CreateMock() + { + var sub = Substitute.For(); + return sub; + } + + [Benchmark(Description = "FakeItEasy")] + public object FakeItEasy_CreateMock() + { + var fake = A.Fake(); + return fake; + } + + [Benchmark(Description = "TUnit.Mocks (Repository)")] + public object TUnitMocks_CreateMock_Repository() + { + var mock = Mock.Of(); + return mock.Object; + } + + [Benchmark(Description = "Moq (Repository)")] + public object Moq_CreateMock_Repository() + { + var mock = new Moq.Mock(); + return mock.Object; + } + + [Benchmark(Description = "NSubstitute (Repository)")] + public object NSubstitute_CreateMock_Repository() + { + var sub = Substitute.For(); + return sub; + } + + [Benchmark(Description = "FakeItEasy (Repository)")] + public object FakeItEasy_CreateMock_Repository() + { + var fake = A.Fake(); + return fake; + } +} diff --git a/TUnit.Mocks.Benchmarks/Program.cs b/TUnit.Mocks.Benchmarks/Program.cs new file mode 100644 index 0000000000..c19c445dc9 --- /dev/null +++ b/TUnit.Mocks.Benchmarks/Program.cs @@ -0,0 +1,4 @@ +using BenchmarkDotNet.Running; +using TUnit.Mocks.Benchmarks; + +BenchmarkSwitcher.FromAssembly(typeof(MockCreationBenchmarks).Assembly).Run(args); diff --git a/TUnit.Mocks.Benchmarks/SetupBenchmarks.cs b/TUnit.Mocks.Benchmarks/SetupBenchmarks.cs new file mode 100644 index 0000000000..d0790d661f --- /dev/null +++ b/TUnit.Mocks.Benchmarks/SetupBenchmarks.cs @@ -0,0 +1,101 @@ +using BenchmarkDotNet.Attributes; +using FakeItEasy; +using Moq; +using NSubstitute; + +namespace TUnit.Mocks.Benchmarks; + +[MemoryDiagnoser] +[SimpleJob] +[JsonExporterAttribute.Full] +[MarkdownExporterAttribute.GitHub] +public class SetupBenchmarks +{ + [Benchmark(Description = "TUnit.Mocks")] + public object TUnitMocks_Setup() + { + var mock = Mock.Of(); + mock.Add(TUnitArg.Any(), TUnitArg.Any()).Returns(42); + mock.Format(TUnitArg.Any()).Returns("formatted"); + mock.Divide(TUnitArg.Any(), TUnitArg.Any()).Returns(1.5); + return mock.Object; + } + + [Benchmark(Description = "Moq")] + public object Moq_Setup() + { + var mock = new Moq.Mock(); + mock.Setup(x => x.Add(It.IsAny(), It.IsAny())).Returns(42); + mock.Setup(x => x.Format(It.IsAny())).Returns("formatted"); + mock.Setup(x => x.Divide(It.IsAny(), It.IsAny())).Returns(1.5); + return mock.Object; + } + + [Benchmark(Description = "NSubstitute")] + public object NSubstitute_Setup() + { + var sub = Substitute.For(); + sub.Add(NSubstitute.Arg.Any(), NSubstitute.Arg.Any()).Returns(42); + sub.Format(NSubstitute.Arg.Any()).Returns("formatted"); + sub.Divide(NSubstitute.Arg.Any(), NSubstitute.Arg.Any()).Returns(1.5); + return sub; + } + + [Benchmark(Description = "FakeItEasy")] + public object FakeItEasy_Setup() + { + var fake = A.Fake(); + A.CallTo(() => fake.Add(A.Ignored, A.Ignored)).Returns(42); + A.CallTo(() => fake.Format(A.Ignored)).Returns("formatted"); + A.CallTo(() => fake.Divide(A.Ignored, A.Ignored)).Returns(1.5); + return fake; + } + + [Benchmark(Description = "TUnit.Mocks (Multiple)")] + public object TUnitMocks_MultipleSetups() + { + var mock = Mock.Of(); + mock.GetById(1).Returns(new User { Id = 1, Name = "Alice" }); + mock.GetById(2).Returns(new User { Id = 2, Name = "Bob" }); + mock.GetById(3).Returns(new User { Id = 3, Name = "Charlie" }); + mock.Exists(TUnitArg.Any()).Returns(true); + mock.GetAll().Returns(new List()); + return mock.Object; + } + + [Benchmark(Description = "Moq (Multiple)")] + public object Moq_MultipleSetups() + { + var mock = new Moq.Mock(); + mock.Setup(x => x.GetById(1)).Returns(new User { Id = 1, Name = "Alice" }); + mock.Setup(x => x.GetById(2)).Returns(new User { Id = 2, Name = "Bob" }); + mock.Setup(x => x.GetById(3)).Returns(new User { Id = 3, Name = "Charlie" }); + mock.Setup(x => x.Exists(It.IsAny())).Returns(true); + mock.Setup(x => x.GetAll()).Returns(new List()); + return mock.Object; + } + + [Benchmark(Description = "NSubstitute (Multiple)")] + public object NSubstitute_MultipleSetups() + { + var sub = Substitute.For(); + sub.GetById(1).Returns(new User { Id = 1, Name = "Alice" }); + sub.GetById(2).Returns(new User { Id = 2, Name = "Bob" }); + sub.GetById(3).Returns(new User { Id = 3, Name = "Charlie" }); + sub.Exists(NSubstitute.Arg.Any()).Returns(true); + sub.GetAll().Returns(new List()); + return sub; + } + + [Benchmark(Description = "FakeItEasy (Multiple)")] + public object FakeItEasy_MultipleSetups() + { + var fake = A.Fake(); + A.CallTo(() => fake.GetById(1)).Returns(new User { Id = 1, Name = "Alice" }); + A.CallTo(() => fake.GetById(2)).Returns(new User { Id = 2, Name = "Bob" }); + A.CallTo(() => fake.GetById(3)).Returns(new User { Id = 3, Name = "Charlie" }); + A.CallTo(() => fake.Exists(A.Ignored)).Returns(true); + A.CallTo(() => fake.GetAll()).Returns(new List()); + return fake; + } +} diff --git a/TUnit.Mocks.Benchmarks/TUnit.Mocks.Benchmarks.csproj b/TUnit.Mocks.Benchmarks/TUnit.Mocks.Benchmarks.csproj new file mode 100644 index 0000000000..54701f64b9 --- /dev/null +++ b/TUnit.Mocks.Benchmarks/TUnit.Mocks.Benchmarks.csproj @@ -0,0 +1,29 @@ + + + + Exe + net10.0 + enable + enable + false + disable + + + + + + + + + + + + + + + + + + + diff --git a/TUnit.Mocks.Benchmarks/VerificationBenchmarks.cs b/TUnit.Mocks.Benchmarks/VerificationBenchmarks.cs new file mode 100644 index 0000000000..324f0fc3d6 --- /dev/null +++ b/TUnit.Mocks.Benchmarks/VerificationBenchmarks.cs @@ -0,0 +1,157 @@ +using BenchmarkDotNet.Attributes; +using FakeItEasy; +using Moq; +using NSubstitute; + +namespace TUnit.Mocks.Benchmarks; + +[MemoryDiagnoser] +[SimpleJob] +[JsonExporterAttribute.Full] +[MarkdownExporterAttribute.GitHub] +public class VerificationBenchmarks +{ + [Benchmark(Description = "TUnit.Mocks")] + public void TUnitMocks_Verify() + { + var mock = Mock.Of(); + mock.Add(TUnitArg.Any(), TUnitArg.Any()).Returns(42); + var calc = mock.Object; + calc.Add(1, 2); + calc.Add(3, 4); + + mock.Add(TUnitArg.Any(), TUnitArg.Any()).WasCalled(Times.Exactly(2)); + } + + [Benchmark(Description = "Moq")] + public void Moq_Verify() + { + var mock = new Moq.Mock(); + mock.Setup(x => x.Add(It.IsAny(), It.IsAny())).Returns(42); + var calc = mock.Object; + calc.Add(1, 2); + calc.Add(3, 4); + + mock.Verify(x => x.Add(It.IsAny(), It.IsAny()), Moq.Times.Exactly(2)); + } + + [Benchmark(Description = "NSubstitute")] + public void NSubstitute_Verify() + { + var sub = Substitute.For(); + sub.Add(NSubstitute.Arg.Any(), NSubstitute.Arg.Any()).Returns(42); + sub.Add(1, 2); + sub.Add(3, 4); + + sub.Received(2).Add(NSubstitute.Arg.Any(), NSubstitute.Arg.Any()); + } + + [Benchmark(Description = "FakeItEasy")] + public void FakeItEasy_Verify() + { + var fake = A.Fake(); + A.CallTo(() => fake.Add(A.Ignored, A.Ignored)).Returns(42); + fake.Add(1, 2); + fake.Add(3, 4); + + A.CallTo(() => fake.Add(A.Ignored, A.Ignored)).MustHaveHappenedTwiceExactly(); + } + + [Benchmark(Description = "TUnit.Mocks (Never)")] + public void TUnitMocks_VerifyNever() + { + var mock = Mock.Of(); + mock.Format(TUnitArg.Any()).WasNeverCalled(); + } + + [Benchmark(Description = "Moq (Never)")] + public void Moq_VerifyNever() + { + var mock = new Moq.Mock(); + mock.Verify(x => x.Format(It.IsAny()), Moq.Times.Never()); + } + + [Benchmark(Description = "NSubstitute (Never)")] + public void NSubstitute_VerifyNever() + { + var sub = Substitute.For(); + sub.DidNotReceive().Format(NSubstitute.Arg.Any()); + } + + [Benchmark(Description = "FakeItEasy (Never)")] + public void FakeItEasy_VerifyNever() + { + var fake = A.Fake(); + A.CallTo(() => fake.Format(A.Ignored)).MustNotHaveHappened(); + } + + [Benchmark(Description = "TUnit.Mocks (Multiple)")] + public void TUnitMocks_VerifyMultiple() + { + var mock = Mock.Of(); + mock.GetById(TUnitArg.Any()).Returns(new User { Id = 1, Name = "Test" }); + mock.Exists(TUnitArg.Any()).Returns(true); + + var repo = mock.Object; + repo.GetById(1); + repo.GetById(2); + repo.Exists(1); + repo.Save(new User { Id = 3, Name = "New" }); + + mock.GetById(TUnitArg.Any()).WasCalled(Times.Exactly(2)); + mock.Exists(TUnitArg.Any()).WasCalled(Times.Once); + mock.Save(TUnitArg.Any()).WasCalled(Times.Once); + } + + [Benchmark(Description = "Moq (Multiple)")] + public void Moq_VerifyMultiple() + { + var mock = new Moq.Mock(); + mock.Setup(x => x.GetById(It.IsAny())).Returns(new User { Id = 1, Name = "Test" }); + mock.Setup(x => x.Exists(It.IsAny())).Returns(true); + + var repo = mock.Object; + repo.GetById(1); + repo.GetById(2); + repo.Exists(1); + repo.Save(new User { Id = 3, Name = "New" }); + + mock.Verify(x => x.GetById(It.IsAny()), Moq.Times.Exactly(2)); + mock.Verify(x => x.Exists(It.IsAny()), Moq.Times.Once()); + mock.Verify(x => x.Save(It.IsAny()), Moq.Times.Once()); + } + + [Benchmark(Description = "NSubstitute (Multiple)")] + public void NSubstitute_VerifyMultiple() + { + var sub = Substitute.For(); + sub.GetById(NSubstitute.Arg.Any()).Returns(new User { Id = 1, Name = "Test" }); + sub.Exists(NSubstitute.Arg.Any()).Returns(true); + + sub.GetById(1); + sub.GetById(2); + sub.Exists(1); + sub.Save(new User { Id = 3, Name = "New" }); + + sub.Received(2).GetById(NSubstitute.Arg.Any()); + sub.Received(1).Exists(NSubstitute.Arg.Any()); + sub.Received(1).Save(NSubstitute.Arg.Any()); + } + + [Benchmark(Description = "FakeItEasy (Multiple)")] + public void FakeItEasy_VerifyMultiple() + { + var fake = A.Fake(); + A.CallTo(() => fake.GetById(A.Ignored)).Returns(new User { Id = 1, Name = "Test" }); + A.CallTo(() => fake.Exists(A.Ignored)).Returns(true); + + fake.GetById(1); + fake.GetById(2); + fake.Exists(1); + fake.Save(new User { Id = 3, Name = "New" }); + + A.CallTo(() => fake.GetById(A.Ignored)).MustHaveHappenedTwiceExactly(); + A.CallTo(() => fake.Exists(A.Ignored)).MustHaveHappenedOnceExactly(); + A.CallTo(() => fake.Save(A.Ignored)).MustHaveHappenedOnceExactly(); + } +} diff --git a/docs/docs/benchmarks/mocks/Callback.md b/docs/docs/benchmarks/mocks/Callback.md new file mode 100644 index 0000000000..d385602991 --- /dev/null +++ b/docs/docs/benchmarks/mocks/Callback.md @@ -0,0 +1,21 @@ +--- +title: "Mock Benchmark: Callback" +description: "Mock callback performance — TUnit.Mocks vs Moq vs NSubstitute vs FakeItEasy" +sidebar_position: 6 +--- + +# Callback Benchmark + +:::info Awaiting First Run +This benchmark will be automatically populated after the first [Mock Benchmarks workflow](https://github.com/thomhurst/TUnit/actions/workflows/mock-benchmarks.yml) run completes. +::: + +## 📊 About This Benchmark + +This benchmark measures how fast each mocking library can register and execute callbacks during mock method invocations, both simple callbacks and callbacks with argument capture. + +--- + +:::note Methodology +View the [mock benchmarks overview](/docs/benchmarks/mocks) for methodology details and environment information. +::: diff --git a/docs/docs/benchmarks/mocks/CombinedWorkflow.md b/docs/docs/benchmarks/mocks/CombinedWorkflow.md new file mode 100644 index 0000000000..da9bf74426 --- /dev/null +++ b/docs/docs/benchmarks/mocks/CombinedWorkflow.md @@ -0,0 +1,21 @@ +--- +title: "Mock Benchmark: CombinedWorkflow" +description: "End-to-end mock workflow performance — TUnit.Mocks vs Moq vs NSubstitute vs FakeItEasy" +sidebar_position: 7 +--- + +# Combined Workflow Benchmark + +:::info Awaiting First Run +This benchmark will be automatically populated after the first [Mock Benchmarks workflow](https://github.com/thomhurst/TUnit/actions/workflows/mock-benchmarks.yml) run completes. +::: + +## 📊 About This Benchmark + +This benchmark simulates a real-world test scenario measuring the complete workflow: create mocks → configure setups → invoke methods → verify calls. This represents the most realistic usage pattern. + +--- + +:::note Methodology +View the [mock benchmarks overview](/docs/benchmarks/mocks) for methodology details and environment information. +::: diff --git a/docs/docs/benchmarks/mocks/Invocation.md b/docs/docs/benchmarks/mocks/Invocation.md new file mode 100644 index 0000000000..8eadc55f8b --- /dev/null +++ b/docs/docs/benchmarks/mocks/Invocation.md @@ -0,0 +1,21 @@ +--- +title: "Mock Benchmark: Invocation" +description: "Mock method invocation performance — TUnit.Mocks vs Moq vs NSubstitute vs FakeItEasy" +sidebar_position: 4 +--- + +# Invocation Benchmark + +:::info Awaiting First Run +This benchmark will be automatically populated after the first [Mock Benchmarks workflow](https://github.com/thomhurst/TUnit/actions/workflows/mock-benchmarks.yml) run completes. +::: + +## 📊 About This Benchmark + +This benchmark measures the overhead of calling methods on mock objects. Once mocks are set up, this measures the per-call cost of dispatching through the mock. + +--- + +:::note Methodology +View the [mock benchmarks overview](/docs/benchmarks/mocks) for methodology details and environment information. +::: diff --git a/docs/docs/benchmarks/mocks/MockCreation.md b/docs/docs/benchmarks/mocks/MockCreation.md new file mode 100644 index 0000000000..08aa3b1945 --- /dev/null +++ b/docs/docs/benchmarks/mocks/MockCreation.md @@ -0,0 +1,27 @@ +--- +title: "Mock Benchmark: MockCreation" +description: "Mock instance creation performance — TUnit.Mocks vs Moq vs NSubstitute vs FakeItEasy" +sidebar_position: 2 +--- + +# MockCreation Benchmark + +:::info Awaiting First Run +This benchmark will be automatically populated after the first [Mock Benchmarks workflow](https://github.com/thomhurst/TUnit/actions/workflows/mock-benchmarks.yml) run completes. +::: + +## 📊 About This Benchmark + +This benchmark measures how fast each mocking library can create mock instances of interfaces. This is critical for test performance since every test typically creates one or more mocks. + +**Libraries compared:** +- **TUnit.Mocks** — Source-generated mock factories (no runtime proxy generation) +- **Moq** — Runtime proxy generation via Castle.DynamicProxy +- **NSubstitute** — Runtime proxy generation via Castle.DynamicProxy +- **FakeItEasy** — Runtime proxy generation via Castle.DynamicProxy + +--- + +:::note Methodology +View the [mock benchmarks overview](/docs/benchmarks/mocks) for methodology details and environment information. +::: diff --git a/docs/docs/benchmarks/mocks/Setup.md b/docs/docs/benchmarks/mocks/Setup.md new file mode 100644 index 0000000000..66d9ebd0cf --- /dev/null +++ b/docs/docs/benchmarks/mocks/Setup.md @@ -0,0 +1,21 @@ +--- +title: "Mock Benchmark: Setup" +description: "Mock behavior configuration performance — TUnit.Mocks vs Moq vs NSubstitute vs FakeItEasy" +sidebar_position: 3 +--- + +# Setup Benchmark + +:::info Awaiting First Run +This benchmark will be automatically populated after the first [Mock Benchmarks workflow](https://github.com/thomhurst/TUnit/actions/workflows/mock-benchmarks.yml) run completes. +::: + +## 📊 About This Benchmark + +This benchmark measures how fast each mocking library can configure mock behaviors including return values, argument matchers, and multiple setups. + +--- + +:::note Methodology +View the [mock benchmarks overview](/docs/benchmarks/mocks) for methodology details and environment information. +::: diff --git a/docs/docs/benchmarks/mocks/Verification.md b/docs/docs/benchmarks/mocks/Verification.md new file mode 100644 index 0000000000..224918872a --- /dev/null +++ b/docs/docs/benchmarks/mocks/Verification.md @@ -0,0 +1,21 @@ +--- +title: "Mock Benchmark: Verification" +description: "Mock call verification performance — TUnit.Mocks vs Moq vs NSubstitute vs FakeItEasy" +sidebar_position: 5 +--- + +# Verification Benchmark + +:::info Awaiting First Run +This benchmark will be automatically populated after the first [Mock Benchmarks workflow](https://github.com/thomhurst/TUnit/actions/workflows/mock-benchmarks.yml) run completes. +::: + +## 📊 About This Benchmark + +This benchmark measures how fast each mocking library can verify that methods were called with expected arguments and the correct number of times. + +--- + +:::note Methodology +View the [mock benchmarks overview](/docs/benchmarks/mocks) for methodology details and environment information. +::: diff --git a/docs/docs/benchmarks/mocks/index.md b/docs/docs/benchmarks/mocks/index.md new file mode 100644 index 0000000000..2ef4d2f277 --- /dev/null +++ b/docs/docs/benchmarks/mocks/index.md @@ -0,0 +1,70 @@ +--- +title: Mock Library Benchmarks +description: Performance comparisons between TUnit.Mocks, Moq, NSubstitute, and FakeItEasy +sidebar_position: 1 +--- + +# Mock Library Benchmarks + +:::info Awaiting First Run +These benchmarks will be automatically populated after the first [Mock Benchmarks workflow](https://github.com/thomhurst/TUnit/actions/workflows/mock-benchmarks.yml) run completes. +::: + +## 🚀 Overview + +These benchmarks compare **TUnit.Mocks** (source-generated, AOT-compatible) against the most popular .NET mocking libraries that use runtime proxy generation: + +| Library | Approach | AOT Compatible | +|---------|----------|----------------| +| **TUnit.Mocks** | Source-generated at compile time | ✅ Yes | +| **Moq** | Runtime proxy via Castle.DynamicProxy | ❌ No | +| **NSubstitute** | Runtime proxy via Castle.DynamicProxy | ❌ No | +| **FakeItEasy** | Runtime proxy via Castle.DynamicProxy | ❌ No | + +## 📊 Benchmark Categories + +- [Mock Creation](MockCreation) - Mock instance creation performance +- [Setup](Setup) - Mock behavior configuration (returns, matchers) +- [Invocation](Invocation) - Calling methods on mock objects +- [Verification](Verification) - Verifying mock method calls +- [Callback](Callback) - Callback registration and execution +- [Combined Workflow](CombinedWorkflow) - Full workflow: create → setup → invoke → verify + +## 📈 What's Measured + +Each benchmark category tests a specific aspect of mocking library usage: + +- **MockCreation** — How fast can each library create a mock instance? +- **Setup** — How fast can you configure return values and argument matchers? +- **Invocation** — Once set up, how fast are method calls on the mock? +- **Verification** — How fast can you verify that methods were called correctly? +- **Callback** — How fast are callbacks triggered during mock invocations? +- **CombinedWorkflow** — The full real-world pattern: create → setup → invoke → verify + +## 🔧 Methodology + +- **Tool**: BenchmarkDotNet +- **OS**: Ubuntu Latest (GitHub Actions) +- **Statistical Rigor**: Multiple iterations with warm-up and outlier detection +- **Memory**: Allocation tracking enabled via `[MemoryDiagnoser]` + +### Why Source-Generated Mocks? + +TUnit.Mocks generates mock implementations at compile time, eliminating: +- Runtime proxy generation overhead +- Dynamic assembly emission +- Reflection-based method dispatch + +This makes TUnit.Mocks compatible with **Native AOT** and **IL trimming**, while also providing performance benefits for standard .NET execution. + +### Source Code + +All benchmark source code is available in the [`TUnit.Mocks.Benchmarks`](https://github.com/thomhurst/TUnit/tree/main/TUnit.Mocks.Benchmarks) directory. + +--- + +:::note Continuous Benchmarking +These benchmarks run automatically daily via [GitHub Actions](https://github.com/thomhurst/TUnit/actions/workflows/mock-benchmarks.yml). + +Each benchmark runs multiple iterations with statistical analysis to ensure accuracy. Results may vary based on hardware and test characteristics. +::: diff --git a/docs/static/benchmarks/mocks/latest.json b/docs/static/benchmarks/mocks/latest.json new file mode 100644 index 0000000000..5b75dedab7 --- /dev/null +++ b/docs/static/benchmarks/mocks/latest.json @@ -0,0 +1,10 @@ +{ + "timestamp": "placeholder", + "environment": {}, + "categories": {}, + "stats": { + "categoryCount": 0, + "totalBenchmarks": 0, + "lastUpdated": "placeholder" + } +} diff --git a/docs/static/benchmarks/mocks/summary.json b/docs/static/benchmarks/mocks/summary.json new file mode 100644 index 0000000000..6cdbd7a439 --- /dev/null +++ b/docs/static/benchmarks/mocks/summary.json @@ -0,0 +1,6 @@ +{ + "categories": [], + "timestamp": "placeholder", + "environment": "Awaiting first benchmark run", + "libraries": ["TUnit.Mocks", "Moq", "NSubstitute", "FakeItEasy"] +}