Repository navigation
Add fuzz tests for the analyzers and the code fixers #1641
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
+2,343
−1
Merged
Changes from all commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| # Fuzz the analyzers and the code fixers, and open an issue for each new finding | ||
| # See tests/Meziantou.Analyzer.Fuzz/README.md | ||
| name: fuzz | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| duration-minutes: | ||
| description: "Duration of the fuzzing, in minutes" | ||
| type: number | ||
| default: 240 | ||
| report: | ||
| description: "Open an issue for each new finding" | ||
| type: boolean | ||
| default: true | ||
| schedule: | ||
| - cron: "0 3 * * 0" | ||
| timezone: "America/Toronto" | ||
|
|
||
| concurrency: | ||
| group: fuzz | ||
| cancel-in-progress: false | ||
|
|
||
| env: | ||
| DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 | ||
| DOTNET_NOLOGO: true | ||
| SHARPFUZZ_VERSION: 2.3.0 | ||
| # https://github.com/Metalnem/libfuzzer-dotnet/releases | ||
| LIBFUZZER_DOTNET_URL: https://github.com/Metalnem/libfuzzer-dotnet/releases/download/v2025.05.02.0904/libfuzzer-dotnet-ubuntu | ||
| LIBFUZZER_DOTNET_SHA256: c2c2a90d94c409a4af339a0d4f244e0442c5a5d249be0f1252ba07871f285958 | ||
|
|
||
| defaults: | ||
| run: | ||
| shell: pwsh | ||
|
|
||
| jobs: | ||
| fuzz: | ||
| # The scheduled runs would open issues in the forks | ||
| if: github.event_name == 'workflow_dispatch' || github.repository == 'meziantou/Meziantou.Analyzer' | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 350 | ||
| permissions: | ||
| contents: read | ||
| actions: read # Download the corpus of the previous run | ||
| steps: | ||
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | ||
| - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6 | ||
|
|
||
| - name: Install SharpFuzz and libfuzzer-dotnet | ||
| run: | | ||
| dotnet tool install --global SharpFuzz.CommandLine --version $env:SHARPFUZZ_VERSION | ||
|
|
||
| $libFuzzer = Join-Path $env:RUNNER_TEMP "libfuzzer-dotnet" | ||
| Invoke-WebRequest $env:LIBFUZZER_DOTNET_URL -OutFile $libFuzzer | ||
| $hash = (Get-FileHash $libFuzzer -Algorithm SHA256).Hash | ||
| if ($hash -ne $env:LIBFUZZER_DOTNET_SHA256) { | ||
| Write-Error "Unexpected SHA-256 of libfuzzer-dotnet: $hash" | ||
| exit 1 | ||
| } | ||
|
|
||
| chmod +x $libFuzzer | ||
| "LIBFUZZER=$libFuzzer" >> $env:GITHUB_ENV | ||
|
|
||
| - name: Download the corpus of the previous run | ||
| env: | ||
| GH_TOKEN: ${{ github.token }} | ||
| run: | | ||
| $directory = Join-Path $env:RUNNER_TEMP "previous-corpus" | ||
| $runs = gh run list --repo $env:GITHUB_REPOSITORY --workflow fuzz.yml --status completed --limit 10 --json databaseId | ConvertFrom-Json | ||
| foreach ($run in $runs) { | ||
| gh run download $run.databaseId --repo $env:GITHUB_REPOSITORY --name fuzz-corpus --dir $directory | ||
| if ($LASTEXITCODE -eq 0) { | ||
| Write-Host "Using the corpus of the run $($run.databaseId)" | ||
| "PREVIOUS_CORPUS=$directory" >> $env:GITHUB_ENV | ||
| break | ||
| } | ||
| } | ||
|
|
||
| exit 0 | ||
|
|
||
| - name: Fuzz | ||
| env: | ||
| DURATION_MINUTES: ${{ inputs.duration-minutes || 240 }} | ||
| run: ./tests/Meziantou.Analyzer.Fuzz/eng/run.ps1 -LibFuzzer $env:LIBFUZZER -DurationMinutes $env:DURATION_MINUTES -OutputDirectory artifacts/fuzz -PreviousCorpus $env:PREVIOUS_CORPUS | ||
|
|
||
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | ||
| if: always() | ||
| with: | ||
| name: fuzz-corpus | ||
| path: artifacts/fuzz/corpus-merged | ||
| if-no-files-found: ignore | ||
| retention-days: 30 | ||
|
|
||
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | ||
| if: always() | ||
| with: | ||
| name: fuzz-findings | ||
| path: artifacts/fuzz/findings | ||
| if-no-files-found: ignore | ||
| retention-days: 90 | ||
|
|
||
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | ||
| if: always() | ||
| with: | ||
| name: fuzz-logs | ||
| path: | | ||
| artifacts/fuzz/logs | ||
| artifacts/fuzz/artifacts | ||
| artifacts/fuzz/state/suppressed.txt | ||
| if-no-files-found: ignore | ||
| retention-days: 14 | ||
|
|
||
| # A separate job, so the token that can write the issues is not available to the fuzzed code | ||
| report: | ||
| needs: fuzz | ||
| if: github.event_name == 'schedule' || inputs.report | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| contents: read | ||
| issues: write | ||
| steps: | ||
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | ||
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | ||
| continue-on-error: true # There is no artifact when there is no finding | ||
| with: | ||
| name: fuzz-findings | ||
| path: artifacts/fuzz/findings | ||
| - name: Open the issues | ||
| env: | ||
| GH_TOKEN: ${{ github.token }} | ||
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | ||
| run: ./tests/Meziantou.Analyzer.Fuzz/eng/report.ps1 -FindingsDirectory artifacts/fuzz/findings -RunUrl $env:RUN_URL | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,193 @@ | ||
| using System.Collections.Concurrent; | ||
| using System.Collections.Immutable; | ||
| using Microsoft.CodeAnalysis; | ||
| using Microsoft.CodeAnalysis.CodeActions; | ||
| using Microsoft.CodeAnalysis.CodeFixes; | ||
| using Microsoft.CodeAnalysis.Diagnostics; | ||
| using Microsoft.CodeAnalysis.Text; | ||
|
|
||
| namespace Meziantou.Analyzer.Fuzz; | ||
|
|
||
| /// <summary> | ||
| /// Run the analyzers on a C# file, then the code fixers on the diagnostics they report, and collect the defects. | ||
| /// </summary> | ||
| internal sealed class AnalysisEngine(ComponentFilter filter, int maxFixedDiagnostics = 20) | ||
| { | ||
| public async Task<IReadOnlyList<Finding>> RunAsync(string source, CancellationToken cancellationToken = default) | ||
| { | ||
| var findings = new List<Finding>(); | ||
| try | ||
| { | ||
| using var workspace = new AdhocWorkspace(); | ||
| var projectId = ProjectId.CreateNewId(); | ||
| var documentId = DocumentId.CreateNewId(projectId); | ||
| var solution = workspace.CurrentSolution | ||
| .AddProject(ProjectInfo.Create( | ||
| projectId, | ||
| VersionStamp.Default, | ||
| name: "Fuzz", | ||
| assemblyName: "Fuzz", | ||
| LanguageNames.CSharp, | ||
| compilationOptions: Components.CompilationOptions, | ||
| parseOptions: Components.ParseOptions, | ||
| metadataReferences: Components.MetadataReferences, | ||
| analyzerReferences: Components.SourceGenerators)) | ||
| .AddDocument(documentId, "Fuzz.cs", SourceText.From(source)); | ||
|
|
||
| var document = solution.GetDocument(documentId)!; | ||
| var compilation = (await document.Project.GetCompilationAsync(cancellationToken).ConfigureAwait(false))!; | ||
| var diagnostics = await RunAnalyzersAsync(compilation, findings, cancellationToken).ConfigureAwait(false); | ||
| if (!filter.CodeFixProviders.IsEmpty) | ||
| { | ||
| var hasErrors = compilation.GetDiagnostics(cancellationToken).Any(diagnostic => diagnostic.Severity is DiagnosticSeverity.Error); | ||
| await RunCodeFixesAsync(document, diagnostics, hasErrors, findings, cancellationToken).ConfigureAwait(false); | ||
| } | ||
| } | ||
| catch (Exception ex) when (ex is not OperationCanceledException) | ||
| { | ||
| findings.Add(Finding.FromException(FindingKind.HarnessException, "Meziantou.Analyzer.Fuzz", [], ex)); | ||
| } | ||
|
|
||
| return [.. findings.DistinctBy(finding => finding.Signature)]; | ||
| } | ||
|
|
||
| private async Task<ImmutableArray<Diagnostic>> RunAnalyzersAsync(Compilation compilation, List<Finding> findings, CancellationToken cancellationToken) | ||
| { | ||
| if (filter.Analyzers.IsEmpty) | ||
| return []; | ||
|
|
||
| var exceptions = new ConcurrentQueue<(DiagnosticAnalyzer Analyzer, Exception Exception)>(); | ||
| var options = new CompilationWithAnalyzersOptions( | ||
| new AnalyzerOptions([]), | ||
| onAnalyzerException: (exception, analyzer, _) => exceptions.Enqueue((analyzer, exception)), | ||
| concurrentAnalysis: false, // Keep the coverage deterministic, so libFuzzer can tell which mutations are interesting | ||
| logAnalyzerExecutionTime: false, | ||
| reportSuppressedDiagnostics: false); | ||
|
|
||
| var compilationWithAnalyzers = compilation.WithAnalyzers(filter.Analyzers, options); | ||
| var diagnostics = await compilationWithAnalyzers.GetAllDiagnosticsAsync(cancellationToken).ConfigureAwait(false); | ||
| foreach (var (analyzer, exception) in exceptions) | ||
| { | ||
| if (exception is OperationCanceledException) | ||
| continue; | ||
|
|
||
| findings.Add(Finding.FromException(FindingKind.AnalyzerException, analyzer, Components.GetDiagnosticIds(analyzer), exception)); | ||
| } | ||
|
|
||
| return diagnostics; | ||
| } | ||
|
|
||
| private async Task RunCodeFixesAsync(Document document, ImmutableArray<Diagnostic> diagnostics, bool hasErrors, List<Finding> findings, CancellationToken cancellationToken) | ||
| { | ||
| var fixableDiagnostics = diagnostics | ||
| .Where(diagnostic => diagnostic.Location.IsInSource && !diagnostic.IsSuppressed && filter.CodeFixProviders.ContainsKey(diagnostic.Id)) | ||
| .DistinctBy(diagnostic => (diagnostic.Id, diagnostic.Location.SourceSpan)) | ||
| .Take(maxFixedDiagnostics); | ||
|
|
||
| foreach (var diagnostic in fixableDiagnostics) | ||
| { | ||
| foreach (var provider in filter.CodeFixProviders[diagnostic.Id]) | ||
| { | ||
| await RunCodeFixAsync(document, provider, diagnostic, hasErrors, findings, cancellationToken).ConfigureAwait(false); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private static async Task RunCodeFixAsync(Document document, CodeFixProvider provider, Diagnostic diagnostic, bool hasErrors, List<Finding> findings, CancellationToken cancellationToken) | ||
| { | ||
| try | ||
| { | ||
| var actions = new List<CodeAction>(); | ||
| var context = new CodeFixContext(document, diagnostic, (action, _) => actions.Add(action), cancellationToken); | ||
| await provider.RegisterCodeFixesAsync(context).ConfigureAwait(false); | ||
|
|
||
| foreach (var action in Flatten(actions)) | ||
| { | ||
| var operations = await action.GetOperationsAsync(cancellationToken).ConfigureAwait(false); | ||
| var changedSolution = operations.OfType<ApplyChangesOperation>().LastOrDefault()?.ChangedSolution; | ||
| if (changedSolution is null || !await HasTextChangesAsync(document.Project.Solution, changedSolution, cancellationToken).ConfigureAwait(false)) | ||
| { | ||
| findings.Add(new Finding | ||
| { | ||
| Kind = FindingKind.CodeFixDoesNotChangeCode, | ||
| Component = Components.GetName(provider), | ||
| DiagnosticIds = [diagnostic.Id], | ||
| CodeActionTitle = action.Title, | ||
| }); | ||
| continue; | ||
| } | ||
|
|
||
| // The fuzzed code rarely compiles, and a code fix cannot be expected to fix the errors of the code | ||
| if (hasErrors || changedSolution.GetProject(document.Project.Id) is not { } changedProject) | ||
| continue; | ||
|
|
||
| var changedCompilation = (await changedProject.GetCompilationAsync(cancellationToken).ConfigureAwait(false))!; | ||
| var errors = changedCompilation.GetDiagnostics(cancellationToken).Where(diagnostic => diagnostic.Severity is DiagnosticSeverity.Error).ToList(); | ||
| if (errors.Count > 0) | ||
| { | ||
| var fixedDocument = changedSolution.GetDocument(document.Id); | ||
| findings.Add(new Finding | ||
| { | ||
| Kind = FindingKind.CodeFixBreaksCode, | ||
| Component = Components.GetName(provider), | ||
| DiagnosticIds = [diagnostic.Id], | ||
| CompilerErrors = [.. errors.Select(error => error.Id).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal)], | ||
| Message = string.Join('\n', errors.Take(10).Select(error => error.ToString())), | ||
| CodeActionTitle = action.Title, | ||
| FixedSource = fixedDocument is null ? null : (await fixedDocument.GetTextAsync(cancellationToken).ConfigureAwait(false)).ToString(), | ||
| }); | ||
| } | ||
| } | ||
| } | ||
| catch (Exception ex) when (ex is not OperationCanceledException) | ||
| { | ||
| findings.Add(Finding.FromException(FindingKind.CodeFixException, provider, [diagnostic.Id], ex)); | ||
| } | ||
| } | ||
|
|
||
| private static IEnumerable<CodeAction> Flatten(IEnumerable<CodeAction> actions) | ||
| { | ||
| foreach (var action in actions) | ||
| { | ||
| if (action.NestedActions.IsEmpty) | ||
| { | ||
| yield return action; | ||
| continue; | ||
| } | ||
|
|
||
| foreach (var nestedAction in Flatten(action.NestedActions)) | ||
| { | ||
| yield return nestedAction; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private static async Task<bool> HasTextChangesAsync(Solution originalSolution, Solution changedSolution, CancellationToken cancellationToken) | ||
| { | ||
| var changes = changedSolution.GetChanges(originalSolution); | ||
| if (changes.GetAddedProjects().Any() || changes.GetRemovedProjects().Any()) | ||
| return true; | ||
|
|
||
| foreach (var projectChanges in changes.GetProjectChanges()) | ||
| { | ||
| if (projectChanges.GetAddedDocuments().Any() || projectChanges.GetRemovedDocuments().Any() | ||
| || projectChanges.GetAddedAdditionalDocuments().Any() || projectChanges.GetRemovedAdditionalDocuments().Any() | ||
| || projectChanges.GetChangedAdditionalDocuments().Any() || projectChanges.GetAddedAnalyzerConfigDocuments().Any() | ||
| || projectChanges.GetRemovedAnalyzerConfigDocuments().Any() || projectChanges.GetChangedAnalyzerConfigDocuments().Any() | ||
| || projectChanges.GetAddedMetadataReferences().Any() || projectChanges.GetRemovedMetadataReferences().Any()) | ||
| { | ||
| return true; | ||
| } | ||
|
|
||
| foreach (var documentId in projectChanges.GetChangedDocuments(onlyGetDocumentsWithTextChanges: true)) | ||
| { | ||
| var originalText = await originalSolution.GetDocument(documentId)!.GetTextAsync(cancellationToken).ConfigureAwait(false); | ||
| var changedText = await changedSolution.GetDocument(documentId)!.GetTextAsync(cancellationToken).ConfigureAwait(false); | ||
| if (!originalText.ContentEquals(changedText)) | ||
| return true; | ||
| } | ||
| } | ||
|
|
||
| return false; | ||
| } | ||
| } |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Add timezone (america/toronto)