From bf4c8b525253ac32a2f3f99600584e24aab70ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9rald=20Barr=C3=A9?= Date: Sat, 17 Oct 2026 16:17:02 -0400 Subject: [PATCH 1/3] Add fuzz tests for the analyzers and the code fixers A SharpFuzz/libFuzzer harness runs all the analyzers and code fixers on C# code mutated from the snippets of the tests. SharpFuzz instruments only the analyzer assemblies, so the coverage of the rules guides libFuzzer. The harness reports analyzer exceptions, code fixer exceptions, code fixes that introduce compiler errors or do not change the code, process crashes, and hangs. The weekly fuzz workflow fuzzes for 4 hours, reuses the corpus of the previous run, triages and minimizes the findings, and opens a deduplicated "fuzzing" issue per finding (at most 10 per run). The issues are exempted from the stale workflow, and the CI builds the harness to keep it compiling. --- .github/workflows/ci.yml | 2 + .github/workflows/close-issues.yml | 2 +- .github/workflows/fuzz.yml | 130 ++++++++ Meziantou.Analyzer.slnx | 2 + .../Meziantou.Analyzer.Fuzz/AnalysisEngine.cs | 193 +++++++++++ tests/Meziantou.Analyzer.Fuzz/Campaign.cs | 303 ++++++++++++++++++ tests/Meziantou.Analyzer.Fuzz/ChildProcess.cs | 152 +++++++++ .../ChildProcessResult.cs | 7 + .../CommandLineArguments.cs | 51 +++ .../ComponentFilter.cs | 65 ++++ tests/Meziantou.Analyzer.Fuzz/Components.cs | 150 +++++++++ .../CorpusExtractor.cs | 136 ++++++++ tests/Meziantou.Analyzer.Fuzz/Finding.cs | 114 +++++++ tests/Meziantou.Analyzer.Fuzz/FindingKind.cs | 34 ++ tests/Meziantou.Analyzer.Fuzz/FindingStore.cs | 100 ++++++ .../FuzzEnvironment.cs | 42 +++ .../FuzzFindingException.cs | 24 ++ tests/Meziantou.Analyzer.Fuzz/FuzzTarget.cs | 57 ++++ .../GeneratorAssemblyLoader.cs | 37 +++ .../Meziantou.Analyzer.Fuzz.csproj | 38 +++ tests/Meziantou.Analyzer.Fuzz/Program.cs | 41 +++ tests/Meziantou.Analyzer.Fuzz/README.md | 79 +++++ tests/Meziantou.Analyzer.Fuzz/ReproCommand.cs | 88 +++++ .../Meziantou.Analyzer.Fuzz/SharpFuzzTrace.cs | 24 ++ .../StackTraceParser.cs | 73 +++++ tests/Meziantou.Analyzer.Fuzz/Triage.cs | 127 ++++++++ tests/Meziantou.Analyzer.Fuzz/eng/report.ps1 | 200 ++++++++++++ tests/Meziantou.Analyzer.Fuzz/eng/run.ps1 | 64 ++++ 28 files changed, 2334 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/fuzz.yml create mode 100644 tests/Meziantou.Analyzer.Fuzz/AnalysisEngine.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/Campaign.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/ChildProcess.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/ChildProcessResult.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/CommandLineArguments.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/ComponentFilter.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/Components.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/CorpusExtractor.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/Finding.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/FindingKind.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/FindingStore.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/FuzzEnvironment.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/FuzzFindingException.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/FuzzTarget.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/GeneratorAssemblyLoader.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/Meziantou.Analyzer.Fuzz.csproj create mode 100644 tests/Meziantou.Analyzer.Fuzz/Program.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/README.md create mode 100644 tests/Meziantou.Analyzer.Fuzz/ReproCommand.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/SharpFuzzTrace.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/StackTraceParser.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/Triage.cs create mode 100644 tests/Meziantou.Analyzer.Fuzz/eng/report.ps1 create mode 100644 tests/Meziantou.Analyzer.Fuzz/eng/run.ps1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c8163f5d..1d19386b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,8 @@ jobs: # Meziantou.Analyzer.Pack references one project per Roslyn version, so this builds all of them - run: dotnet build src/Meziantou.Analyzer.Pack/Meziantou.Analyzer.Pack.csproj --configuration Release /p:RunAnalyzers=true - run: dotnet build src/Meziantou.Analyzer.Annotations/Meziantou.Analyzer.Annotations.csproj --configuration Release /p:RunAnalyzers=true + # The fuzz tests only run in the fuzz workflow, so this keeps them compiling + - run: dotnet build tests/Meziantou.Analyzer.Fuzz/Meziantou.Analyzer.Fuzz.csproj --configuration Release /p:RunAnalyzers=true create_nuget: runs-on: ubuntu-latest diff --git a/.github/workflows/close-issues.yml b/.github/workflows/close-issues.yml index 01cd73dc2..6e3291d49 100644 --- a/.github/workflows/close-issues.yml +++ b/.github/workflows/close-issues.yml @@ -22,6 +22,6 @@ jobs: days-before-pr-close: 60 stale-pr-message: "This pull request is stale because it has been open for 60 days with no activity." close-pr-message: "This pull request was closed because it has been inactive for 14 days since being marked as stale." - exempt-issue-labels: "not-stale,need-help" + exempt-issue-labels: "not-stale,need-help,fuzzing" exempt-pr-labels: "not-stale,need-help" repo-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 000000000..ba02601ef --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,130 @@ +# Fuzz the analyzers and the code fixers, and open an issue for each new finding +# See tests/Meziantou.Analyzer.Fuzz/README.md +name: fuzz +on: + workflow_dispatch: + inputs: + duration-minutes: + description: "Duration of the fuzzing, in minutes" + type: number + default: 240 + report: + description: "Open an issue for each new finding" + type: boolean + default: true + schedule: + - cron: "0 3 * * 0" + +concurrency: + group: fuzz + cancel-in-progress: false + +env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + DOTNET_NOLOGO: true + SHARPFUZZ_VERSION: 2.3.0 + # https://github.com/Metalnem/libfuzzer-dotnet/releases + LIBFUZZER_DOTNET_URL: https://github.com/Metalnem/libfuzzer-dotnet/releases/download/v2025.05.02.0904/libfuzzer-dotnet-ubuntu + LIBFUZZER_DOTNET_SHA256: c2c2a90d94c409a4af339a0d4f244e0442c5a5d249be0f1252ba07871f285958 + +defaults: + run: + shell: pwsh + +jobs: + fuzz: + # The scheduled runs would open issues in the forks + if: github.event_name == 'workflow_dispatch' || github.repository == 'meziantou/Meziantou.Analyzer' + runs-on: ubuntu-latest + timeout-minutes: 350 + permissions: + contents: read + actions: read # Download the corpus of the previous run + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6 + + - name: Install SharpFuzz and libfuzzer-dotnet + run: | + dotnet tool install --global SharpFuzz.CommandLine --version $env:SHARPFUZZ_VERSION + + $libFuzzer = Join-Path $env:RUNNER_TEMP "libfuzzer-dotnet" + Invoke-WebRequest $env:LIBFUZZER_DOTNET_URL -OutFile $libFuzzer + $hash = (Get-FileHash $libFuzzer -Algorithm SHA256).Hash + if ($hash -ne $env:LIBFUZZER_DOTNET_SHA256) { + Write-Error "Unexpected SHA-256 of libfuzzer-dotnet: $hash" + exit 1 + } + + chmod +x $libFuzzer + "LIBFUZZER=$libFuzzer" >> $env:GITHUB_ENV + + - name: Download the corpus of the previous run + env: + GH_TOKEN: ${{ github.token }} + run: | + $directory = Join-Path $env:RUNNER_TEMP "previous-corpus" + $runs = gh run list --repo $env:GITHUB_REPOSITORY --workflow fuzz.yml --status completed --limit 10 --json databaseId | ConvertFrom-Json + foreach ($run in $runs) { + gh run download $run.databaseId --repo $env:GITHUB_REPOSITORY --name fuzz-corpus --dir $directory + if ($LASTEXITCODE -eq 0) { + Write-Host "Using the corpus of the run $($run.databaseId)" + "PREVIOUS_CORPUS=$directory" >> $env:GITHUB_ENV + break + } + } + + exit 0 + + - name: Fuzz + env: + DURATION_MINUTES: ${{ inputs.duration-minutes || 240 }} + run: ./tests/Meziantou.Analyzer.Fuzz/eng/run.ps1 -LibFuzzer $env:LIBFUZZER -DurationMinutes $env:DURATION_MINUTES -OutputDirectory artifacts/fuzz -PreviousCorpus $env:PREVIOUS_CORPUS + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: fuzz-corpus + path: artifacts/fuzz/corpus-merged + if-no-files-found: ignore + retention-days: 30 + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: fuzz-findings + path: artifacts/fuzz/findings + if-no-files-found: ignore + retention-days: 90 + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: fuzz-logs + path: | + artifacts/fuzz/logs + artifacts/fuzz/artifacts + artifacts/fuzz/state/suppressed.txt + if-no-files-found: ignore + retention-days: 14 + + # A separate job, so the token that can write the issues is not available to the fuzzed code + report: + needs: fuzz + if: github.event_name == 'schedule' || inputs.report + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + continue-on-error: true # There is no artifact when there is no finding + with: + name: fuzz-findings + path: artifacts/fuzz/findings + - name: Open the issues + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: ./tests/Meziantou.Analyzer.Fuzz/eng/report.ps1 -FindingsDirectory artifacts/fuzz/findings -RunUrl $env:RUN_URL diff --git a/Meziantou.Analyzer.slnx b/Meziantou.Analyzer.slnx index 907a6bd3d..44a858e3f 100644 --- a/Meziantou.Analyzer.slnx +++ b/Meziantou.Analyzer.slnx @@ -2,6 +2,7 @@ + @@ -23,6 +24,7 @@ + diff --git a/tests/Meziantou.Analyzer.Fuzz/AnalysisEngine.cs b/tests/Meziantou.Analyzer.Fuzz/AnalysisEngine.cs new file mode 100644 index 000000000..8d3bbe9b1 --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/AnalysisEngine.cs @@ -0,0 +1,193 @@ +using System.Collections.Concurrent; +using System.Collections.Immutable; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CodeActions; +using Microsoft.CodeAnalysis.CodeFixes; +using Microsoft.CodeAnalysis.Diagnostics; +using Microsoft.CodeAnalysis.Text; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// Run the analyzers on a C# file, then the code fixers on the diagnostics they report, and collect the defects. +/// +internal sealed class AnalysisEngine(ComponentFilter filter, int maxFixedDiagnostics = 20) +{ + public async Task> RunAsync(string source, CancellationToken cancellationToken = default) + { + var findings = new List(); + try + { + using var workspace = new AdhocWorkspace(); + var projectId = ProjectId.CreateNewId(); + var documentId = DocumentId.CreateNewId(projectId); + var solution = workspace.CurrentSolution + .AddProject(ProjectInfo.Create( + projectId, + VersionStamp.Default, + name: "Fuzz", + assemblyName: "Fuzz", + LanguageNames.CSharp, + compilationOptions: Components.CompilationOptions, + parseOptions: Components.ParseOptions, + metadataReferences: Components.MetadataReferences, + analyzerReferences: Components.SourceGenerators)) + .AddDocument(documentId, "Fuzz.cs", SourceText.From(source)); + + var document = solution.GetDocument(documentId)!; + var compilation = (await document.Project.GetCompilationAsync(cancellationToken).ConfigureAwait(false))!; + var diagnostics = await RunAnalyzersAsync(compilation, findings, cancellationToken).ConfigureAwait(false); + if (!filter.CodeFixProviders.IsEmpty) + { + var hasErrors = compilation.GetDiagnostics(cancellationToken).Any(diagnostic => diagnostic.Severity is DiagnosticSeverity.Error); + await RunCodeFixesAsync(document, diagnostics, hasErrors, findings, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + findings.Add(Finding.FromException(FindingKind.HarnessException, "Meziantou.Analyzer.Fuzz", [], ex)); + } + + return [.. findings.DistinctBy(finding => finding.Signature)]; + } + + private async Task> RunAnalyzersAsync(Compilation compilation, List findings, CancellationToken cancellationToken) + { + if (filter.Analyzers.IsEmpty) + return []; + + var exceptions = new ConcurrentQueue<(DiagnosticAnalyzer Analyzer, Exception Exception)>(); + var options = new CompilationWithAnalyzersOptions( + new AnalyzerOptions([]), + onAnalyzerException: (exception, analyzer, _) => exceptions.Enqueue((analyzer, exception)), + concurrentAnalysis: false, // Keep the coverage deterministic, so libFuzzer can tell which mutations are interesting + logAnalyzerExecutionTime: false, + reportSuppressedDiagnostics: false); + + var compilationWithAnalyzers = compilation.WithAnalyzers(filter.Analyzers, options); + var diagnostics = await compilationWithAnalyzers.GetAllDiagnosticsAsync(cancellationToken).ConfigureAwait(false); + foreach (var (analyzer, exception) in exceptions) + { + if (exception is OperationCanceledException) + continue; + + findings.Add(Finding.FromException(FindingKind.AnalyzerException, analyzer, Components.GetDiagnosticIds(analyzer), exception)); + } + + return diagnostics; + } + + private async Task RunCodeFixesAsync(Document document, ImmutableArray diagnostics, bool hasErrors, List findings, CancellationToken cancellationToken) + { + var fixableDiagnostics = diagnostics + .Where(diagnostic => diagnostic.Location.IsInSource && !diagnostic.IsSuppressed && filter.CodeFixProviders.ContainsKey(diagnostic.Id)) + .DistinctBy(diagnostic => (diagnostic.Id, diagnostic.Location.SourceSpan)) + .Take(maxFixedDiagnostics); + + foreach (var diagnostic in fixableDiagnostics) + { + foreach (var provider in filter.CodeFixProviders[diagnostic.Id]) + { + await RunCodeFixAsync(document, provider, diagnostic, hasErrors, findings, cancellationToken).ConfigureAwait(false); + } + } + } + + private static async Task RunCodeFixAsync(Document document, CodeFixProvider provider, Diagnostic diagnostic, bool hasErrors, List findings, CancellationToken cancellationToken) + { + try + { + var actions = new List(); + var context = new CodeFixContext(document, diagnostic, (action, _) => actions.Add(action), cancellationToken); + await provider.RegisterCodeFixesAsync(context).ConfigureAwait(false); + + foreach (var action in Flatten(actions)) + { + var operations = await action.GetOperationsAsync(cancellationToken).ConfigureAwait(false); + var changedSolution = operations.OfType().LastOrDefault()?.ChangedSolution; + if (changedSolution is null || !await HasTextChangesAsync(document.Project.Solution, changedSolution, cancellationToken).ConfigureAwait(false)) + { + findings.Add(new Finding + { + Kind = FindingKind.CodeFixDoesNotChangeCode, + Component = Components.GetName(provider), + DiagnosticIds = [diagnostic.Id], + CodeActionTitle = action.Title, + }); + continue; + } + + // The fuzzed code rarely compiles, and a code fix cannot be expected to fix the errors of the code + if (hasErrors || changedSolution.GetProject(document.Project.Id) is not { } changedProject) + continue; + + var changedCompilation = (await changedProject.GetCompilationAsync(cancellationToken).ConfigureAwait(false))!; + var errors = changedCompilation.GetDiagnostics(cancellationToken).Where(diagnostic => diagnostic.Severity is DiagnosticSeverity.Error).ToList(); + if (errors.Count > 0) + { + var fixedDocument = changedSolution.GetDocument(document.Id); + findings.Add(new Finding + { + Kind = FindingKind.CodeFixBreaksCode, + Component = Components.GetName(provider), + DiagnosticIds = [diagnostic.Id], + CompilerErrors = [.. errors.Select(error => error.Id).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal)], + Message = string.Join('\n', errors.Take(10).Select(error => error.ToString())), + CodeActionTitle = action.Title, + FixedSource = fixedDocument is null ? null : (await fixedDocument.GetTextAsync(cancellationToken).ConfigureAwait(false)).ToString(), + }); + } + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + findings.Add(Finding.FromException(FindingKind.CodeFixException, provider, [diagnostic.Id], ex)); + } + } + + private static IEnumerable Flatten(IEnumerable actions) + { + foreach (var action in actions) + { + if (action.NestedActions.IsEmpty) + { + yield return action; + continue; + } + + foreach (var nestedAction in Flatten(action.NestedActions)) + { + yield return nestedAction; + } + } + } + + private static async Task HasTextChangesAsync(Solution originalSolution, Solution changedSolution, CancellationToken cancellationToken) + { + var changes = changedSolution.GetChanges(originalSolution); + if (changes.GetAddedProjects().Any() || changes.GetRemovedProjects().Any()) + return true; + + foreach (var projectChanges in changes.GetProjectChanges()) + { + if (projectChanges.GetAddedDocuments().Any() || projectChanges.GetRemovedDocuments().Any() + || projectChanges.GetAddedAdditionalDocuments().Any() || projectChanges.GetRemovedAdditionalDocuments().Any() + || projectChanges.GetChangedAdditionalDocuments().Any() || projectChanges.GetAddedAnalyzerConfigDocuments().Any() + || projectChanges.GetRemovedAnalyzerConfigDocuments().Any() || projectChanges.GetChangedAnalyzerConfigDocuments().Any() + || projectChanges.GetAddedMetadataReferences().Any() || projectChanges.GetRemovedMetadataReferences().Any()) + { + return true; + } + + foreach (var documentId in projectChanges.GetChangedDocuments(onlyGetDocumentsWithTextChanges: true)) + { + var originalText = await originalSolution.GetDocument(documentId)!.GetTextAsync(cancellationToken).ConfigureAwait(false); + var changedText = await changedSolution.GetDocument(documentId)!.GetTextAsync(cancellationToken).ConfigureAwait(false); + if (!originalText.ContentEquals(changedText)) + return true; + } + } + + return false; + } +} diff --git a/tests/Meziantou.Analyzer.Fuzz/Campaign.cs b/tests/Meziantou.Analyzer.Fuzz/Campaign.cs new file mode 100644 index 000000000..3ca71529b --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/Campaign.cs @@ -0,0 +1,303 @@ +using System.Globalization; +using System.Text.Json; +using Meziantou.Framework; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// A fuzzing session: start one libFuzzer worker per core until the deadline, triage the inputs that stop a worker, +/// then merge the corpus for the next session and minimize the inputs of the findings. +/// +/// +/// The output folder contains: +/// +/// seeds/: the snippets of the tests, and csharp.dict, the libFuzzer dictionary +/// corpus/: the inputs discovered by the workers, shared by all of them +/// corpus-merged/: the minimal corpus that covers the same code as the corpus and the previous corpus +/// findings/: one folder per finding, see +/// artifacts/, logs/, state/: the crashes saved by libFuzzer, the output of each run, and the files of the workers +/// +/// +internal sealed class Campaign +{ + private const int MaxInputLength = 16 * 1024; + private const int MaxConsecutiveUnexplainedStops = 5; + + private readonly FullPath _libFuzzer; + private readonly FullPath _output; + private readonly FullPath? _previousCorpus; + private readonly int _timeoutSeconds; + private readonly Lock _suppressedComponentsLock = new(); + + private Campaign(FullPath libFuzzer, FullPath output, FullPath? previousCorpus, int timeoutSeconds) + { + _libFuzzer = libFuzzer; + _output = output; + _previousCorpus = previousCorpus is { } path && Directory.Exists(path) && Directory.EnumerateFiles(path).Any() ? path : null; + _timeoutSeconds = timeoutSeconds; + } + + private FullPath SeedsDirectory => _output / "seeds"; + private FullPath DictionaryPath => _output / "seeds" / "csharp.dict"; + private FullPath CorpusDirectory => _output / "corpus"; + private FullPath MergedCorpusDirectory => _output / "corpus-merged"; + private FullPath FindingsDirectory => _output / "findings"; + private FullPath ArtifactsDirectory => _output / "artifacts"; + private FullPath LogsDirectory => _output / "logs"; + private FullPath StateDirectory => _output / "state"; + private FullPath SuppressedComponentsFile => _output / "state" / "suppressed.txt"; + + public static async Task RunAsync(CommandLineArguments arguments) + { + var campaign = new Campaign( + FullPath.FromPath(arguments.GetRequiredString("libfuzzer")), + FullPath.FromPath(arguments.GetRequiredString("output")), + arguments.GetString("previous-corpus") is { } previousCorpus ? FullPath.FromPath(previousCorpus) : null, + arguments.GetInt32("timeout", 30)); + + var workerCount = arguments.GetInt32("workers", Environment.ProcessorCount); + var duration = TimeSpan.FromMinutes(arguments.GetInt32("duration", 60)); + var mergeDuration = TimeSpan.FromMinutes(arguments.GetInt32("merge-minutes", 30)); + var minimizeDuration = TimeSpan.FromMinutes(arguments.GetInt32("minimize-minutes", 30)); + + foreach (var directory in new[] { campaign.CorpusDirectory, campaign.FindingsDirectory, campaign.ArtifactsDirectory, campaign.LogsDirectory, campaign.StateDirectory }) + { + Directory.CreateDirectory(directory); + } + + var seedCount = CorpusExtractor.Extract(FullPath.FromPath(arguments.GetRequiredString("tests")), campaign.SeedsDirectory, campaign.DictionaryPath, workerCount); + Log($"{seedCount} seeds, {workerCount} workers, fuzzing for {duration}"); + + var deadline = DateTimeOffset.UtcNow + duration; + await Task.WhenAll(Enumerable.Range(0, workerCount).Select(index => campaign.RunWorkerAsync(index, deadline))).ConfigureAwait(false); + + await campaign.MergeCorpusAsync(mergeDuration).ConfigureAwait(false); + await campaign.MinimizeFindingsAsync(minimizeDuration).ConfigureAwait(false); + campaign.PrintSummary(); + return 0; + } + + private async Task RunWorkerAsync(int index, DateTimeOffset deadline) + { + var currentInputFile = StateDirectory / $"worker{index}.input"; + var consecutiveUnexplainedStops = 0; + for (var run = 0; ; run++) + { + var remaining = deadline - DateTimeOffset.UtcNow; + if (remaining < TimeSpan.FromMinutes(1)) + return; + + var name = $"worker{index}-run{run}"; + var logFile = LogsDirectory / $"{name}.log"; + var artifactPrefix = $"{ArtifactsDirectory / name}-"; + List libFuzzerArguments = + [ + .. GetCommonArguments(), + $"-seed={(index * 1000) + run + 1}", + $"-max_total_time={(int)remaining.TotalSeconds}", + $"-artifact_prefix={artifactPrefix}", + "-print_final_stats=1", + .. GetHarnessArguments(), + CorpusDirectory, + SeedsDirectory / $"shard-{index}", + ]; + + if (_previousCorpus is { } previousCorpus) + { + libFuzzerArguments.Add(previousCorpus); + } + + var environment = GetHarnessEnvironment(saveFindings: true); + environment[FuzzEnvironment.CurrentInputFileVariable] = currentInputFile; + + Log($"[{name}] Starting libFuzzer for {remaining}"); + var result = await ChildProcess.RunAsync(_libFuzzer, libFuzzerArguments, environment, logFile, remaining + TimeSpan.FromMinutes(5), isolateProcessGroup: true, CancellationToken.None).ConfigureAwait(false); + if (result.ExitCode is 0) + return; + + // libFuzzer saves the inputs that time out or crash, but the harness is another process, so when it dies without reporting + // the crash to libFuzzer, the input is the last one the harness wrote + var timeoutArtifact = Directory.EnumerateFiles(ArtifactsDirectory, $"{name}-timeout-*").FirstOrDefault(); + var crashArtifact = Directory.EnumerateFiles(ArtifactsDirectory, $"{name}-crash-*").FirstOrDefault(); + var (kind, input) = (timeoutArtifact, crashArtifact) switch + { + ({ } path, _) => ("timeout", FullPath.FromPath(path)), + (_, { } path) => ("crash", FullPath.FromPath(path)), + _ => ("crash", CopyCurrentInput(currentInputFile, FullPath.FromPath($"{artifactPrefix}process-crash"))), + }; + + Log($"[{name}] libFuzzer stopped with exit code {result.ExitCode?.ToString(CultureInfo.InvariantCulture) ?? "(timeout)"}, triaging the {kind} of {input}"); + var finding = File.Exists(input) ? await Triage.RunAsync(kind, input, logFile, TimeSpan.FromSeconds(_timeoutSeconds * 2), CancellationToken.None).ConfigureAwait(false) : null; + if (finding is null || finding.Component is "unknown") + { + consecutiveUnexplainedStops++; + Log($"[{name}] Cannot find the component responsible for the {kind} ({consecutiveUnexplainedStops} consecutive)"); + if (consecutiveUnexplainedStops >= MaxConsecutiveUnexplainedStops) + { + Log($"[{name}] Stopping the worker"); + return; + } + } + else + { + consecutiveUnexplainedStops = 0; + } + + if (finding is not null) + { + Log($"[{name}] {finding.Title} ({finding.Signature})"); + FindingStore.Save(FindingsDirectory, finding, Triage.ReadInput(input)); + if (finding.Component is not "unknown") + { + // Do not run the component anymore, so the workers find other issues instead of this one again + lock (_suppressedComponentsLock) + { + File.AppendAllLines(SuppressedComponentsFile, [finding.Component]); + } + } + } + } + } + + private async Task MergeCorpusAsync(TimeSpan duration) + { + Directory.CreateDirectory(MergedCorpusDirectory); + List arguments = ["-merge=1", .. GetCommonArguments(), $"-artifact_prefix={ArtifactsDirectory / "merge"}-", .. GetHarnessArguments(), MergedCorpusDirectory, CorpusDirectory]; + if (_previousCorpus is { } previousCorpus) + { + arguments.Add(previousCorpus); + } + + Log("Merging the corpus"); + var result = await ChildProcess.RunAsync(_libFuzzer, arguments, GetHarnessEnvironment(saveFindings: true), LogsDirectory / "merge.log", duration, isolateProcessGroup: true, CancellationToken.None).ConfigureAwait(false); + if (result.ExitCode is 0) + return; + + Log($"Cannot merge the corpus (exit code: {result.ExitCode?.ToString(CultureInfo.InvariantCulture) ?? "(timeout)"}), keeping all the inputs"); + foreach (var directory in new[] { CorpusDirectory, _previousCorpus }) + { + if (directory is null) + continue; + + foreach (var file in Directory.EnumerateFiles(directory.Value)) + { + File.Copy(file, MergedCorpusDirectory / Path.GetFileName(file), overwrite: true); + } + } + } + + /// + /// Ask libFuzzer to minimize the smallest input of each finding: the harness throws for this finding only, so libFuzzer considers the input as a crash. + /// The crashes and the timeouts are not minimized, as libFuzzer does not consider the death of the harness process as a crash. + /// + private async Task MinimizeFindingsAsync(TimeSpan duration) + { + var deadline = DateTimeOffset.UtcNow + duration; + var engine = new AnalysisEngine(ComponentFilter.Create(only: [], suppressed: new HashSet(StringComparer.Ordinal), runCodeFixes: true)); + foreach (var findingDirectory in Directory.EnumerateDirectories(FindingsDirectory).Select(FullPath.FromPath).Order()) + { + var remaining = deadline - DateTimeOffset.UtcNow; + if (remaining < TimeSpan.FromSeconds(30)) + { + Log("No time left to minimize the other findings"); + return; + } + + var kind = ReadFindingKind(findingDirectory); + if (kind is null or FindingKind.Crash or FindingKind.Timeout or FindingKind.RoslynException or FindingKind.HarnessException || FindingStore.GetSmallestInput(findingDirectory) is not { } input) + continue; + + var signature = findingDirectory.Name; + var minimized = findingDirectory / FindingStore.MinimizedInputFileName; + var seconds = (int)Math.Min(120, remaining.TotalSeconds); + List arguments = ["-minimize_crash=1", $"-max_total_time={seconds}", $"-exact_artifact_path={minimized}", .. GetCommonArguments(), $"-artifact_prefix={StateDirectory / "minimize"}-", .. GetHarnessArguments(), input]; + var environment = GetHarnessEnvironment(saveFindings: false); + environment[FuzzEnvironment.TargetSignatureVariable] = signature; + + Log($"Minimizing {signature}"); + await ChildProcess.RunAsync(_libFuzzer, arguments, environment, LogsDirectory / $"minimize-{signature}.log", TimeSpan.FromSeconds(seconds + 120), isolateProcessGroup: true, CancellationToken.None).ConfigureAwait(false); + + // libFuzzer keeps the inputs that crash, so make sure the minimized input reproduces the same finding + if (File.Exists(minimized)) + { + var findings = await engine.RunAsync(Triage.ReadInput(minimized)).ConfigureAwait(false); + if (findings.FirstOrDefault(finding => finding.Signature == signature) is { } finding) + { + FindingStore.SaveMinimized(findingDirectory, finding); + } + else + { + Log($"The minimized input of {signature} does not reproduce the finding"); + File.Delete(minimized); + } + } + } + } + + private void PrintSummary() + { + var findings = Directory.EnumerateDirectories(FindingsDirectory) + .Select(FullPath.FromPath) + .Select(directory => (Directory: directory, Kind: ReadFindingKind(directory))) + .ToList(); + + Log($"{findings.Count} findings"); + foreach (var group in findings.GroupBy(finding => finding.Kind).OrderBy(group => group.Key)) + { + Log($" {group.Key}: {group.Count()}"); + } + } + + private static FindingKind? ReadFindingKind(FullPath findingDirectory) + { + var path = findingDirectory / FindingStore.FindingFileName; + if (!File.Exists(path)) + return null; + + using var document = JsonDocument.Parse(File.ReadAllText(path)); + return Enum.TryParse(document.RootElement.GetProperty("kind").GetString(), ignoreCase: false, out var kind) ? kind : null; + } + + private IEnumerable GetCommonArguments() + { + yield return $"-timeout={_timeoutSeconds}"; + yield return $"-max_len={MaxInputLength}"; + yield return $"-dict={DictionaryPath}"; + } + + private static IEnumerable GetHarnessArguments() + { + // libfuzzer-dotnet starts the harness with " ", so the harness must be a single file + var (fileName, arguments) = ChildProcess.GetHarnessCommand(); + yield return $"--target_path={fileName}"; + if (arguments.Length > 0) + { + yield return $"--target_arg={arguments[0]}"; + } + } + + private Dictionary GetHarnessEnvironment(bool saveFindings) + { + return new Dictionary(StringComparer.Ordinal) + { + [FuzzEnvironment.FindingsDirectoryVariable] = saveFindings ? FindingsDirectory : null, + [FuzzEnvironment.SuppressedComponentsFileVariable] = SuppressedComponentsFile, + + // Turn the inputs that make an analyzer allocate without limit into OutOfMemoryException, + // instead of letting the operating system kill the process (4 GB) + ["DOTNET_GCHeapHardLimit"] = "0x100000000", + }; + } + + private static FullPath CopyCurrentInput(FullPath currentInputFile, FullPath destination) + { + if (File.Exists(currentInputFile)) + { + File.Copy(currentInputFile, destination, overwrite: true); + } + + return destination; + } + + private static void Log(string message) => Console.WriteLine($"[{DateTimeOffset.UtcNow:HH:mm:ss}] {message}"); +} diff --git a/tests/Meziantou.Analyzer.Fuzz/ChildProcess.cs b/tests/Meziantou.Analyzer.Fuzz/ChildProcess.cs new file mode 100644 index 000000000..7fb5e13fd --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/ChildProcess.cs @@ -0,0 +1,152 @@ +using System.Diagnostics; +using System.Globalization; +using System.Text; + +namespace Meziantou.Analyzer.Fuzz; + +internal static class ChildProcess +{ + /// + /// Run the harness in a new process, so a stack overflow or a hang does not stop the current process. + /// + public static Task RunHarnessAsync(IEnumerable arguments, TimeSpan timeout, CancellationToken cancellationToken) + { + var (fileName, prefix) = GetHarnessCommand(); + return RunAsync(fileName, [.. prefix, .. arguments], environment: null, outputFile: null, timeout, isolateProcessGroup: false, cancellationToken); + } + + /// The command that starts the harness: the dotnet host and the harness assembly, or the apphost. + public static (string FileName, string[] Arguments) GetHarnessCommand() + { + var processPath = Environment.ProcessPath!; + if (string.Equals(Path.GetFileNameWithoutExtension(processPath), "dotnet", StringComparison.OrdinalIgnoreCase)) + return (processPath, [typeof(ChildProcess).Assembly.Location]); + + return (processPath, []); + } + + /// + /// Run a process until it exits or the timeout expires, then kill what remains of it. + /// + /// + /// Start the process in a new process group (Linux only), and kill the whole group when the process exits. libFuzzer does not stop the harness + /// when an input times out, so the harness keeps running the input forever, and keeps the redirected output open, after libFuzzer exits. + /// + public static async Task RunAsync(string fileName, IEnumerable arguments, IReadOnlyDictionary? environment, string? outputFile, TimeSpan timeout, bool isolateProcessGroup, CancellationToken cancellationToken) + { + var startInfo = new ProcessStartInfo(isolateProcessGroup ? "setsid" : fileName) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + + if (isolateProcessGroup) + { + // setsid does not fork, as the processes started by .NET are not process group leaders, + // so the id of the process is the id of the new process group + startInfo.ArgumentList.Add(fileName); + } + + foreach (var argument in arguments) + { + startInfo.ArgumentList.Add(argument); + } + + if (environment is not null) + { + foreach (var (name, value) in environment) + { + startInfo.Environment[name] = value; + } + } + + using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + var output = new StringBuilder(); + var outputLock = new Lock(); + await using var writer = outputFile is null ? null : new StreamWriter(outputFile, append: false, Encoding.UTF8) { AutoFlush = true }; + void OnData(object sender, DataReceivedEventArgs e) + { + if (e.Data is null) + return; + + lock (outputLock) + { + // Only the beginning and the end of the output of libFuzzer matter, so the output in memory is bounded + if (output.Length < 1_000_000) + { + output.AppendLine(e.Data); + } + + writer?.WriteLine(e.Data); + } + } + + // Unlike WaitForExitAsync, the Exited event does not wait for the end of the redirected output + var exited = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + process.Exited += (_, _) => exited.TrySetResult(); + process.OutputDataReceived += OnData; + process.ErrorDataReceived += OnData; + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + var timedOut = false; + try + { + await exited.Task.WaitAsync(timeout, cancellationToken).ConfigureAwait(false); + } + catch (TimeoutException) + { + timedOut = true; + } + finally + { + if (isolateProcessGroup) + { + KillProcessGroup(process.Id); + } + else + { + KillProcessTree(process); + } + } + + // The redirected output ends when all the processes that inherited it exit + try + { + await process.WaitForExitAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None).ConfigureAwait(false); + } + catch (TimeoutException) + { + } + + lock (outputLock) + { + return new ChildProcessResult(timedOut ? null : process.ExitCode, timedOut, output.ToString()); + } + } + + public static void KillProcessTree(Process process) + { + try + { + process.Kill(entireProcessTree: true); + } + catch (InvalidOperationException) + { + // The process already exited + } + } + + private static void KillProcessGroup(int processGroupId) + { + // A negative id sends the signal to all the processes of the group + var startInfo = new ProcessStartInfo("kill") { UseShellExecute = false }; + startInfo.ArgumentList.Add("-KILL"); + startInfo.ArgumentList.Add("--"); + startInfo.ArgumentList.Add((-processGroupId).ToString(CultureInfo.InvariantCulture)); + using var process = Process.Start(startInfo)!; + process.WaitForExit(); + } +} diff --git a/tests/Meziantou.Analyzer.Fuzz/ChildProcessResult.cs b/tests/Meziantou.Analyzer.Fuzz/ChildProcessResult.cs new file mode 100644 index 000000000..4059d820a --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/ChildProcessResult.cs @@ -0,0 +1,7 @@ +namespace Meziantou.Analyzer.Fuzz; + +internal sealed record ChildProcessResult(int? ExitCode, bool TimedOut, string Output) +{ + /// The harness exits with 0 when there is no finding and 1 when there are findings, any other exit code is a crash. + public bool IsHarnessCrash => !TimedOut && ExitCode is not (0 or 1); +} diff --git a/tests/Meziantou.Analyzer.Fuzz/CommandLineArguments.cs b/tests/Meziantou.Analyzer.Fuzz/CommandLineArguments.cs new file mode 100644 index 000000000..f59cd4e97 --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/CommandLineArguments.cs @@ -0,0 +1,51 @@ +using System.Globalization; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// The arguments of a subcommand: --name value options, --name flags (when followed by another option or nothing), and positional values. +/// +internal sealed class CommandLineArguments +{ + private static readonly HashSet Flags = new(StringComparer.Ordinal) { "no-fixes", "json" }; + + private readonly Dictionary _options = new(StringComparer.Ordinal); + + public List Positional { get; } = []; + + public static CommandLineArguments Parse(ReadOnlySpan args) + { + var result = new CommandLineArguments(); + for (var i = 0; i < args.Length; i++) + { + var arg = args[i]; + if (arg.StartsWith("--", StringComparison.Ordinal)) + { + var name = arg[2..]; + if (Flags.Contains(name) || i + 1 >= args.Length) + { + result._options[name] = null; + } + else + { + result._options[name] = args[i + 1]; + i++; + } + } + else + { + result.Positional.Add(arg); + } + } + + return result; + } + + public bool HasFlag(string name) => _options.ContainsKey(name); + + public string? GetString(string name) => _options.GetValueOrDefault(name); + + public string GetRequiredString(string name) => GetString(name) ?? throw new ArgumentException($"The option '--{name}' is required", nameof(name)); + + public int GetInt32(string name, int defaultValue) => GetString(name) is { } value ? int.Parse(value, CultureInfo.InvariantCulture) : defaultValue; +} diff --git a/tests/Meziantou.Analyzer.Fuzz/ComponentFilter.cs b/tests/Meziantou.Analyzer.Fuzz/ComponentFilter.cs new file mode 100644 index 000000000..7c81fea9e --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/ComponentFilter.cs @@ -0,0 +1,65 @@ +using System.Collections.Immutable; +using Microsoft.CodeAnalysis.CodeFixes; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// The analyzers and code fixers to run on an input. +/// +internal sealed class ComponentFilter +{ + private ComponentFilter(ImmutableArray analyzers, ImmutableArray codeFixProviders) + { + Analyzers = analyzers; + CodeFixProviders = codeFixProviders + .SelectMany(provider => provider.FixableDiagnosticIds.Select(id => (Id: id, Provider: provider))) + .GroupBy(item => item.Id, StringComparer.Ordinal) + .ToImmutableDictionary(group => group.Key, group => group.Select(item => item.Provider).ToImmutableArray(), StringComparer.Ordinal); + } + + public ImmutableArray Analyzers { get; } + + public ImmutableDictionary> CodeFixProviders { get; } + + /// + /// Create a filter. + /// + /// + /// The full names of the components to run. When they are analyzers, only these analyzers run, without code fixers. + /// When they are code fixers, all the analyzers run, so the diagnostics the code fixers fix are reported. + /// + /// The full names of the components not to run. + /// Whether to run the code fixers. + public static ComponentFilter Create(IReadOnlyCollection only, IReadOnlySet suppressed, bool runCodeFixes) + { + foreach (var name in only) + { + if (Components.FindByName(name) is null) + throw new ArgumentException($"Cannot find the analyzer or the code fixer '{name}'", nameof(only)); + } + + var onlyAnalyzers = Components.Analyzers.Where(analyzer => only.Contains(Components.GetName(analyzer))).ToImmutableArray(); + var onlyCodeFixProviders = Components.CodeFixProviders.Where(provider => only.Contains(Components.GetName(provider))).ToImmutableArray(); + + var analyzers = !onlyAnalyzers.IsEmpty + ? onlyAnalyzers + : Components.Analyzers.Where(analyzer => !suppressed.Contains(Components.GetName(analyzer))).ToImmutableArray(); + + ImmutableArray codeFixProviders; + if (!runCodeFixes || (!onlyAnalyzers.IsEmpty && onlyCodeFixProviders.IsEmpty)) + { + codeFixProviders = []; + } + else if (!onlyCodeFixProviders.IsEmpty) + { + codeFixProviders = onlyCodeFixProviders; + } + else + { + codeFixProviders = Components.CodeFixProviders.Where(provider => !suppressed.Contains(Components.GetName(provider))).ToImmutableArray(); + } + + return new ComponentFilter(analyzers, codeFixProviders); + } +} diff --git a/tests/Meziantou.Analyzer.Fuzz/Components.cs b/tests/Meziantou.Analyzer.Fuzz/Components.cs new file mode 100644 index 000000000..33ee2bff9 --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/Components.cs @@ -0,0 +1,150 @@ +using System.Collections.Immutable; +using System.Globalization; +using System.Reflection; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CodeFixes; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// The analyzers and code fixers under test, and the options used to compile the fuzzed code. +/// They are created once per process, as creating them is much slower than analyzing an input. +/// +internal static class Components +{ + private static readonly Assembly[] Assemblies = [typeof(Rules.CommaAnalyzer).Assembly, typeof(Rules.CommaFixer).Assembly]; + + // The assemblies of the application that must not be referenced by the fuzzed code + private static readonly HashSet ExcludedReferences = new(StringComparer.OrdinalIgnoreCase) + { + "dnlib", + "Meziantou.Analyzer", + "Meziantou.Analyzer.CodeFixers", + "Meziantou.Analyzer.Fuzz", + "Meziantou.Framework.FullPath", + "SharpFuzz", + "SharpFuzz.Common", + }; + + /// The analyzers and the diagnostic suppressors. + public static ImmutableArray Analyzers { get; } = CreateInstances(); + + public static ImmutableArray CodeFixProviders { get; } = CreateInstances(); + + /// + /// The assemblies of the application: the .NET and ASP.NET Core frameworks, and the libraries referenced by the project file. + /// + public static ImmutableArray MetadataReferences { get; } = CreateMetadataReferences(); + + /// + /// The source generators of the .NET reference pack, such as the one that implements the members annotated with [GeneratedRegex], + /// so the code fixes that rely on them produce code that compiles. + /// + public static ImmutableArray SourceGenerators { get; } = CreateSourceGenerators(); + + public static CSharpParseOptions ParseOptions { get; } = new(LanguageVersion.Preview); + + /// + /// The rules disabled by default are enabled, so all the rules run. is the option of the tests, + /// which some rules need to analyze the non-public members of the referenced assemblies. + /// + public static CSharpCompilationOptions CompilationOptions { get; } = new CSharpCompilationOptions( + OutputKind.DynamicallyLinkedLibrary, + nullableContextOptions: NullableContextOptions.Enable, + allowUnsafe: true) + .WithMetadataImportOptions(MetadataImportOptions.All) + .WithSpecificDiagnosticOptions(Analyzers + .SelectMany(analyzer => analyzer.SupportedDiagnostics) + .Select(descriptor => descriptor.Id) + .Distinct(StringComparer.Ordinal) + .Select(id => KeyValuePair.Create(id, ReportDiagnostic.Warn))); + + public static IEnumerable All => Analyzers.Cast().Concat(CodeFixProviders); + + public static string GetName(object component) => component.GetType().FullName!; + + public static ImmutableArray GetDiagnosticIds(object component) + { + IEnumerable ids = component switch + { + DiagnosticSuppressor suppressor => suppressor.SupportedSuppressions.Select(suppression => suppression.SuppressedDiagnosticId), + DiagnosticAnalyzer analyzer => analyzer.SupportedDiagnostics.Select(descriptor => descriptor.Id), + CodeFixProvider codeFixProvider => codeFixProvider.FixableDiagnosticIds, + _ => [], + }; + + return [.. ids.Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal)]; + } + + public static object? FindByName(string name) => All.FirstOrDefault(component => GetName(component) == name); + + /// + /// Find the analyzer or the code fixer that declares the first frame of a stack trace that belongs to one. + /// The stack traces separate the nested types with a '.', whereas uses a '+'. + /// + public static object? FindByStackFrames(IEnumerable frames) + { + foreach (var frame in frames) + { + var component = All + .Where(component => frame.StartsWith(GetName(component).Replace('+', '.') + ".", StringComparison.Ordinal)) + .MaxBy(component => GetName(component).Length); + if (component is not null) + return component; + } + + return null; + } + + private static ImmutableArray CreateInstances() + { + return + [ + .. Assemblies + .SelectMany(assembly => assembly.GetExportedTypes()) + .Where(type => !type.IsAbstract && typeof(T).IsAssignableFrom(type)) + .OrderBy(type => type.FullName, StringComparer.Ordinal) + .Select(type => (T)Activator.CreateInstance(type)!), + ]; + } + + private static ImmutableArray CreateSourceGenerators() + { + // The runtime is in /shared/Microsoft.NETCore.App/, and the reference pack of the SDK in /packs/Microsoft.NETCore.App.Ref/ + var runtimeDirectory = new DirectoryInfo(Path.GetDirectoryName(typeof(object).Assembly.Location)!); + var packDirectory = new DirectoryInfo(Path.Combine(runtimeDirectory.Parent!.Parent!.Parent!.FullName, "packs", "Microsoft.NETCore.App.Ref")); + if (!packDirectory.Exists) + return []; + + var versionDirectory = packDirectory.EnumerateDirectories() + .Where(directory => directory.Name == runtimeDirectory.Name || directory.Name.StartsWith(Environment.Version.Major.ToString(CultureInfo.InvariantCulture) + ".", StringComparison.Ordinal)) + .OrderByDescending(directory => directory.Name == runtimeDirectory.Name) + .ThenByDescending(directory => Version.TryParse(directory.Name, out var version) ? version : null) + .FirstOrDefault(); + + var generatorDirectory = versionDirectory is null ? null : new DirectoryInfo(Path.Combine(versionDirectory.FullName, "analyzers", "dotnet", "cs")); + if (generatorDirectory is not { Exists: true }) + return []; + + var paths = generatorDirectory.EnumerateFiles("*.dll").Select(file => file.FullName).Order(StringComparer.Ordinal).ToList(); + var loader = new GeneratorAssemblyLoader(paths); + return [.. paths.Select(path => new AnalyzerFileReference(path, loader))]; + } + + private static ImmutableArray CreateMetadataReferences() + { + var paths = ((string?)AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES"))?.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries) ?? []; + return + [ + .. paths + .Where(path => Path.GetFileNameWithoutExtension(path) is var name + && !ExcludedReferences.Contains(name) + && !name.StartsWith("Microsoft.CodeAnalysis", StringComparison.OrdinalIgnoreCase) + && !name.StartsWith("System.Composition", StringComparison.OrdinalIgnoreCase)) + .Order(StringComparer.Ordinal) + .Select(path => MetadataReference.CreateFromFile(path)), + ]; + } +} diff --git a/tests/Meziantou.Analyzer.Fuzz/CorpusExtractor.cs b/tests/Meziantou.Analyzer.Fuzz/CorpusExtractor.cs new file mode 100644 index 000000000..39271e923 --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/CorpusExtractor.cs @@ -0,0 +1,136 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using Meziantou.Framework; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// Create the seed corpus from the code snippets of the tests, which are raw string literals, and the libFuzzer dictionary from the C# tokens. +/// +internal static partial class CorpusExtractor +{ + private const int MinSnippetLength = 20; + + private static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); + + // Common identifiers of the APIs the rules know about, so the mutations can produce code that the rules analyze + private static readonly string[] DictionaryIdentifiers = + [ + "Task", "ValueTask", "CancellationToken", "ConfigureAwait", "GetAwaiter", "Result", "Wait", "Thread", "Sleep", + "IDisposable", "IAsyncDisposable", "Dispose", "DisposeAsync", "Equals", "GetHashCode", "ToString", "CompareTo", + "StringComparison", "StringComparer", "Ordinal", "OrdinalIgnoreCase", "CultureInfo", "InvariantCulture", "IFormatProvider", + "Enumerable", "Select", "Where", "First", "FirstOrDefault", "Any", "Count", "ToList", "ToArray", "OrderBy", + "List", "Dictionary", "IEnumerable", "Span", "ReadOnlySpan", "Array", "Empty", + "Regex", "RegexOptions", "DateTime", "Now", "UtcNow", "DateTimeOffset", "TimeSpan", "Guid", "Encoding", + "Console", "WriteLine", "Debug", "Assert", "Exception", "ArgumentException", "ArgumentNullException", "nameof", + "Attribute", "Obsolete", "Flags", "Serializable", "StructLayout", "DllImport", "LibraryImport", "CallerMemberName", + "EventHandler", "EventArgs", "Lazy", "Lock", "Monitor", "Interlocked", "Volatile", "Stream", "File", "Path", + "HttpClient", "JsonSerializer", "ILogger", "LoggerMessage", "Fact", "Test", "TestMethod", "Theory", "InlineData", + ]; + + public static int RunCommand(CommandLineArguments arguments) + { + var count = Extract( + FullPath.FromPath(arguments.GetRequiredString("tests")), + FullPath.FromPath(arguments.GetRequiredString("output")), + FullPath.FromPath(arguments.GetRequiredString("dictionary")), + arguments.GetInt32("shards", 1)); + + Console.WriteLine($"{count} snippets extracted"); + return 0; + } + + /// + /// Write the snippets to <output>/shard-<n>, so each libFuzzer worker starts with a part of the seeds only, + /// as running all the seeds before fuzzing takes a long time. + /// + public static int Extract(FullPath testsDirectory, FullPath outputDirectory, FullPath dictionaryPath, int shardCount) + { + var snippets = new SortedDictionary(StringComparer.Ordinal); + foreach (var file in Directory.EnumerateFiles(testsDirectory, "*.cs", SearchOption.AllDirectories)) + { + var tree = CSharpSyntaxTree.ParseText(File.ReadAllText(file), Components.ParseOptions); + foreach (var token in tree.GetRoot().DescendantTokens()) + { + if (!token.IsKind(SyntaxKind.SingleLineRawStringLiteralToken) && !token.IsKind(SyntaxKind.MultiLineRawStringLiteralToken)) + continue; + + var snippet = RemoveMarkup(token.ValueText); + if (snippet.Length < MinSnippetLength || !snippet.AsSpan().ContainsAny('{', ';')) + continue; + + snippets.TryAdd(Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(snippet))), snippet); + } + } + + var index = 0; + foreach (var (hash, snippet) in snippets) + { + var path = outputDirectory / $"shard-{(index % shardCount).ToString(CultureInfo.InvariantCulture)}" / hash; + path.CreateParentDirectory(); + File.WriteAllText(path, snippet, Utf8NoBom); + index++; + } + + dictionaryPath.CreateParentDirectory(); + File.WriteAllLines(dictionaryPath, GetDictionaryEntries().Distinct(StringComparer.Ordinal).Select(FormatDictionaryEntry)); + return snippets.Count; + } + + // The tests mark the expected diagnostics with [|code|] and {|id:code|} + private static string RemoveMarkup(string snippet) + { + snippet = DiagnosticWithIdMarkupRegex().Replace(snippet, ""); + return snippet.Replace("|}", "", StringComparison.Ordinal).Replace("[|", "", StringComparison.Ordinal).Replace("|]", "", StringComparison.Ordinal); + } + + private static IEnumerable GetDictionaryEntries() + { + foreach (var kind in SyntaxFacts.GetKeywordKinds().Concat(SyntaxFacts.GetContextualKeywordKinds()).Concat(SyntaxFacts.GetPunctuationKinds())) + { + var text = SyntaxFacts.GetText(kind); + if (text.Length > 0) + yield return text; + } + + foreach (var text in new[] { "\"\"\"", "$\"", "@\"", "u8", "#nullable enable", "#pragma warning disable", "#if DEBUG", "#endif", "#region", "#endregion", "///", "/// ", "" }) + { + yield return text; + } + + foreach (var identifier in DictionaryIdentifiers) + { + yield return identifier; + } + } + + // https://llvm.org/docs/LibFuzzer.html#dictionaries + private static string FormatDictionaryEntry(string value) + { + var sb = new StringBuilder("\""); + foreach (var b in Encoding.UTF8.GetBytes(value)) + { + if (b is (byte)'"' or (byte)'\\') + { + sb.Append('\\').Append((char)b); + } + else if (b is >= 0x20 and < 0x7F) + { + sb.Append((char)b); + } + else + { + sb.Append(CultureInfo.InvariantCulture, $"\\x{b:X2}"); + } + } + + return sb.Append('"').ToString(); + } + + [GeneratedRegex(@"\{\|[^:|{}]*:", RegexOptions.None, matchTimeoutMilliseconds: 10_000)] + private static partial Regex DiagnosticWithIdMarkupRegex(); +} diff --git a/tests/Meziantou.Analyzer.Fuzz/Finding.cs b/tests/Meziantou.Analyzer.Fuzz/Finding.cs new file mode 100644 index 000000000..509235f96 --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/Finding.cs @@ -0,0 +1,114 @@ +using System.Collections.Immutable; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// A defect found by the fuzzer. The findings with the same are the same defect. +/// +internal sealed class Finding +{ + private const int SignatureFrameCount = 5; + + public required FindingKind Kind { get; init; } + + /// The full name of the analyzer or of the code fixer. + public required string Component { get; init; } + + /// The rules of the analyzer, or the rule whose diagnostic the code fixer fixes. + public ImmutableArray DiagnosticIds { get; init; } = []; + + public string? ExceptionType { get; init; } + + public string? Message { get; init; } + + public string? StackTrace { get; init; } + + /// The frames of the analyzer assemblies, normalized so they do not depend on the compiler-generated names. + public ImmutableArray Frames { get; init; } = []; + + /// The errors that the code fix introduces. + public ImmutableArray CompilerErrors { get; init; } = []; + + public string? CodeActionTitle { get; init; } + + /// The code after the code fix is applied. + public string? FixedSource { get; init; } + + public string Signature => field ??= ComputeSignature(); + + public string Title => field ??= ComputeTitle(); + + public static Finding FromException(FindingKind kind, object component, ImmutableArray diagnosticIds, Exception exception) + { + while (exception is AggregateException { InnerExceptions.Count: 1 } or TargetInvocationException { InnerException: not null }) + { + exception = exception.InnerException!; + } + + var text = exception.ToString(); + var frames = StackTraceParser.GetAnalyzerFrames(text); + return new Finding + { + Kind = frames.IsEmpty && kind is FindingKind.AnalyzerException or FindingKind.CodeFixException ? FindingKind.RoslynException : kind, + Component = component as string ?? Components.GetName(component), + DiagnosticIds = diagnosticIds, + ExceptionType = exception.GetType().FullName, + Message = exception.Message, + StackTrace = text, + Frames = frames, + }; + } + + private string ComputeSignature() + { + IEnumerable parts = Kind switch + { + FindingKind.CodeFixBreaksCode => [Kind.ToString(), Component, .. DiagnosticIds, .. CompilerErrors], + FindingKind.CodeFixDoesNotChangeCode or FindingKind.Timeout => [Kind.ToString(), Component, .. DiagnosticIds], + _ => [Kind.ToString(), Component, ExceptionType ?? "", .. Frames.Take(SignatureFrameCount)], + }; + + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(string.Join('\n', parts))); + return Convert.ToHexStringLower(hash)[..16]; + } + + private string ComputeTitle() + { + var rules = DiagnosticIds.Length switch + { + 0 => "", + <= 3 => string.Join(", ", DiagnosticIds) + " ", + _ => string.Join(", ", DiagnosticIds.Take(3)) + ", … ", + }; + + var component = GetShortName(Component); + var location = Frames.IsEmpty ? "" : " in " + GetShortFrame(Frames[0]); + var description = Kind switch + { + FindingKind.AnalyzerException or FindingKind.CodeFixException or FindingKind.RoslynException or FindingKind.HarnessException => $"throws {GetShortName(ExceptionType ?? "an exception")}{location}", + FindingKind.CodeFixBreaksCode => $"code fix introduces {string.Join(", ", CompilerErrors)}", + FindingKind.CodeFixDoesNotChangeCode => "registers a code fix that does not change the code", + FindingKind.Crash => $"crashes the process{(Message is null ? "" : ": " + Message)}{location}", + FindingKind.Timeout => "does not complete", + _ => Kind.ToString(), + }; + + return $"[Fuzzing] {rules}({component}) {description}"; + } + + private static string GetShortName(string name) => name[(name.LastIndexOfAny(['.', '+']) + 1)..]; + + private static string GetShortFrame(string frame) + { + var index = frame.AsSpan().LastIndexOf('.'); + if (index > 0) + { + index = frame.AsSpan(0, index).LastIndexOf('.'); + } + + return frame[(index + 1)..]; + } +} diff --git a/tests/Meziantou.Analyzer.Fuzz/FindingKind.cs b/tests/Meziantou.Analyzer.Fuzz/FindingKind.cs new file mode 100644 index 000000000..8cf44f4e8 --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/FindingKind.cs @@ -0,0 +1,34 @@ +using System.Text.Json.Serialization; + +namespace Meziantou.Analyzer.Fuzz; + +[JsonConverter(typeof(JsonStringEnumConverter))] +internal enum FindingKind +{ + /// An analyzer or a diagnostic suppressor throws (AD0001). + AnalyzerException, + + /// A code fixer throws when it registers its code fixes, or when one of them is computed or applied. + CodeFixException, + + /// The input compiles without errors, but a code fix introduces compiler errors. + CodeFixBreaksCode, + + /// A code fixer registers a code fix that does not change the code. + CodeFixDoesNotChangeCode, + + /// The process crashes, e.g. on a stack overflow. + Crash, + + /// An analyzer or a code fixer does not complete. + Timeout, + + /// + /// Roslyn throws while it runs an analyzer or applies a code fix, and no frame of the analyzer assemblies is in the stack trace, + /// e.g. the formatter fails on the invalid code of the input. These are not reported, as they are rarely defects of the rules. + /// + RoslynException, + + /// The harness itself or Roslyn throws, outside of an analyzer or a code fixer. + HarnessException, +} diff --git a/tests/Meziantou.Analyzer.Fuzz/FindingStore.cs b/tests/Meziantou.Analyzer.Fuzz/FindingStore.cs new file mode 100644 index 000000000..f089f6387 --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/FindingStore.cs @@ -0,0 +1,100 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Meziantou.Framework; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// Save the inputs that reproduce a finding as <signature>/input-<hash>.cs, each one with the details of the finding +/// it produces in <signature>/input-<hash>.json (the stack trace and the fixed code depend on the input). +/// <signature>/finding.json is the finding of the first input. +/// Several workers save to the same folder, so the files are only created, never updated. +/// +internal static class FindingStore +{ + public const string FindingFileName = "finding.json"; + public const string MinimizedInputFileName = "minimized.cs"; + + // The smallest inputs are kept, as they are the easiest to understand and to minimize + private const int MaxInputsPerFinding = 5; + + private static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); + private static readonly Dictionary SmallestInputLengths = new(StringComparer.Ordinal); + private static readonly Lock SmallestInputLengthsLock = new(); + + public static JsonSerializerOptions JsonOptions { get; } = new(JsonSerializerDefaults.Web) + { + RespectNullableAnnotations = true, + RespectRequiredConstructorParameters = true, + WriteIndented = true, + }; + + public static JsonSerializerOptions CompactJsonOptions { get; } = new(JsonOptions) { WriteIndented = false }; + + public static void Save(FullPath directory, Finding finding, string input) + { + lock (SmallestInputLengthsLock) + { + if (SmallestInputLengths.TryGetValue(finding.Signature, out var length) && length <= input.Length) + return; + + SmallestInputLengths[finding.Signature] = input.Length; + } + + var findingDirectory = directory / finding.Signature; + Directory.CreateDirectory(findingDirectory); + var json = JsonSerializer.Serialize(finding, JsonOptions); + var inputPath = findingDirectory / $"input-{Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(input)))}.cs"; + TryCreateFile(findingDirectory / FindingFileName, json); + TryCreateFile(GetFindingPath(inputPath), json); + TryCreateFile(inputPath, input); + + foreach (var file in GetInputs(findingDirectory).Skip(MaxInputsPerFinding)) + { + try + { + File.Delete(file); + File.Delete(GetFindingPath(file)); + } + catch (IOException) + { + } + } + } + + /// Save the finding produced by the minimized input of a finding. + public static void SaveMinimized(FullPath findingDirectory, Finding finding) + { + File.WriteAllText(GetFindingPath(findingDirectory / MinimizedInputFileName), JsonSerializer.Serialize(finding, JsonOptions), Utf8NoBom); + } + + /// The file that contains the finding produced by an input. + public static FullPath GetFindingPath(FullPath inputPath) => inputPath.ChangeExtension(".json"); + + /// The inputs of a finding, the smallest first. + public static IEnumerable GetInputs(FullPath findingDirectory) + { + return Directory.EnumerateFiles(findingDirectory, "input-*.cs") + .Select(path => new FileInfo(path)) + .OrderBy(file => file.Length) + .ThenBy(file => file.Name, StringComparer.Ordinal) + .Select(file => FullPath.FromPath(file.FullName)); + } + + public static FullPath? GetSmallestInput(FullPath findingDirectory) => GetInputs(findingDirectory).Select(path => (FullPath?)path).FirstOrDefault(); + + private static void TryCreateFile(FullPath path, string content) + { + try + { + using var stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None); + using var writer = new StreamWriter(stream, Utf8NoBom); + writer.Write(content); + } + catch (IOException) when (File.Exists(path)) + { + // Another worker saved the same file + } + } +} diff --git a/tests/Meziantou.Analyzer.Fuzz/FuzzEnvironment.cs b/tests/Meziantou.Analyzer.Fuzz/FuzzEnvironment.cs new file mode 100644 index 000000000..df30f531c --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/FuzzEnvironment.cs @@ -0,0 +1,42 @@ +using Meziantou.Framework; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// The environment variables that configure the harness. libFuzzer starts the harness with a single argument, +/// so the settings of the fuzzing session cannot be passed on the command line. +/// +internal static class FuzzEnvironment +{ + /// The folder where the findings are saved. Nothing is saved when it is not set. + public const string FindingsDirectoryVariable = "FUZZ_FINDINGS_DIR"; + + /// A file listing the full names of the analyzers and code fixers not to run, one per line, such as the ones that already crashed the process. + public const string SuppressedComponentsFileVariable = "FUZZ_SUPPRESSED_FILE"; + + /// When set, the harness throws for the finding with this signature, so libFuzzer can minimize its input, and saves nothing. + public const string TargetSignatureVariable = "FUZZ_TARGET_SIGNATURE"; + + /// The file where the harness writes each input before running it, as libFuzzer does not save the input that kills the harness process. + public const string CurrentInputFileVariable = "FUZZ_CURRENT_INPUT_FILE"; + + public static FullPath? FindingsDirectory => GetPath(FindingsDirectoryVariable); + + public static FullPath? SuppressedComponentsFile => GetPath(SuppressedComponentsFileVariable); + + public static string? TargetSignature => GetString(TargetSignatureVariable); + + public static FullPath? CurrentInputFile => GetPath(CurrentInputFileVariable); + + public static IReadOnlySet GetSuppressedComponents() + { + if (SuppressedComponentsFile is not { } path || !File.Exists(path)) + return new HashSet(StringComparer.Ordinal); + + return File.ReadAllLines(path).Select(line => line.Trim()).Where(line => line.Length > 0).ToHashSet(StringComparer.Ordinal); + } + + private static string? GetString(string name) => Environment.GetEnvironmentVariable(name) is { Length: > 0 } value ? value : null; + + private static FullPath? GetPath(string name) => GetString(name) is { } value ? FullPath.FromPath(value) : null; +} diff --git a/tests/Meziantou.Analyzer.Fuzz/FuzzFindingException.cs b/tests/Meziantou.Analyzer.Fuzz/FuzzFindingException.cs new file mode 100644 index 000000000..7db3b837c --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/FuzzFindingException.cs @@ -0,0 +1,24 @@ +namespace Meziantou.Analyzer.Fuzz; + +/// Report a finding to libFuzzer as a crash, so libFuzzer can minimize its input. +internal sealed class FuzzFindingException : Exception +{ + public FuzzFindingException() + { + } + + public FuzzFindingException(string message) + : base(message) + { + } + + public FuzzFindingException(string message, Exception innerException) + : base(message, innerException) + { + } + + public FuzzFindingException(Finding finding) + : base($"{finding.Title} ({finding.Signature})\n{finding.StackTrace}") + { + } +} diff --git a/tests/Meziantou.Analyzer.Fuzz/FuzzTarget.cs b/tests/Meziantou.Analyzer.Fuzz/FuzzTarget.cs new file mode 100644 index 000000000..e736cc538 --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/FuzzTarget.cs @@ -0,0 +1,57 @@ +using System.Text; +using SharpFuzz; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// The function libFuzzer calls for each input. The findings are saved instead of being reported to libFuzzer as crashes, +/// as libFuzzer stops on the first crash. Only the inputs that kill the process or that do not complete stop libFuzzer. +/// +internal sealed class FuzzTarget +{ + private const int MaxInputLength = 64 * 1024; + + private readonly AnalysisEngine _engine; + private readonly string? _targetSignature = FuzzEnvironment.TargetSignature; + private readonly Meziantou.Framework.FullPath? _findingsDirectory = FuzzEnvironment.FindingsDirectory; + private readonly Meziantou.Framework.FullPath? _currentInputFile = FuzzEnvironment.CurrentInputFile; + + private FuzzTarget() + { + _engine = new AnalysisEngine(ComponentFilter.Create(only: [], FuzzEnvironment.GetSuppressedComponents(), runCodeFixes: true)); + } + + public static void RunLibFuzzer() + { + // The instrumented assemblies must not run before libFuzzer.Run allocates the coverage buffer, which includes their module initializers + FuzzTarget? target = null; + Fuzzer.LibFuzzer.Run(data => (target ??= new FuzzTarget()).Execute(data)); + } + + private void Execute(ReadOnlySpan data) + { + if (data.Length > MaxInputLength) + return; + + if (_currentInputFile is { } currentInputFile) + { + File.WriteAllBytes(currentInputFile, data); + } + + var source = Encoding.UTF8.GetString(data); + var findings = _engine.RunAsync(source).GetAwaiter().GetResult(); + foreach (var finding in findings) + { + if (_targetSignature is not null) + { + // Report the finding to libFuzzer, so it can minimize the input + if (finding.Signature == _targetSignature) + throw new FuzzFindingException(finding); + } + else if (_findingsDirectory is { } findingsDirectory) + { + FindingStore.Save(findingsDirectory, finding, source); + } + } + } +} diff --git a/tests/Meziantou.Analyzer.Fuzz/GeneratorAssemblyLoader.cs b/tests/Meziantou.Analyzer.Fuzz/GeneratorAssemblyLoader.cs new file mode 100644 index 000000000..a57cafd0a --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/GeneratorAssemblyLoader.cs @@ -0,0 +1,37 @@ +using System.Reflection; +using System.Runtime.Loader; +using Microsoft.CodeAnalysis; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// Load the source generators of the .NET reference pack in their own context, as the tests do (see GeneratorAssemblyLoader in the test project). +/// The assemblies the generators share with the harness, such as Microsoft.CodeAnalysis, resolve to the ones already loaded. +/// +internal sealed class GeneratorAssemblyLoader : IAnalyzerAssemblyLoader +{ + private readonly Dictionary _pathsBySimpleName = new(StringComparer.OrdinalIgnoreCase); + private readonly AssemblyLoadContext _context; + + public GeneratorAssemblyLoader(IEnumerable paths) + { + foreach (var path in paths) + { + _pathsBySimpleName[Path.GetFileNameWithoutExtension(path)] = path; + } + + _context = new AssemblyLoadContext("SourceGenerators", isCollectible: false); + _context.Resolving += Resolve; + } + + public void AddDependencyLocation(string fullPath) + { + } + + public Assembly LoadFromPath(string fullPath) => _context.LoadFromAssemblyPath(fullPath); + + private Assembly? Resolve(AssemblyLoadContext context, AssemblyName assemblyName) => + assemblyName.Name is { } simpleName && _pathsBySimpleName.TryGetValue(simpleName, out var path) + ? context.LoadFromAssemblyPath(path) + : null; +} diff --git a/tests/Meziantou.Analyzer.Fuzz/Meziantou.Analyzer.Fuzz.csproj b/tests/Meziantou.Analyzer.Fuzz/Meziantou.Analyzer.Fuzz.csproj new file mode 100644 index 000000000..3e7a5340c --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/Meziantou.Analyzer.Fuzz.csproj @@ -0,0 +1,38 @@ + + + + Exe + false + true + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/Meziantou.Analyzer.Fuzz/Program.cs b/tests/Meziantou.Analyzer.Fuzz/Program.cs new file mode 100644 index 000000000..5c98d0372 --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/Program.cs @@ -0,0 +1,41 @@ +using Meziantou.Analyzer.Fuzz; + +// libFuzzer starts the harness without arguments, and sends the inputs through shared memory +if (args.Length is 0) +{ + FuzzTarget.RunLibFuzzer(); + return 0; +} + +// The assemblies are instrumented when the harness runs from the output of eng/run.ps1, +// and the instrumentation writes to a buffer that only libFuzzer allocates +SharpFuzzTrace.EnsureInitialized(); + +var arguments = CommandLineArguments.Parse(args.AsSpan(1)); +return args[0] switch +{ + "repro" => await ReproCommand.RunAsync(arguments), + "triage" => await Triage.RunCommandAsync(arguments), + "corpus" => CorpusExtractor.RunCommand(arguments), + "campaign" => await Campaign.RunAsync(arguments), + _ => PrintUsage(), +}; + +static int PrintUsage() +{ + Console.Error.WriteLine(""" + Usage: + Meziantou.Analyzer.Fuzz + Run under libFuzzer (see eng/run.ps1) + Meziantou.Analyzer.Fuzz repro [--only [,...]] [--no-fixes] ... + Run the analyzers and the code fixers on the inputs and print the findings + Meziantou.Analyzer.Fuzz triage --kind --input [--log ] [--timeout ] + Find the rule that crashes the process or does not complete + Meziantou.Analyzer.Fuzz corpus --tests --output --dictionary [--shards ] + Extract the code snippets of the tests to use as the seed corpus + Meziantou.Analyzer.Fuzz campaign --libfuzzer --tests --output [--duration ] [--workers ] + [--timeout ] [--previous-corpus ] [--merge-minutes ] [--minimize-minutes ] + Fuzz with several libFuzzer workers, triage the crashes and minimize the findings + """); + return 2; +} diff --git a/tests/Meziantou.Analyzer.Fuzz/README.md b/tests/Meziantou.Analyzer.Fuzz/README.md new file mode 100644 index 000000000..b91d10aac --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/README.md @@ -0,0 +1,79 @@ +# Fuzz tests + +The fuzz tests run all the analyzers and the code fixers on C# code that [libFuzzer](https://llvm.org/docs/LibFuzzer.html) mutates, +starting from the code snippets of the tests. [SharpFuzz](https://github.com/Metalnem/sharpfuzz) instruments `Meziantou.Analyzer.dll` +and `Meziantou.Analyzer.CodeFixers.dll`, so libFuzzer keeps the mutations that execute new branches of the rules. + +The harness reports these defects: + +| Kind | Description | +|------|-------------| +| `AnalyzerException` | An analyzer or a diagnostic suppressor throws (AD0001) | +| `CodeFixException` | A code fixer throws when it registers its code fixes, or when one of them is computed or applied | +| `CodeFixBreaksCode` | The input compiles without errors, but a code fix introduces compiler errors | +| `CodeFixDoesNotChangeCode` | A code fixer registers a code fix that does not change the code | +| `Crash` | The process crashes, e.g. on a stack overflow | +| `Timeout` | An analyzer or a code fixer does not complete within 60 seconds (twice the libFuzzer timeout) | + +The exceptions thrown by Roslyn without a frame of the analyzers in their stack trace (`RoslynException`), such as the formatter failing on the invalid code +of an input, and the exceptions of the harness itself (`HarnessException`) are saved, but no issue is opened for them. + +All the rules are enabled, including the ones disabled by default, and the code is compiled with the .NET and ASP.NET Core +assemblies of the harness, the libraries referenced by the project file, and the source generators of the .NET SDK. + +Each defect has a signature, computed from its kind, the analyzer or the code fixer, the exception, and the frames of the analyzer. +The findings are saved in `findings//`: the smallest inputs that reproduce them (`input-*.cs`), the input minimized by libFuzzer +(`minimized.cs`), and the details of the finding produced by each input (`input-*.json`, `minimized.json`, and `finding.json` for the first one). + +## Weekly run + +The [fuzz workflow](../../.github/workflows/fuzz.yml) fuzzes for 4 hours every Sunday, starting from the snippets of the tests and the corpus of the +previous run. It then opens an issue labeled `fuzzing` for each new finding, at most 10 per run. The signature is in the body of the issue, so: + +- a finding with an open issue, or with an issue closed as not planned, is not reported again +- a finding with an issue closed as completed is reported in a new issue, as a regression + +The workflow can be started manually with a shorter duration, and without opening issues. The findings, the corpus, and the logs are attached to the run. + +## Reproducing a finding + +The `repro` command runs the analyzers and the code fixers on files, on any OS, and prints the findings with their details: + +```shell +dotnet run --project tests/Meziantou.Analyzer.Fuzz --configuration Release -- repro input.cs +``` + +`--only ` runs only some analyzers or code fixers (full type names, separated by commas), and `--no-fixes` does not run the code fixers. +A folder runs all its files, for instance the corpus of a run or the snippets of the tests: + +```shell +dotnet run --project tests/Meziantou.Analyzer.Fuzz --configuration Release -- corpus --tests tests/Meziantou.Analyzer.Test --output artifacts/fuzz-seeds --dictionary artifacts/fuzz-seeds/csharp.dict +dotnet run --project tests/Meziantou.Analyzer.Fuzz --configuration Release -- repro artifacts/fuzz-seeds +``` + +## Fuzzing locally + +libFuzzer requires Linux, or WSL on Windows. Install SharpFuzz and [libfuzzer-dotnet](https://github.com/Metalnem/libfuzzer-dotnet/releases), then run: + +```shell +dotnet tool install --global SharpFuzz.CommandLine +pwsh tests/Meziantou.Analyzer.Fuzz/eng/run.ps1 -LibFuzzer ./libfuzzer-dotnet-ubuntu -DurationMinutes 30 +``` + +The script publishes the harness to `artifacts/fuzz/bin`, instruments the analyzers, and starts the campaign: + +1. One libFuzzer worker per core, sharing the corpus. Each worker starts with a part of the snippets of the tests. +2. The analyzers save their findings without stopping libFuzzer. When an input crashes the process or does not complete, libFuzzer stops, + the campaign finds the responsible analyzer or code fixer by replaying the input with groups of them, disables it, and restarts the worker. +3. The corpus is merged in `artifacts/fuzz/corpus-merged`, to pass to the next campaign with `-PreviousCorpus`. +4. libFuzzer minimizes the input of each finding, except the crashes and the timeouts. This only works on x64: libfuzzer-dotnet reports + the findings with `__builtin_trap()`, which raises `SIGTRAP` on Arm64, and libFuzzer does not handle this signal. + +`pwsh tests/Meziantou.Analyzer.Fuzz/eng/report.ps1 -FindingsDirectory artifacts/fuzz/findings -DryRun` prints the issues without creating them. + +## Not covered yet + +- The options of the rules (`.editorconfig`), which use their default values +- The Fix All providers and the refactorings +- The Roslyn versions other than the default one +- The analysis of generated code (`MEZIANTOU_ANALYZER_GENERATED_CODE`) diff --git a/tests/Meziantou.Analyzer.Fuzz/ReproCommand.cs b/tests/Meziantou.Analyzer.Fuzz/ReproCommand.cs new file mode 100644 index 000000000..75ae0c903 --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/ReproCommand.cs @@ -0,0 +1,88 @@ +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using Meziantou.Framework; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// Run the analyzers and the code fixers on files without libFuzzer, to reproduce a finding on any OS. +/// +internal static class ReproCommand +{ + public static async Task RunAsync(CommandLineArguments arguments) + { + var only = arguments.GetString("only")?.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) ?? []; + var filter = ComponentFilter.Create(only, FuzzEnvironment.GetSuppressedComponents(), runCodeFixes: !arguments.HasFlag("no-fixes")); + var json = arguments.HasFlag("json"); + var findingsDirectory = FuzzEnvironment.FindingsDirectory; + + var files = arguments.Positional.SelectMany(GetFiles).ToList(); + if (files.Count is 0) + { + Console.Error.WriteLine("No input file"); + return 2; + } + + var engine = new AnalysisEngine(filter); + var signatures = new HashSet(StringComparer.Ordinal); + var findingCount = 0; + var stopwatch = Stopwatch.StartNew(); + foreach (var file in files) + { + var source = Encoding.UTF8.GetString(await File.ReadAllBytesAsync(file).ConfigureAwait(false)); + var findings = await engine.RunAsync(source).ConfigureAwait(false); + foreach (var finding in findings) + { + findingCount++; + signatures.Add(finding.Signature); + if (findingsDirectory is { } directory) + { + FindingStore.Save(directory, finding, source); + } + + if (json) + { + Console.WriteLine(JsonSerializer.Serialize(new { file = file.Value, finding }, FindingStore.CompactJsonOptions)); + } + else + { + Console.WriteLine($"{file}: {finding.Title} ({finding.Signature})"); + if (files.Count is 1) + { + PrintDetails(finding); + } + } + } + } + + Console.Error.WriteLine($"{files.Count} inputs, {findingCount} findings, {signatures.Count} distinct findings, {stopwatch.Elapsed.TotalMilliseconds / files.Count:F0} ms per input"); + return findingCount > 0 ? 1 : 0; + } + + private static void PrintDetails(Finding finding) + { + foreach (var value in new[] { finding.CodeActionTitle, finding.Message, finding.StackTrace, finding.FixedSource }) + { + if (value is not null) + { + Console.WriteLine(value); + Console.WriteLine(); + } + } + } + + private static IEnumerable GetFiles(string path) + { + var fullPath = FullPath.FromPath(path); + if (Directory.Exists(fullPath)) + { + // libFuzzer names the inputs of the corpus after their hash, without extension + return Directory.EnumerateFiles(fullPath, "*", SearchOption.AllDirectories) + .Order(StringComparer.Ordinal) + .Select(FullPath.FromPath); + } + + return [fullPath]; + } +} diff --git a/tests/Meziantou.Analyzer.Fuzz/SharpFuzzTrace.cs b/tests/Meziantou.Analyzer.Fuzz/SharpFuzzTrace.cs new file mode 100644 index 000000000..0b16ec243 --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/SharpFuzzTrace.cs @@ -0,0 +1,24 @@ +using System.Runtime.InteropServices; + +namespace Meziantou.Analyzer.Fuzz; + +internal static class SharpFuzzTrace +{ + // The size of the coverage map of libFuzzer (MAP_SIZE in libfuzzer-dotnet) + private const int MapSize = 1 << 16; + + /// + /// The instrumented assemblies record the coverage in , which is only allocated + /// when the harness runs under libFuzzer. Allocate it, so the instrumented assemblies can run outside of libFuzzer. + /// + public static void EnsureInitialized() + { + unsafe + { + if (SharpFuzz.Common.Trace.SharedMem is null) + { + SharpFuzz.Common.Trace.SharedMem = (byte*)NativeMemory.AllocZeroed(MapSize); + } + } + } +} diff --git a/tests/Meziantou.Analyzer.Fuzz/StackTraceParser.cs b/tests/Meziantou.Analyzer.Fuzz/StackTraceParser.cs new file mode 100644 index 000000000..3fe0a714b --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/StackTraceParser.cs @@ -0,0 +1,73 @@ +using System.Collections.Immutable; +using System.Globalization; +using System.Text.RegularExpressions; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// Extract the frames of the analyzer from the text of an exception, or from the output of a process that crashed (e.g. "Stack overflow."). +/// +internal static partial class StackTraceParser +{ + public static ImmutableArray GetAnalyzerFrames(string text) + { + return [.. GetFrames(text).Where(IsAnalyzerFrame).Select(Normalize)]; + } + + /// The lines of the text that are stack frames, at most , with the consecutive duplicates of a stack overflow collapsed. + public static string GetFrameLines(string text, int maxCount) + { + var lines = new List(); + string? previous = null; + var repeatCount = 0; + foreach (var line in FrameRegex().Matches(text).Select(match => match.Value.TrimEnd())) + { + if (line == previous) + { + repeatCount++; + continue; + } + + AddRepeatCount(); + if (lines.Count >= maxCount) + break; + + lines.Add(line); + previous = line; + } + + AddRepeatCount(); + return string.Join('\n', lines); + + void AddRepeatCount() + { + if (repeatCount > 0) + { + lines.Add($" ... repeated {repeatCount.ToString(CultureInfo.InvariantCulture)} more times"); + repeatCount = 0; + } + } + } + + private static IEnumerable GetFrames(string text) => FrameRegex().Matches(text).Select(match => match.Groups["method"].Value.Trim()); + + // Meziantou.Framework.Roslyn is a source package, compiled into the analyzer assemblies + private static bool IsAnalyzerFrame(string method) + { + return (method.StartsWith("Meziantou.Analyzer.", StringComparison.Ordinal) && !method.StartsWith("Meziantou.Analyzer.Fuzz.", StringComparison.Ordinal)) + || method.StartsWith("Meziantou.Framework.Roslyn.", StringComparison.Ordinal); + } + + // The names generated by the compiler for the lambdas, the local functions and the state machines contain + // ordinals that change when the code is edited, so they are removed to keep the signatures of the findings stable + private static string Normalize(string method) + { + return string.Join('.', method.Split('.').Select(segment => segment.AsSpan().IndexOfAny('<', '>', '|') >= 0 ? DigitsRegex().Replace(segment, "") : segment)); + } + + [GeneratedRegex(@"^[ \t]*at (?[^(\r\n]+)\(.*$", RegexOptions.Multiline | RegexOptions.ExplicitCapture, matchTimeoutMilliseconds: 10_000)] + private static partial Regex FrameRegex(); + + [GeneratedRegex("[0-9]+", RegexOptions.None, matchTimeoutMilliseconds: 10_000)] + private static partial Regex DigitsRegex(); +} diff --git a/tests/Meziantou.Analyzer.Fuzz/Triage.cs b/tests/Meziantou.Analyzer.Fuzz/Triage.cs new file mode 100644 index 000000000..cb182c9fa --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/Triage.cs @@ -0,0 +1,127 @@ +using System.Collections.Immutable; +using System.Text; +using System.Text.Json; +using Meziantou.Framework; + +namespace Meziantou.Analyzer.Fuzz; + +/// +/// Find the analyzer or the code fixer responsible for an input that stops libFuzzer: the harness process crashed, or did not complete in time. +/// +internal static class Triage +{ + public static async Task RunCommandAsync(CommandLineArguments arguments) + { + var kind = arguments.GetRequiredString("kind"); + var input = FullPath.FromPath(arguments.GetRequiredString("input")); + var log = arguments.GetString("log") is { } logPath ? FullPath.FromPath(logPath) : (FullPath?)null; + var timeout = TimeSpan.FromSeconds(arguments.GetInt32("timeout", 60)); + + var finding = await RunAsync(kind, input, log, timeout, CancellationToken.None).ConfigureAwait(false); + if (finding is null) + { + Console.WriteLine("The input does not reproduce the issue"); + return 0; + } + + Console.WriteLine(JsonSerializer.Serialize(finding, FindingStore.JsonOptions)); + return 1; + } + + /// Find the component responsible for the input, or when the input does not reproduce the issue. + public static async Task RunAsync(string kind, FullPath input, FullPath? log, TimeSpan timeout, CancellationToken cancellationToken) + { + if (kind is "timeout") + return await FindComponentThatDoesNotCompleteAsync(input, timeout, cancellationToken).ConfigureAwait(false); + + var output = log is { } logPath && File.Exists(logPath) ? await File.ReadAllTextAsync(logPath, cancellationToken).ConfigureAwait(false) : ""; + if (StackTraceParser.GetAnalyzerFrames(output).IsEmpty) + { + var result = await ChildProcess.RunHarnessAsync(["repro", input], timeout, cancellationToken).ConfigureAwait(false); + if (result.TimedOut) + return await FindComponentThatDoesNotCompleteAsync(input, timeout, cancellationToken).ConfigureAwait(false); + + if (!result.IsHarnessCrash) + return null; + + output = result.Output; + } + + var frames = StackTraceParser.GetAnalyzerFrames(output); + var component = Components.FindByStackFrames(frames); + return new Finding + { + Kind = FindingKind.Crash, + Component = component is null ? "unknown" : Components.GetName(component), + DiagnosticIds = component is null ? [] : Components.GetDiagnosticIds(component), + Message = GetCrashReason(output), + StackTrace = StackTraceParser.GetFrameLines(output, maxCount: 100), + Frames = frames, + }; + } + + private static string? GetCrashReason(string output) + { + if (output.Contains("Stack overflow", StringComparison.Ordinal)) + return "stack overflow"; + + if (output.Contains("OutOfMemoryException", StringComparison.Ordinal)) + return "out of memory"; + + return output.Split('\n').FirstOrDefault(line => line.StartsWith("Unhandled exception.", StringComparison.Ordinal))?.Trim(); + } + + /// + /// Replay the input with groups of components, keeping the group that does not complete, until one component remains. + /// The analyzers run without the code fixers first, as the code fixers need the diagnostics of all the analyzers. + /// + private static async Task FindComponentThatDoesNotCompleteAsync(FullPath input, TimeSpan timeout, CancellationToken cancellationToken) + { + var analyzersResult = await ChildProcess.RunHarnessAsync(["repro", "--no-fixes", input], timeout, cancellationToken).ConfigureAwait(false); + ImmutableArray candidates; + string[] extraArguments; + if (analyzersResult.TimedOut) + { + candidates = [.. Components.Analyzers]; + extraArguments = ["--no-fixes"]; + } + else + { + var allResult = await ChildProcess.RunHarnessAsync(["repro", input], timeout, cancellationToken).ConfigureAwait(false); + if (!allResult.TimedOut) + return null; + + candidates = [.. Components.CodeFixProviders]; + extraArguments = []; + } + + var groupCount = Math.Max(2, Environment.ProcessorCount); + while (candidates.Length > 1) + { + var groups = candidates.Chunk((candidates.Length + groupCount - 1) / groupCount).ToList(); + var results = await Task.WhenAll(groups.Select(group => + ChildProcess.RunHarnessAsync(["repro", "--only", string.Join(',', group.Select(Components.GetName)), .. extraArguments, input], timeout, cancellationToken))).ConfigureAwait(false); + + var index = Array.FindIndex(results, result => result.TimedOut); + if (index < 0) + { + // The components only hang together, or the hang is not deterministic + candidates = []; + break; + } + + candidates = [.. groups[index]]; + } + + var component = candidates.Length is 1 ? candidates[0] : null; + return new Finding + { + Kind = FindingKind.Timeout, + Component = component is null ? "unknown" : Components.GetName(component), + DiagnosticIds = component is null ? [] : Components.GetDiagnosticIds(component), + Message = $"The input is not analyzed within {timeout.TotalSeconds:F0} seconds", + }; + } + + public static string ReadInput(FullPath path) => Encoding.UTF8.GetString(File.ReadAllBytes(path)); +} diff --git a/tests/Meziantou.Analyzer.Fuzz/eng/report.ps1 b/tests/Meziantou.Analyzer.Fuzz/eng/report.ps1 new file mode 100644 index 000000000..195990fd4 --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/eng/report.ps1 @@ -0,0 +1,200 @@ +<# +.SYNOPSIS +Open a GitHub issue for each new finding of a fuzzing session. + +.DESCRIPTION +The issues are labeled "fuzzing" and contain the signature of their finding, so a finding is reported once: +- an open issue, or an issue closed as not planned or duplicate, means the finding is known, and nothing is done +- an issue closed as completed means the finding is a regression, and a new issue references it +#> +param( + [Parameter(Mandatory = $true)] + [string]$FindingsDirectory, + [string]$Repository = $env:GITHUB_REPOSITORY, + [string]$CommitSha = $env:GITHUB_SHA, + [string]$RunUrl, + [int]$MaxIssues = 10, + [switch]$DryRun +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true + +$label = "fuzzing" + +# The body of an issue is limited to 65536 characters +$maxInputLength = 20000 +$maxDetailsLength = 12000 + +function Get-Fence([string]$text) { + # A fence longer than any sequence of backticks of the content, so the content cannot close it + $longest = 2 + foreach ($match in [regex]::Matches($text, '`+')) { + $longest = [Math]::Max($longest, $match.Length) + } + + return '`' * ($longest + 1) +} + +function Format-CodeBlock([string]$text, [string]$language, [int]$maxLength) { + $suffix = "" + if ($text.Length -gt $maxLength) { + $text = $text.Substring(0, $maxLength) + $suffix = "`n_Truncated, the full content is in the fuzz-findings artifact of the run._`n" + } + + $fence = Get-Fence $text + return "$fence$language`n$($text.TrimEnd())`n$fence`n$suffix" +} + +function Get-ExistingIssues([string]$signature) { + $json = gh issue list --repo $Repository --label $label --state all --search "$signature in:body" --json "number,state,stateReason" --limit 20 + return @($json | ConvertFrom-Json) +} + +function Get-IssueBody($finding, [string]$inputText, [string]$inputName, $regressionOf) { + $lines = [System.Collections.Generic.List[string]]::new() + if ($regressionOf) { + $lines.Add("This finding was fixed in #$regressionOf, but it reproduces again.") + $lines.Add("") + } + + $source = "the [fuzz tests](https://github.com/$Repository/blob/main/tests/Meziantou.Analyzer.Fuzz/README.md)" + if ($RunUrl) { + $source += " ([run]($RunUrl))" + } + + $lines.Add("This issue was found by $source.") + $lines.Add("") + $lines.Add("- Kind: ``$($finding.kind)``") + if ($finding.diagnosticIds.Count -gt 0) { + $lines.Add("- Rules: $(($finding.diagnosticIds | ForEach-Object { "[$_](https://github.com/$Repository/blob/main/docs/Rules/$_.md)" }) -join ', ')") + } + + $lines.Add("- Component: ``$($finding.component)``") + if ($finding.codeActionTitle) { + $lines.Add("- Code action: ``$($finding.codeActionTitle)``") + } + + if ($CommitSha) { + $lines.Add("- Commit: $CommitSha") + } + + $lines.Add("- Signature: ``$($finding.signature)``") + $lines.Add("") + $lines.Add("## Input") + $lines.Add("") + $lines.Add((Format-CodeBlock $inputText "cs" $maxInputLength)) + + if ($finding.stackTrace) { + $lines.Add("## Stack trace") + $lines.Add("") + $lines.Add((Format-CodeBlock $finding.stackTrace "" $maxDetailsLength)) + } + elseif ($finding.message) { + $lines.Add("## Details") + $lines.Add("") + $lines.Add((Format-CodeBlock $finding.message "" $maxDetailsLength)) + } + + if ($finding.fixedSource) { + $lines.Add("## Code after the code fix") + $lines.Add("") + $lines.Add((Format-CodeBlock $finding.fixedSource "cs" $maxDetailsLength)) + } + + $lines.Add("## Reproduce") + $lines.Add("") + $lines.Add("Save the input as ``$inputName``, then run:") + $lines.Add("") + $lines.Add('```shell') + $lines.Add("dotnet run --project tests/Meziantou.Analyzer.Fuzz --configuration Release -- repro $inputName") + $lines.Add('```') + $lines.Add("") + $lines.Add("") + return $lines -join "`n" +} + +if (-not (Test-Path $FindingsDirectory)) { + Write-Host "No finding" + exit 0 +} + +$findings = @(Get-ChildItem $FindingsDirectory -Directory | ForEach-Object { + $findingFile = Join-Path $_.FullName "finding.json" + if (Test-Path $findingFile) { + $minimized = Join-Path $_.FullName "minimized.cs" + $inputFile = if (Test-Path $minimized) { Get-Item $minimized } else { Get-ChildItem $_.FullName -Filter "input-*.cs" | Sort-Object Length, Name | Select-Object -First 1 } + if ($inputFile) { + # The stack trace and the fixed code depend on the input, so they are read from the file of the input + $inputFindingFile = [System.IO.Path]::ChangeExtension($inputFile.FullName, ".json") + if (Test-Path $inputFindingFile) { + $findingFile = $inputFindingFile + } + + [pscustomobject]@{ + Finding = Get-Content $findingFile -Raw | ConvertFrom-Json + InputFile = $inputFile + } + } + } + } | Sort-Object { $_.Finding.kind }, { $_.Finding.signature }) + +Write-Host "$($findings.Count) findings" +if (-not $DryRun) { + gh label create $label --repo $Repository --color "D93F0B" --description "Found by the fuzz tests" --force +} + +$createdCount = 0 +foreach ($item in $findings) { + $finding = $item.Finding + + # The exceptions of the harness or of Roslyn outside of the rules are rarely defects of the analyzers + if ($finding.kind -in "RoslynException", "HarnessException" -or $finding.component -eq "unknown") { + Write-Host "Skipping $($finding.signature): $($finding.title)" + continue + } + + $regressionOf = $null + if (-not $DryRun) { + $existingIssues = Get-ExistingIssues $finding.signature + $knownIssue = $existingIssues | Where-Object { $_.state -eq "OPEN" -or $_.stateReason -ne "COMPLETED" } | Select-Object -First 1 + if ($knownIssue) { + Write-Host "Known in #$($knownIssue.number): $($finding.title)" + continue + } + + $fixedIssue = $existingIssues | Sort-Object number -Descending | Select-Object -First 1 + if ($fixedIssue) { + $regressionOf = $fixedIssue.number + } + } + + if ($createdCount -ge $MaxIssues) { + Write-Host "Not reported, as $MaxIssues issues were already created: $($finding.title)" + continue + } + + $inputText = [System.IO.File]::ReadAllText($item.InputFile.FullName) + $body = Get-IssueBody $finding $inputText "$($finding.signature).cs" $regressionOf + if ($DryRun) { + Write-Host "::group::$($finding.title)" + Write-Host $body + Write-Host "::endgroup::" + } + else { + $bodyFile = New-TemporaryFile + try { + [System.IO.File]::WriteAllText($bodyFile.FullName, $body) + gh issue create --repo $Repository --title $finding.title --body-file $bodyFile.FullName --label $label --label bug + } + finally { + Remove-Item $bodyFile + } + } + + $createdCount++ +} + +Write-Host "$createdCount issues created" diff --git a/tests/Meziantou.Analyzer.Fuzz/eng/run.ps1 b/tests/Meziantou.Analyzer.Fuzz/eng/run.ps1 new file mode 100644 index 000000000..e7446fb34 --- /dev/null +++ b/tests/Meziantou.Analyzer.Fuzz/eng/run.ps1 @@ -0,0 +1,64 @@ +<# +.SYNOPSIS +Build the fuzzing harness, instrument the analyzers with SharpFuzz, and fuzz them with libFuzzer. + +.DESCRIPTION +Requires Linux (or WSL), the SharpFuzz.CommandLine tool (dotnet tool install --global SharpFuzz.CommandLine), +and libfuzzer-dotnet (https://github.com/Metalnem/libfuzzer-dotnet/releases). +The findings are saved in /findings, and the corpus to reuse in the next session in /corpus-merged. +#> +param( + [Parameter(Mandatory = $true)] + [string]$LibFuzzer, + [int]$DurationMinutes = 60, + [int]$Workers = [Environment]::ProcessorCount, + [int]$TimeoutSeconds = 30, + [string]$OutputDirectory, + [string]$PreviousCorpus, + [int]$MergeMinutes = 30, + [int]$MinimizeMinutes = 30, + [string]$SharpFuzz = "sharpfuzz" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true + +$repositoryRoot = (Resolve-Path (Join-Path $PSScriptRoot "../../..")).Path +if (-not $OutputDirectory) { + $OutputDirectory = Join-Path $repositoryRoot "artifacts/fuzz" +} + +$OutputDirectory = [System.IO.Path]::GetFullPath($OutputDirectory) +$LibFuzzer = (Resolve-Path $LibFuzzer).Path +$binDirectory = Join-Path $OutputDirectory "bin" + +if (Test-Path $binDirectory) { + Remove-Item -Recurse -Force $binDirectory +} + +dotnet publish (Join-Path $PSScriptRoot "../Meziantou.Analyzer.Fuzz.csproj") --configuration Release --output $binDirectory + +# Only the analyzers are instrumented, so the coverage that guides libFuzzer is the one of the rules, not the one of Roslyn +foreach ($assembly in "Meziantou.Analyzer.dll", "Meziantou.Analyzer.CodeFixers.dll") { + Write-Host "Instrumenting $assembly" + & $SharpFuzz (Join-Path $binDirectory $assembly) +} + +$arguments = @( + "campaign", + "--libfuzzer", $LibFuzzer, + "--tests", (Join-Path $repositoryRoot "tests/Meziantou.Analyzer.Test"), + "--output", $OutputDirectory, + "--duration", $DurationMinutes, + "--workers", $Workers, + "--timeout", $TimeoutSeconds, + "--merge-minutes", $MergeMinutes, + "--minimize-minutes", $MinimizeMinutes +) + +if ($PreviousCorpus) { + $arguments += @("--previous-corpus", [System.IO.Path]::GetFullPath($PreviousCorpus)) +} + +dotnet (Join-Path $binDirectory "Meziantou.Analyzer.Fuzz.dll") @arguments From 743ec09ded7eb1d49fb0e34cc5978249e45995a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9rald=20Barr=C3=A9?= Date: Sat, 17 Oct 2026 16:23:22 -0400 Subject: [PATCH 2/3] Reference a patched System.Security.Cryptography.Pkcs NuGet.Packaging, referenced by the Roslyn testing packages and by NuGet.Protocol, depends on versions of System.Security.Cryptography.Pkcs that have known vulnerabilities (GHSA-846f-f764-76mx, GHSA-2fmv-rv2c-59x7), and the NuGet audit fails the restore of the test projects and of ListDotNetTypes. --- src/ListDotNetTypes/ListDotNetTypes.csproj | 2 ++ tests/Meziantou.Analyzer.Test/Directory.Build.props | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/ListDotNetTypes/ListDotNetTypes.csproj b/src/ListDotNetTypes/ListDotNetTypes.csproj index 5f05c3994..d3d7a68a1 100644 --- a/src/ListDotNetTypes/ListDotNetTypes.csproj +++ b/src/ListDotNetTypes/ListDotNetTypes.csproj @@ -7,6 +7,8 @@ + + diff --git a/tests/Meziantou.Analyzer.Test/Directory.Build.props b/tests/Meziantou.Analyzer.Test/Directory.Build.props index 968ba7c7f..f5ce32916 100644 --- a/tests/Meziantou.Analyzer.Test/Directory.Build.props +++ b/tests/Meziantou.Analyzer.Test/Directory.Build.props @@ -28,6 +28,8 @@ + + From 2d6f0d8416049d708e3b66d1e955edb799561c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9rald=20Barr=C3=A9?= Date: Sat, 17 Oct 2026 16:32:21 -0400 Subject: [PATCH 3/3] Run the NuGet migrations before the concurrent CI steps, schedule the fuzzing in Toronto time The first dotnet command of a runner runs the NuGet migrations under a named mutex, and creating the mutex intermittently fails (IOException on 'NuGet-Migrations') when the parallel and background steps start dotnet at the same time. Running one dotnet command first creates the migration file, so the concurrent commands skip the mutex. The weekly fuzzing runs at 03:00 America/Toronto instead of 03:00 UTC. --- .github/workflows/ci.yml | 4 ++++ .github/workflows/fuzz.yml | 1 + 2 files changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d19386b7..116e96eab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,6 +90,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Setup .NET Core uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6 + # The first dotnet command of a runner runs the NuGet migrations, which can fail when several dotnet commands start them at the same time + - run: dotnet nuget locals global-packages --list # create_nuget skips the analyzers and dotnet test does not run them, so this job reports their diagnostics - parallel: # Meziantou.Analyzer.Pack references one project per Roslyn version, so this builds all of them @@ -107,6 +109,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Setup .NET Core uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6 + # The first dotnet command of a runner runs the NuGet migrations, which can fail when several dotnet commands start them at the same time + - run: dotnet nuget locals global-packages --list # The analyzers are skipped as the build_with_analyzers job runs them on the same projects # Meziantou.Analyzer.Annotations does not depend on the analyzer, so it is packed while the analyzer is built and packed diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index ba02601ef..64707f9a0 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -14,6 +14,7 @@ on: default: true schedule: - cron: "0 3 * * 0" + timezone: "America/Toronto" concurrency: group: fuzz