From 2fbe248456816a4ade284b625ca1edcfc093c4bf Mon Sep 17 00:00:00 2001 From: Jared Parsons Date: Sun, 5 May 2024 10:48:22 -0700 Subject: [PATCH] Support for manipulating generated files (#128) * Get generated files binary log closes #125 * Command for dumping generated files to disk * Generated command * more tests * more tests --- .../BinaryLogReaderTests.cs | 13 +++ .../CompilationDataTests.cs | 13 +++ .../ProgramTests.cs | 73 +++++++++++-- .../SolutionFixture.cs | 13 +++ src/Basic.CompilerLog.Util/BinaryLogReader.cs | 14 +++ src/Basic.CompilerLog.Util/CompilationData.cs | 26 ++++- src/Basic.CompilerLog/Constants.cs | 1 - src/Basic.CompilerLog/Program.cs | 103 +++++++++++++++--- 8 files changed, 232 insertions(+), 24 deletions(-) diff --git a/src/Basic.CompilerLog.UnitTests/BinaryLogReaderTests.cs b/src/Basic.CompilerLog.UnitTests/BinaryLogReaderTests.cs index 677e847..15b8161 100644 --- a/src/Basic.CompilerLog.UnitTests/BinaryLogReaderTests.cs +++ b/src/Basic.CompilerLog.UnitTests/BinaryLogReaderTests.cs @@ -147,5 +147,18 @@ public void ReadDeletedPdb() var data = reader.ReadAllCompilationData().Single(); var diagnostic = data.GetDiagnostics().Where(x => x.Severity == DiagnosticSeverity.Error).Single(); Assert.Contains("Can't find portable pdb file for", diagnostic.GetMessage()); + + Assert.Throws(() => reader.ReadAllGeneratedFiles(data.CompilerCall)); + } + + [Fact] + public void ReadGeneratedFiles() + { + using var reader = BinaryLogReader.Create(Fixture.Console.Value.BinaryLogPath!, BasicAnalyzerKind.None); + var compilerCall = reader.ReadAllCompilerCalls().Single(); + var generatedFiles = reader.ReadAllGeneratedFiles(compilerCall); + Assert.Single(generatedFiles); + var tuple = generatedFiles.Single(); + Assert.True(tuple.Stream.TryGetBuffer(out var _)); } } diff --git a/src/Basic.CompilerLog.UnitTests/CompilationDataTests.cs b/src/Basic.CompilerLog.UnitTests/CompilationDataTests.cs index b73e1ca..5331bda 100644 --- a/src/Basic.CompilerLog.UnitTests/CompilationDataTests.cs +++ b/src/Basic.CompilerLog.UnitTests/CompilationDataTests.cs @@ -124,4 +124,17 @@ public void GetCompilationAfterGeneratorsDiagnostics() _ = data.GetCompilationAfterGenerators(out var diagnostics); Assert.NotEmpty(diagnostics); } + + [Fact] + public void GetGeneratedSyntaxTrees() + { + using var reader = CompilerLogReader.Create(Fixture.Console.Value.CompilerLogPath); + var data = reader.ReadAllCompilationData().Single(); + var trees = data.GetGeneratedSyntaxTrees(); + Assert.Single(trees); + + trees = data.GetGeneratedSyntaxTrees(out var diagnostics); + Assert.Single(trees); + Assert.Empty(diagnostics); + } } diff --git a/src/Basic.CompilerLog.UnitTests/ProgramTests.cs b/src/Basic.CompilerLog.UnitTests/ProgramTests.cs index dea649d..2d46c40 100644 --- a/src/Basic.CompilerLog.UnitTests/ProgramTests.cs +++ b/src/Basic.CompilerLog.UnitTests/ProgramTests.cs @@ -292,7 +292,7 @@ public void ResponseSingleLine() { var exitCode = RunCompLog($"rsp {Fixture.SolutionBinaryLogPath} -p console.csproj -s"); Assert.Equal(Constants.ExitSuccess, exitCode); - var rsp = Path.Combine(RootDirectory, @".complog", "console", "build.rsp"); + var rsp = Path.Combine(RootDirectory, @".complog", "rsp", "console", "build.rsp"); Assert.True(File.Exists(rsp)); var lines = File.ReadAllLines(rsp); @@ -316,7 +316,7 @@ public void ResponseProjectFilter() { var exitCode = RunCompLog($"rsp {Fixture.SolutionBinaryLogPath} -p console.csproj"); Assert.Equal(Constants.ExitSuccess, exitCode); - var rsp = Path.Combine(RootDirectory, @".complog", "console", "build.rsp"); + var rsp = Path.Combine(RootDirectory, @".complog", "rsp", "console", "build.rsp"); Assert.True(File.Exists(rsp)); Assert.Contains("Program.cs", File.ReadAllLines(rsp)); } @@ -342,7 +342,7 @@ public void ResponseAll() { var exitCode = RunCompLog($"rsp {Fixture.SolutionBinaryLogPath}"); Assert.Equal(Constants.ExitSuccess, exitCode); - var rsp = Path.Combine(RootDirectory, @".complog", "console", "build.rsp"); + var rsp = Path.Combine(RootDirectory, @".complog", "rsp", "console", "build.rsp"); Assert.True(File.Exists(rsp)); Assert.Contains("Program.cs", File.ReadAllLines(rsp)); } @@ -352,8 +352,8 @@ public void ResponseMultiTarget() { var exitCode = RunCompLog($"rsp {Fixture.ClassLibMultiProjectPath}"); Assert.Equal(Constants.ExitSuccess, exitCode); - Assert.True(File.Exists(Path.Combine(RootDirectory, @".complog", "classlibmulti-net6.0", "build.rsp"))); - Assert.True(File.Exists(Path.Combine(RootDirectory, @".complog", "classlibmulti-net7.0", "build.rsp"))); + Assert.True(File.Exists(Path.Combine(RootDirectory, @".complog", "rsp", "classlibmulti-net6.0", "build.rsp"))); + Assert.True(File.Exists(Path.Combine(RootDirectory, @".complog", "rsp", "classlibmulti-net7.0", "build.rsp"))); } [Fact] @@ -417,7 +417,7 @@ public void ExportCompilerLog(string arg, BasicAnalyzerKind? expectedKind) Assert.Equal(Constants.ExitSuccess, RunCompLog($"export -o {exportDir.DirectoryPath} {arg} {logPath} ", RootDirectory)); // Now run the generated build.cmd and see if it succeeds; - var exportPath = Path.Combine(exportDir.DirectoryPath, "console", "export"); + var exportPath = Path.Combine(exportDir.DirectoryPath, "console"); var buildResult = RunBuildCmd(exportPath); Assert.True(buildResult.Succeeded); @@ -493,9 +493,9 @@ public void ReplayConsoleWithEmit(string arg) using var emitDir = new TempDir(); Assert.Equal(Constants.ExitSuccess, RunCompLog($"replay {arg} -o {emitDir.DirectoryPath} {Fixture.SolutionBinaryLogPath}")); - AssertOutput(@"console\emit\console.dll"); - AssertOutput(@"console\emit\console.pdb"); - AssertOutput(@"console\emit\ref\console.dll"); + AssertOutput(@"console\console.dll"); + AssertOutput(@"console\console.pdb"); + AssertOutput(@"console\ref\console.dll"); void AssertOutput(string relativePath) { @@ -603,6 +603,61 @@ public void ReplayWithProject() Assert.Equal(Constants.ExitSuccess, RunCompLog($"replay {Fixture.ConsoleProjectPath}")); } + [Theory] + [CombinatorialData] + public void GeneratedBoth(BasicAnalyzerKind basicAnalyzerKind) + { + RunWithBoth(logPath => + { + AssertCompilerCallReader(void (ICompilerCallReader reader) => AssertCorrectReader(reader, logPath)); + var dir = Root.NewDirectory("generated"); + var (exitCode, output) = RunCompLogEx($"generated {logPath} -p console.csproj -a {basicAnalyzerKind} -o {dir}"); + Assert.Equal(Constants.ExitSuccess, exitCode); + Assert.Single(Directory.EnumerateFiles(dir, "RegexGenerator.g.cs", SearchOption.AllDirectories)); + }); + } + + [Fact] + public void GeneratedBadFilter() + { + RunWithBoth(logPath => + { + AssertCompilerCallReader(void (ICompilerCallReader reader) => AssertCorrectReader(reader, logPath)); + var (exitCode, _) = RunCompLogEx($"generated {logPath} -p console-does-not-exist.csproj"); + Assert.Equal(Constants.ExitFailure, exitCode); + }); + } + + [Fact] + public void GeneratePdbMissing() + { + var dir = Root.NewDirectory(); + RunDotNet($"new console --name example --output .", dir); + RunDotNet("build -bl -nr:false", dir); + + // Delete the PDB + Directory.EnumerateFiles(dir, "*.pdb", SearchOption.AllDirectories).ForEach(File.Delete); + + var (exitCode, output) = RunCompLogEx($"generated {dir} -a None"); + Assert.Equal(Constants.ExitSuccess, exitCode); + Assert.Contains("BCLA0001", output); + } + + [Fact] + public void GeneratedHelp() + { + var (exitCode, output) = RunCompLogEx($"generated -h"); + Assert.Equal(Constants.ExitSuccess, exitCode); + Assert.StartsWith("complog generated [OPTIONS]", output); + } + + [Fact] + public void GeneratedBadArg() + { + var (exitCode, _) = RunCompLogEx($"generated -o"); + Assert.Equal(Constants.ExitFailure, exitCode); + } + [Fact] public void PrintAll() { diff --git a/src/Basic.CompilerLog.UnitTests/SolutionFixture.cs b/src/Basic.CompilerLog.UnitTests/SolutionFixture.cs index cabcaf9..fc84e9a 100644 --- a/src/Basic.CompilerLog.UnitTests/SolutionFixture.cs +++ b/src/Basic.CompilerLog.UnitTests/SolutionFixture.cs @@ -67,6 +67,19 @@ public SolutionFixture(IMessageSink messageSink) ConsoleProjectPath = WithProject("console", string (string dir) => { RunDotnetCommand("new console --name console -o .", dir); + var program = """ + using System; + using System.Text.RegularExpressions; + // This is an amazing resource + var r = Util.GetRegex(); + Console.WriteLine(r); + + partial class Util { + [GeneratedRegex("abc|def", RegexOptions.IgnoreCase, "en-US")] + internal static partial Regex GetRegex(); + } + """; + File.WriteAllText(Path.Combine(dir, "Program.cs"), program, TestBase.DefaultEncoding); return Path.Combine(dir, "console.csproj"); }); diff --git a/src/Basic.CompilerLog.Util/BinaryLogReader.cs b/src/Basic.CompilerLog.Util/BinaryLogReader.cs index b40a1ce..4138218 100644 --- a/src/Basic.CompilerLog.Util/BinaryLogReader.cs +++ b/src/Basic.CompilerLog.Util/BinaryLogReader.cs @@ -305,6 +305,20 @@ public List ReadAllReferenceData(CompilerCall compilerCall) return ReadAllReferenceDataCore(args.MetadataReferences.Select(x => x.Reference), args.MetadataReferences.Length); } + /// + /// Attempt to add all the generated files from generators. When successful the generators + /// don't need to be run when re-hydrating the compilation. + /// + /// + /// This method will throw if the compilation does not have a PDB compatible with generated files + /// available to read + /// + public List<(string FilePath, MemoryStream Stream)> ReadAllGeneratedFiles(CompilerCall compilerCall) + { + var args = ReadCommandLineArguments(compilerCall); + return RoslynUtil.ReadGeneratedFiles(compilerCall, args); + } + private List ReadAllReferenceDataCore(IEnumerable filePaths, int count) { var list = new List(capacity: count); diff --git a/src/Basic.CompilerLog.Util/CompilationData.cs b/src/Basic.CompilerLog.Util/CompilationData.cs index a193b8a..d1cce28 100644 --- a/src/Basic.CompilerLog.Util/CompilationData.cs +++ b/src/Basic.CompilerLog.Util/CompilationData.cs @@ -119,7 +119,9 @@ public ImmutableArray GetGenerators() public Compilation GetCompilationAfterGenerators(CancellationToken cancellationToken = default) => GetCompilationAfterGenerators(out _, cancellationToken); - public Compilation GetCompilationAfterGenerators(out ImmutableArray diagnostics, CancellationToken cancellationToken = default) + public Compilation GetCompilationAfterGenerators( + out ImmutableArray diagnostics, + CancellationToken cancellationToken = default) { if (_afterGenerators is { } tuple) { @@ -141,6 +143,28 @@ public Compilation GetCompilationAfterGenerators(out ImmutableArray return tuple.Item1; } + public List GetGeneratedSyntaxTrees(CancellationToken cancellationToken = default) => + GetGeneratedSyntaxTrees(out _, cancellationToken); + + public List GetGeneratedSyntaxTrees( + out ImmutableArray diagnostics, + CancellationToken cancellationToken = default) + { + var afterCompilation = GetCompilationAfterGenerators(out diagnostics, cancellationToken); + + // This is a bit of a hack to get the number of syntax trees before running the generators. It feels + // a bit disjoint that we have to think of the None case differently here. Possible it may be simpler + // to have the None host go back to faking a ISourceGenerator in memory that just adds the files + // directly. + var originalCount = Compilation.SyntaxTrees.Count(); + if (BasicAnalyzerHost is BasicAnalyzerHostNone none) + { + var generatedCount = none.GeneratedSourceTexts.Length; + originalCount -= generatedCount; + } + return afterCompilation.SyntaxTrees.Skip(originalCount).ToList(); + } + private void EnsureAnalyzersLoaded() { if (!_analyzers.IsDefault) diff --git a/src/Basic.CompilerLog/Constants.cs b/src/Basic.CompilerLog/Constants.cs index 15b1bca..c038837 100644 --- a/src/Basic.CompilerLog/Constants.cs +++ b/src/Basic.CompilerLog/Constants.cs @@ -14,5 +14,4 @@ internal static class Constants internal static TextWriter Out { get; set; } = Console.Out; internal static Action OnCompilerCallReader = _ => { }; - } diff --git a/src/Basic.CompilerLog/Program.cs b/src/Basic.CompilerLog/Program.cs index 8e2aaba..633819d 100644 --- a/src/Basic.CompilerLog/Program.cs +++ b/src/Basic.CompilerLog/Program.cs @@ -24,6 +24,7 @@ "ref" => RunReferences(rest), "rsp" => RunResponseFile(rest), "analyzers" => RunAnalyzers(rest), + "generated" => RunGenerated(rest), "print" => RunPrint(rest), "help" => RunHelp(rest), @@ -220,14 +221,14 @@ int RunReferences(IEnumerable args) using var reader = GetCompilerCallReader(extra, BasicAnalyzerKind.None); var compilerCalls = reader.ReadAllCompilerCalls(options.FilterCompilerCalls); - baseOutputPath = GetBaseOutputPath(baseOutputPath); + baseOutputPath = GetBaseOutputPath(baseOutputPath, "refs"); WriteLine($"Copying references to {baseOutputPath}"); Directory.CreateDirectory(baseOutputPath); for (int i = 0; i < compilerCalls.Count; i++) { var compilerCall = compilerCalls[i]; - var refDirPath = GetOutputPath(baseOutputPath, compilerCalls, i, "refs"); + var refDirPath = Path.Combine(GetOutputPath(baseOutputPath, compilerCalls, i), "refs"); Directory.CreateDirectory(refDirPath); foreach (var data in reader.ReadAllReferenceData(compilerCall)) { @@ -235,7 +236,7 @@ int RunReferences(IEnumerable args) File.WriteAllBytes(filePath, data.ImageBytes); } - var analyzerDirPath = GetOutputPath(baseOutputPath, compilerCalls, i, "analyzers"); + var analyzerDirPath = Path.Combine(GetOutputPath(baseOutputPath, compilerCalls, i), "analyzers"); var groupMap = new Dictionary(PathUtil.Comparer); foreach (var data in reader.ReadAllAnalyzerData(compilerCall)) { @@ -305,7 +306,7 @@ int RunExport(IEnumerable args) var compilerCalls = reader.ReadAllCompilerCalls(options.FilterCompilerCalls); var exportUtil = new ExportUtil(reader, includeAnalyzers: options.IncludeAnalyzers); - baseOutputPath = GetBaseOutputPath(baseOutputPath); + baseOutputPath = GetBaseOutputPath(baseOutputPath, "export"); WriteLine($"Exporting to {baseOutputPath}"); Directory.CreateDirectory(baseOutputPath); @@ -313,7 +314,7 @@ int RunExport(IEnumerable args) for (int i = 0; i < compilerCalls.Count; i++) { var compilerCall = compilerCalls[i]; - var exportDir = GetOutputPath(baseOutputPath, compilerCalls, i, "export"); + var exportDir = GetOutputPath(baseOutputPath, compilerCalls, i); exportUtil.Export(compilerCall, exportDir, sdkDirs); } @@ -353,7 +354,7 @@ int RunResponseFile(IEnumerable args) } using var reader = GetCompilerCallReader(extra, BasicAnalyzerHost.DefaultKind); - baseOutputPath = GetBaseOutputPath(baseOutputPath); + baseOutputPath = GetBaseOutputPath(baseOutputPath, "rsp"); WriteLine($"Generating response files in {baseOutputPath}"); Directory.CreateDirectory(baseOutputPath); @@ -432,7 +433,7 @@ int RunReplay(IEnumerable args) IEmitResult emitResult; if (baseOutputPath is not null) { - var path = GetOutputPath(baseOutputPath, compilerCalls, i, "emit"); + var path = GetOutputPath(baseOutputPath, compilerCalls, i); Directory.CreateDirectory(path); emitResult = compilationData.EmitToDisk(path); } @@ -467,6 +468,80 @@ void PrintUsage() } } +int RunGenerated(IEnumerable args) +{ + string? baseOutputPath = null; + var options = new FilterOptionSet(analyzers: true) + { + { "o|out=", "path to emit to ", void (string b) => baseOutputPath = b }, + }; + + try + { + var extra = options.Parse(args); + if (options.Help) + { + PrintUsage(); + return ExitSuccess; + } + + baseOutputPath = GetBaseOutputPath(baseOutputPath, "generated"); + WriteLine($"Outputting to {baseOutputPath}"); + + using var reader = GetCompilerCallReader(extra, options.BasicAnalyzerKind, checkVersion: true); + var compilerCalls = reader.ReadAllCompilerCalls(options.FilterCompilerCalls); + if (compilerCalls.Count == 0) + { + WriteLine("No compilations found"); + return ExitFailure; + } + + for (int i = 0; i < compilerCalls.Count; i++) + { + var compilerCall = compilerCalls[i]; + var compilationData = reader.ReadCompilationData(compilerCall); + + Write($"{compilerCall.GetDiagnosticName()} ... "); + var generatedTrees = compilationData.GetGeneratedSyntaxTrees(out var diagnostics); + WriteLine($"{generatedTrees.Count} files"); + if (diagnostics.Length > 0) + { + WriteLine("\tDiagnostics"); + foreach (var diagnostic in diagnostics) + { + WriteLine(diagnostic.ToString()); + } + } + + foreach (var generatedTree in generatedTrees) + { + WriteLine($"\t{Path.GetFileName(generatedTree.FilePath)}"); + var fileRelativePath = generatedTree.FilePath.StartsWith(compilerCall.ProjectDirectory, StringComparison.OrdinalIgnoreCase) + ? generatedTree.FilePath.Substring(compilerCall.ProjectDirectory.Length + 1) + : Path.GetFileName(generatedTree.FilePath); + var outputPath = GetOutputPath(baseOutputPath, compilerCalls, i); + var filePath = Path.Combine(outputPath, fileRelativePath); + Directory.CreateDirectory(Path.GetDirectoryName(filePath)!); + File.WriteAllText(filePath, generatedTree.ToString()); + } + } + + return ExitSuccess; + } + catch (OptionException e) + { + WriteLine(e.Message); + PrintUsage(); + return ExitFailure; + } + + void PrintUsage() + { + WriteLine("complog generated [OPTIONS] msbuild.complog"); + options.WriteOptionDescriptions(Out); + } +} + int RunBadCommand(string command) { WriteLine(@$"""{command}"" is not a valid command"); @@ -494,8 +569,8 @@ replay Replay compilations from the log export Export compilation contents, rsp and build files to disk rsp Generate compiler response file projects on this machine ref Copy all references and analyzers to a single directory - diagnostics Print diagnostics for a compilation analyzers Print analyzers / generators used by a compilation + generated Get generated files for the compilation print Print summary of entries in the log help Print help """); @@ -660,11 +735,15 @@ static string GetLogFilePathAfterBuild(string baseDirectory, string buildFileNam return null; } -string GetBaseOutputPath(string? baseOutputPath) +string GetBaseOutputPath(string? baseOutputPath, string? directoryName = null) { if (string.IsNullOrEmpty(baseOutputPath)) { baseOutputPath = ".complog"; + if (directoryName is not null) + { + baseOutputPath = Path.Combine(baseOutputPath, directoryName); + } } if (!Path.IsPathRooted(baseOutputPath)) @@ -675,12 +754,10 @@ string GetBaseOutputPath(string? baseOutputPath) return baseOutputPath; } -string GetOutputPath(string baseOutputPath, List compilerCalls, int index, string? directoryName = null) +string GetOutputPath(string baseOutputPath, List compilerCalls, int index) { var projectName = GetProjectUniqueName(compilerCalls, index); - return string.IsNullOrEmpty(directoryName) - ? Path.Combine(baseOutputPath, projectName) - : Path.Combine(baseOutputPath, projectName, directoryName); + return Path.Combine(baseOutputPath, projectName); } string GetProjectUniqueName(List compilerCalls, int index)