C#: Add binlog support to buildless

csharp/autobuilder/Semmle.Autobuild.CSharp/BUILD.bazel

@@ -14,6 +14,7 @@ codeql_csharp_binary(
         "//csharp/autobuilder/Semmle.Autobuild.Shared",
         "//csharp/extractor/Semmle.Extraction.CSharp",
         "//csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching",
+        "//csharp/extractor/Semmle.Extraction.CSharp.Driver:bin/Semmle.Extraction.CSharp.Driver",
         "//csharp/extractor/Semmle.Extraction.CSharp.Standalone:bin/Semmle.Extraction.CSharp.Standalone",
         "//csharp/extractor/Semmle.Util",
         "@paket.main//microsoft.build",

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

@@ -15,6 +15,7 @@
         private const string extractorOptionPrefix = "CODEQL_EXTRACTOR_CSHARP_OPTION_";
 
         public bool Buildless { get; }
+        public string? Binlog { get; }
 
         public override Language Language => Language.CSharp;
 
@@ -29,7 +30,7 @@
                 actions.GetEnvironmentVariable(extractorOptionPrefix + "BUILDLESS").AsBool("buildless", false) ||
                 actions.GetEnvironmentVariable(buildModeEnvironmentVariable)?.ToLower() == "none";
 
-
+            Binlog = actions.GetEnvironmentVariable(extractorOptionPrefix + "BINLOG");
         }
     }
 

csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

@@ -6,6 +6,7 @@
     <ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj" />
+    <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Driver\Semmle.Extraction.CSharp.Driver.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
     <ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
   </ItemGroup>

csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

@@ -10,7 +10,9 @@ internal class StandaloneBuildRule : IBuildRule<CSharpAutobuildOptions>
     {
         public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
         {
-            return BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
+            return builder.Options.Binlog is string binlog
+                ? BuildScript.Create(_ => Semmle.Extraction.CSharp.Driver.Main(["--binlog", binlog]))
+                : BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
         }
     }
 }

csharp/codeql-extractor.yml

@@ -65,3 +65,9 @@ options:
             - progress+++
         type: string
         pattern: "^(off|errors|warnings|(info|progress)|(debug|progress\\+)|(trace|progress\\+\\+)|progress\\+\\+\\+)$"
+  binlog:
+    title: Binlog
+    description: >
+      [EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted.
+      This option only works when `--build-mode none` is also specified.
+    type: string

csharp/extractor/Semmle.Extraction.CSharp.Driver/BUILD.bazel

@@ -9,7 +9,7 @@ codeql_csharp_binary(
         "*.cs",
         "Properties/*.cs",
     ]),
-    visibility = ["//csharp:__pkg__"],
+    visibility = ["//csharp:__subpackages__"],
     deps = [
         "//csharp/extractor/Semmle.Extraction.CSharp",
     ],

csharp/extractor/Semmle.Extraction.CSharp/BUILD.bazel

@@ -20,6 +20,7 @@ codeql_csharp_library(
         "//csharp/extractor/Semmle.Extraction",
         "//csharp/extractor/Semmle.Extraction.CSharp.Util",
         "//csharp/extractor/Semmle.Util",
+        "@paket.main//basic.compilerlog.util",
         "@paket.main//microsoft.build",
         "@paket.main//microsoft.codeanalysis.csharp",
     ],

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -13,6 +13,8 @@
 using Microsoft.CodeAnalysis.Text;
 using Semmle.Util;
 using Semmle.Util.Logging;
+using Basic.CompilerLog;
+using Basic.CompilerLog.Util;
 
 namespace Semmle.Extraction.CSharp
 {
@@ -92,16 +94,63 @@ public static ILogger MakeLogger(Verbosity verbosity, bool includeConsole)
         /// <param name="args">Command line arguments as passed to csc.exe</param>
         /// <returns><see cref="ExitCode"/></returns>
         public static ExitCode Run(string[] args)
+        {
+            var options = Options.CreateWithEnvironment(args);
+            using var logger = MakeLogger(options.Verbosity, options.Console);
+            if (options.BinaryLogPath is string binlogPath)
+            {
+                logger.LogInfo($"Reading compiler calls from binary log {binlogPath}");
+                using var fileStream = new FileStream(binlogPath, FileMode.Open, FileAccess.Read, FileShare.Read);
+
+                // Filter out compiler calls that aren't interesting for examination
+                var predicate = bool (CompilerCall compilerCall) =>
+                {
+                    if (!compilerCall.IsCSharp)
+                    {
+                        return false;
+                    }
+
+                    return compilerCall.Kind switch
+                    {
+                        CompilerCallKind.XamlPreCompile => false,
+                        CompilerCallKind.Satellite => false,
+                        CompilerCallKind.WpfTemporaryCompile => false,
+                        _ => true
+                    };
+                };
+
+                var compilerCalls = BinaryLogUtil.ReadAllCompilerCalls(fileStream, predicate);
+                var allFailed = true;
+                foreach (var compilerCall in compilerCalls)
+                {
+                    var diagnosticName = compilerCall.GetDiagnosticName();
+                    Console.WriteLine($"Processing {diagnosticName}");
+                    var compilerCallOptions = Options.CreateWithEnvironment([]);
+                    compilerCallOptions.CompilerName = compilerCall.CompilerFilePath;
+                    compilerCallOptions.CompilerArguments.AddRange(compilerCall.GetArguments());
+                    logger.LogInfo($"Running extractor on arguments from binary log. Processing {diagnosticName}.");
+                    var ec = Run(compilerCallOptions, logger);
+                    if (ec != ExitCode.Failed)
+                    {
+                        allFailed &= false;
+                    }
+                }
+
+                return allFailed ? ExitCode.Failed : ExitCode.Ok;
+            }
+            else
+            {
+                return Run(options, logger);
+            }
+        }
+
+        public static ExitCode Run(Options options, ILogger logger)
         {
             var stopwatch = new Stopwatch();
             stopwatch.Start();
-
-            var options = Options.CreateWithEnvironment(args);
             var workingDirectory = Directory.GetCurrentDirectory();
             var compilerArgs = options.CompilerArguments.ToArray();
 
-            using var logger = MakeLogger(options.Verbosity, options.Console);
-
             var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
             var pathTransformer = new PathTransformer(canonicalPathCache);
 
@@ -393,10 +442,10 @@ private static ExitCode AnalyseTracing(
                         .Select(src => src.Path)
                         .ToList();
 
-                    if (compilerArguments.GeneratedFilesOutputDirectory is not null)
-                    {
-                        paths.AddRange(Directory.GetFiles(compilerArguments.GeneratedFilesOutputDirectory, "*.cs", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive }));
-                    }
+                    // if (compilerArguments.GeneratedFilesOutputDirectory is not null)
+                    // {
+                    //     paths.AddRange(Directory.GetFiles(compilerArguments.GeneratedFilesOutputDirectory, "*.cs", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive }));
+                    // }
 
                     return ReadSyntaxTrees(
                         paths.Select(canonicalPathCache.GetCanonicalPath),

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Options.cs

@@ -25,13 +25,15 @@ public sealed class Options : CommonOptions
         /// <summary>
         /// All other arguments passed to the compilation.
         /// </summary>
-        public IList<string> CompilerArguments { get; } = new List<string>();
+        public List<string> CompilerArguments { get; } = new List<string>();
 
         /// <summary>
         /// Holds if assembly information should be prefixed to TRAP labels.
         /// </summary>
         public bool AssemblySensitiveTrap { get; private set; } = false;
 
+        public string? BinaryLogPath { get; set; }
+
         public static Options CreateWithEnvironment(string[] arguments)
         {
             var options = new Options();
@@ -65,6 +67,9 @@ public override bool HandleOption(string key, string value)
                 case "load-sources-from-project":
                     ProjectsToLoad.Add(value);
                     return true;
+                case "binlog":
+                    BinaryLogPath = value;
+                    return true;
                 default:
                     return base.HandleOption(key, value);
             }

csharp/extractor/Semmle.Extraction.CSharp/paket.references

@@ -1,3 +1,3 @@
 Microsoft.Build
 Microsoft.CodeAnalysis.CSharp
-
+Basic.CompilerLog.Util

csharp/paket.dependencies

@@ -4,6 +4,7 @@ source https://api.nuget.org/v3/index.json
 # behave like nuget in choosing transitive dependency versions
 strategy: min
 
+nuget Basic.CompilerLog.Util
 nuget Mono.Posix.NETStandard
 nuget Newtonsoft.Json
 nuget xunit