[clang][DependencyScanning] Move driver-command logic for by-name scanning into DependencyScanningTool

This is the second patch in a series that removes the dependency of
clangDependencyScanning on clangDriver, splitting the work
from #169964 into smaller changes (see comment linked below).

This patch updates the by-name scanning interface in
DependencyScanningWorker to accept only -cc1 command lines and moves
the logic for handling driver-style command lines into
DependencyScanningTool in clangTooling.

Support for -cc1 command lines in by-name scanning is introduced in
this patch.

The next patch will update the remaining parts of
DependencyScanningWorker to operate only on -cc1 command lines,
allowing its dependency on clangDriver to be removed.

#169964 (review)

Author: naveen-seth

clang/include/clang/DependencyScanning/DependencyScannerImpl.h

@@ -17,6 +17,7 @@
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Serialization/ObjectFilePCHContainerReader.h"
+#include "llvm/Support/VirtualFileSystem.h"
 
 namespace clang {
 class DiagnosticConsumer;
@@ -63,15 +64,15 @@ class DependencyScanningAction {
 std::unique_ptr<DiagnosticOptions>
 createDiagOptions(ArrayRef<std::string> CommandLine);
 
-struct DignosticsEngineWithDiagOpts {
+struct DiagnosticsEngineWithDiagOpts {
   // We need to bound the lifetime of the DiagOpts used to create the
   // DiganosticsEngine with the DiagnosticsEngine itself.
   std::unique_ptr<DiagnosticOptions> DiagOpts;
   IntrusiveRefCntPtr<DiagnosticsEngine> DiagEngine;
 
-  DignosticsEngineWithDiagOpts(ArrayRef<std::string> CommandLine,
-                               IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
-                               DiagnosticConsumer &DC);
+  DiagnosticsEngineWithDiagOpts(ArrayRef<std::string> CommandLine,
+                                IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
+                                DiagnosticConsumer &DC);
 };
 
 struct TextDiagnosticsPrinterWithOutput {
@@ -143,22 +144,11 @@ class CompilerInstanceWithContext {
   llvm::StringRef CWD;
   std::vector<std::string> CommandLine;
 
-  // Context - file systems
-  llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS;
-
   // Context - Diagnostics engine.
-  std::unique_ptr<TextDiagnosticsPrinterWithOutput> DiagPrinterWithOS;
-  // DiagConsumer may points to DiagPrinterWithOS->DiagPrinter, or a custom
-  // DiagnosticConsumer passed in from initialize.
   DiagnosticConsumer *DiagConsumer = nullptr;
-  std::unique_ptr<DignosticsEngineWithDiagOpts> DiagEngineWithCmdAndOpts;
+  std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithCmdAndOpts;
 
   // Context - compiler invocation
-  // Compilation's command's arguments may be owned by Alloc when expanded from
-  // response files, so we need to keep Alloc alive in the context.
-  llvm::BumpPtrAllocator Alloc;
-  std::unique_ptr<clang::driver::Driver> Driver;
-  std::unique_ptr<clang::driver::Compilation> Compilation;
   std::unique_ptr<CompilerInvocation> OriginalInvocation;
 
   // Context - output options
@@ -180,15 +170,13 @@ class CompilerInstanceWithContext {
       : Worker(Worker), CWD(CWD), CommandLine(CMD) {};
 
   // The three methods below returns false when they fail, with the detail
-  // accumulated in DiagConsumer.
-  bool initialize(DiagnosticConsumer *DC);
+  // accumulated in \c DiagEngineWithDiagOpts's diagnostic consumer.
+  bool initialize(
+      std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
+      IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS);
   bool computeDependencies(StringRef ModuleName, DependencyConsumer &Consumer,
                            DependencyActionController &Controller);
   bool finalize();
-
-  // The method below turns the return status from the above methods
-  // into an llvm::Error using a default DiagnosticConsumer.
-  llvm::Error handleReturnStatus(bool Success);
 };
 } // namespace dependencies
 } // namespace clang

clang/include/clang/DependencyScanning/DependencyScanningWorker.h

@@ -12,12 +12,14 @@
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/LLVM.h"
+#include "clang/DependencyScanning/DependencyScannerImpl.h"
 #include "clang/DependencyScanning/DependencyScanningService.h"
 #include "clang/DependencyScanning/ModuleDepCollector.h"
 #include "clang/Frontend/PCHContainerOperations.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBufferRef.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include <optional>
 #include <string>
 
@@ -119,42 +121,44 @@ class DependencyScanningWorker {
   /// dependency scanning. They together enable the dependency scanning worker
   /// to more effectively perform scanning for a sequence of modules
   /// by name when the CWD and CommandLine do not change across the queries.
+  /// The initialization function asks the client for a DiagnosticsConsumer
+  /// that it direct the diagnostics to.
 
   /// @brief Initializing the context and the compiler instance.
   /// @param CWD The current working directory used during the scan.
   /// @param CommandLine The commandline used for the scan.
-  /// @return Error if the initializaiton fails.
-  llvm::Error initializeCompilerInstanceWithContextOrError(
-      StringRef CWD, ArrayRef<std::string> CommandLine);
+  /// @return False if the initializaiton fails.
+  bool initializeCompilerInstanceWithContext(StringRef CWD,
+                                             ArrayRef<std::string> CommandLine,
+                                             DiagnosticConsumer &DC);
+
+  /// @brief Initializing the context and the compiler instance.
+  /// @param CWD The current working directory used during the scan.
+  /// @param CommandLine The commandline used for the scan.
+  /// @param DiagEngineWithCmdAndOpts Preconfigured diagnostics engine and
+  /// options associated with the cc1 command line.
+  /// @param FS The overlay file system to use for this compiler instance.
+  /// @return False if the initializaiton fails.
+  bool initializeCompilerInstanceWithContext(
+      StringRef CWD, ArrayRef<std::string> CommandLine,
+      std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithCmdAndOpts,
+      IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS);
 
   /// @brief Performaces dependency scanning for the module whose name is
   ///        specified.
   /// @param ModuleName  The name of the module whose dependency will be
   ///                    scanned.
   /// @param Consumer The dependency consumer that stores the results.
   /// @param Controller The controller for the dependency scanning action.
-  /// @return Error if the scanner incurs errors.
-  llvm::Error computeDependenciesByNameWithContextOrError(
-      StringRef ModuleName, DependencyConsumer &Consumer,
-      DependencyActionController &Controller);
-
-  /// @brief Finalizes the diagnostics engine and deletes the compiler instance.
-  /// @return Error if errors occur during finalization.
-  llvm::Error finalizeCompilerInstanceWithContextOrError();
-
-  /// The three methods below provides the same functionality as the
-  /// three methods above. Instead of returning `llvm::Error`s, these
-  /// three methods return a flag to indicate if the call is successful.
-  /// The initialization function asks the client for a DiagnosticsConsumer
-  /// that it direct the diagnostics to.
-  bool initializeCompilerInstanceWithContext(StringRef CWD,
-                                             ArrayRef<std::string> CommandLine,
-                                             DiagnosticConsumer *DC = nullptr);
+  /// @return False if the scanner incurs errors.
   bool
   computeDependenciesByNameWithContext(StringRef ModuleName,
                                        DependencyConsumer &Consumer,
                                        DependencyActionController &Controller);
-  bool finalizeCompilerInstance();
+
+  /// @brief Finalizes the diagnostics engine and deletes the compiler instance.
+  /// @return False if errors occur during finalization.
+  bool finalizeCompilerInstanceWithContext();
 
   llvm::vfs::FileSystem &getVFS() const { return *DepFS; }
 

clang/include/clang/Tooling/DependencyScanningTool.h

@@ -9,6 +9,7 @@
 #ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H
 #define LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H
 
+#include "clang/DependencyScanning/DependencyScannerImpl.h"
 #include "clang/DependencyScanning/DependencyScanningService.h"
 #include "clang/DependencyScanning/DependencyScanningUtils.h"
 #include "clang/DependencyScanning/DependencyScanningWorker.h"
@@ -119,9 +120,8 @@ class DependencyScanningTool {
   /// @param CWD The current working directory used during the scan.
   /// @param CommandLine The commandline used for the scan.
   /// @return Error if the initializaiton fails.
-  llvm::Error
-  initializeCompilerInstanceWithContext(StringRef CWD,
-                                        ArrayRef<std::string> CommandLine);
+  llvm::Error initializeCompilerInstanceWithContextOrError(
+      StringRef CWD, ArrayRef<std::string> CommandLine);
 
   /// @brief Computes the dependeny for the module named ModuleName.
   /// @param ModuleName The name of the module for which this method computes
@@ -137,8 +137,8 @@ class DependencyScanningTool {
   ///                           arguments for dependencies.
   /// @return An instance of \c TranslationUnitDeps if the scan is successful.
   ///         Otherwise it returns an error.
-  llvm::Expected<dependencies::TranslationUnitDeps>
-  computeDependenciesByNameWithContext(
+  llvm::Expected<clang::dependencies::TranslationUnitDeps>
+  computeDependenciesByNameWithContextOrError(
       StringRef ModuleName,
       const llvm::DenseSet<dependencies::ModuleID> &AlreadySeen,
       dependencies::LookupModuleOutputCallback LookupModuleOutput);
@@ -147,7 +147,7 @@ class DependencyScanningTool {
   ///        diagnostics and deletes the compiler instance. Call this method
   ///        once all names for a same commandline are scanned.
   /// @return Error if an error occured during finalization.
-  llvm::Error finalizeCompilerInstanceWithContext();
+  llvm::Error finalizeCompilerInstanceWithContextOrError();
 
   llvm::vfs::FileSystem &getWorkerVFS() const { return Worker.getVFS(); }
 

clang/lib/DependencyScanning/DependencyScannerImpl.cpp

@@ -13,6 +13,7 @@
 #include "clang/Driver/Driver.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "llvm/ADT/ScopeExit.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/TargetParser/Host.h"
 
 using namespace clang;
@@ -367,7 +368,7 @@ dependencies::createDiagOptions(ArrayRef<std::string> CommandLine) {
   return DiagOpts;
 }
 
-DignosticsEngineWithDiagOpts::DignosticsEngineWithDiagOpts(
+DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts(
     ArrayRef<std::string> CommandLine,
     IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, DiagnosticConsumer &DC) {
   std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
@@ -713,38 +714,31 @@ bool DependencyScanningAction::runInvocation(
   return Result;
 }
 
-bool CompilerInstanceWithContext::initialize(DiagnosticConsumer *DC) {
-  if (DC) {
-    DiagConsumer = DC;
-  } else {
-    DiagPrinterWithOS =
-        std::make_unique<TextDiagnosticsPrinterWithOutput>(CommandLine);
-    DiagConsumer = &DiagPrinterWithOS->DiagPrinter;
+bool CompilerInstanceWithContext::initialize(
+    std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
+    IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) {
+  assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!");
+  DiagEngineWithCmdAndOpts = std::move(DiagEngineWithDiagOpts);
+  DiagConsumer = DiagEngineWithCmdAndOpts->DiagEngine->getClient();
+
+  IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = Worker.DepFS;
+  if (OverlayFS) {
+#ifndef NDEBUG
+    bool SawDepFS = false;
+    OverlayFS->visit([&](llvm::vfs::FileSystem &VFS) {
+      SawDepFS |= &VFS == Worker.DepFS.get();
+    });
+    assert(SawDepFS && "OverlayFS not based on DepFS");
+#endif
+    FS = std::move(OverlayFS);
   }
 
-  std::tie(OverlayFS, CommandLine) = initVFSForByNameScanning(
-      Worker.DepFS, CommandLine, CWD, "ScanningByName");
-
-  DiagEngineWithCmdAndOpts = std::make_unique<DignosticsEngineWithDiagOpts>(
-      CommandLine, OverlayFS, *DiagConsumer);
-
-  std::tie(Driver, Compilation) = buildCompilation(
-      CommandLine, *DiagEngineWithCmdAndOpts->DiagEngine, OverlayFS, Alloc);
-
-  if (!Compilation)
-    return false;
+  // Reset what might have been modified in the previous worker invocation.
+  FS->setCurrentWorkingDirectory(CWD);
 
-  assert(Compilation->getJobs().size() &&
-         "Must have a job list of non-zero size");
-  const driver::Command &Command = *(Compilation->getJobs().begin());
-  const auto &CommandArgs = Command.getArguments();
-  assert(!CommandArgs.empty() && "Cannot have a command with 0 args");
-  assert(StringRef(CommandArgs[0]) == "-cc1" && "Requires a cc1 job.");
-  OriginalInvocation = std::make_unique<CompilerInvocation>();
-
-  if (!CompilerInvocation::CreateFromArgs(*OriginalInvocation, CommandArgs,
-                                          *DiagEngineWithCmdAndOpts->DiagEngine,
-                                          Command.getExecutable())) {
+  OriginalInvocation = createCompilerInvocation(
+      CommandLine, *DiagEngineWithCmdAndOpts->DiagEngine);
+  if (!OriginalInvocation) {
     DiagEngineWithCmdAndOpts->DiagEngine->Report(
         diag::err_fe_expected_compiler_job)
         << llvm::join(CommandLine, " ");
@@ -763,7 +757,7 @@ bool CompilerInstanceWithContext::initialize(DiagnosticConsumer *DC) {
   auto &CI = *CIPtr;
 
   if (!initializeScanCompilerInstance(
-          CI, OverlayFS, DiagEngineWithCmdAndOpts->DiagEngine->getClient(),
+          CI, FS, DiagEngineWithCmdAndOpts->DiagEngine->getClient(),
           Worker.Service, Worker.DepFS))
     return false;
 
@@ -876,11 +870,3 @@ bool CompilerInstanceWithContext::finalize() {
   DiagConsumer->finish();
   return true;
 }
-
-llvm::Error CompilerInstanceWithContext::handleReturnStatus(bool Success) {
-  assert(DiagPrinterWithOS && "Must use the default DiagnosticConsumer.");
-  return Success ? llvm::Error::success()
-                 : llvm::make_error<llvm::StringError>(
-                       DiagPrinterWithOS->DiagnosticsOS.str(),
-                       llvm::inconvertibleErrorCode());
-}

clang/lib/DependencyScanning/DependencyScanningWorker.cpp

@@ -7,10 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/DependencyScanning/DependencyScanningWorker.h"
+#include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticFrontend.h"
 #include "clang/DependencyScanning/DependencyScannerImpl.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Tool.h"
+#include "clang/Serialization/ObjectFilePCHContainerReader.h"
+#include "llvm/Support/VirtualFileSystem.h"
 
 using namespace clang;
 using namespace dependencies;
@@ -100,7 +103,7 @@ bool DependencyScanningWorker::scanDependencies(
     FS = std::move(OverlayFS);
   }
 
-  DignosticsEngineWithDiagOpts DiagEngineWithCmdAndOpts(CommandLine, FS, DC);
+  DiagnosticsEngineWithDiagOpts DiagEngineWithCmdAndOpts(CommandLine, FS, DC);
   DependencyScanningAction Action(Service, WorkingDirectory, Consumer,
                                   Controller, DepFS);
 
@@ -165,33 +168,29 @@ bool DependencyScanningWorker::computeDependencies(
                           DC);
 }
 
-llvm::Error
-DependencyScanningWorker::initializeCompilerInstanceWithContextOrError(
-    StringRef CWD, ArrayRef<std::string> CommandLine) {
-  bool Success = initializeCompilerInstanceWithContext(CWD, CommandLine);
-  return CIWithContext->handleReturnStatus(Success);
-}
-
-llvm::Error
-DependencyScanningWorker::computeDependenciesByNameWithContextOrError(
-    StringRef ModuleName, DependencyConsumer &Consumer,
-    DependencyActionController &Controller) {
-  bool Success =
-      computeDependenciesByNameWithContext(ModuleName, Consumer, Controller);
-  return CIWithContext->handleReturnStatus(Success);
-}
-
-llvm::Error
-DependencyScanningWorker::finalizeCompilerInstanceWithContextOrError() {
-  bool Success = finalizeCompilerInstance();
-  return CIWithContext->handleReturnStatus(Success);
+bool DependencyScanningWorker::initializeCompilerInstanceWithContext(
+    StringRef CWD, ArrayRef<std::string> CommandLine, DiagnosticConsumer &DC) {
+  auto OverlayFSAndArgs =
+      initVFSForByNameScanning(DepFS, CommandLine, CWD, "ScanningByName");
+  auto &OverlayFS = OverlayFSAndArgs.first;
+  const auto &ModifiedCommandLine = OverlayFSAndArgs.second;
+
+  auto DiagEngineWithCmdAndOpts =
+      std::make_unique<DiagnosticsEngineWithDiagOpts>(ModifiedCommandLine,
+                                                      OverlayFS, DC);
+
+  return initializeCompilerInstanceWithContext(
+      CWD, ModifiedCommandLine, std::move(DiagEngineWithCmdAndOpts), OverlayFS);
 }
 
 bool DependencyScanningWorker::initializeCompilerInstanceWithContext(
-    StringRef CWD, ArrayRef<std::string> CommandLine, DiagnosticConsumer *DC) {
+    StringRef CWD, ArrayRef<std::string> CommandLine,
+    std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
+    IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) {
   CIWithContext =
       std::make_unique<CompilerInstanceWithContext>(*this, CWD, CommandLine);
-  return CIWithContext->initialize(DC);
+  return CIWithContext->initialize(std::move(DiagEngineWithDiagOpts),
+                                   OverlayFS);
 }
 
 bool DependencyScanningWorker::computeDependenciesByNameWithContext(
@@ -201,6 +200,6 @@ bool DependencyScanningWorker::computeDependenciesByNameWithContext(
   return CIWithContext->computeDependencies(ModuleName, Consumer, Controller);
 }
 
-bool DependencyScanningWorker::finalizeCompilerInstance() {
+bool DependencyScanningWorker::finalizeCompilerInstanceWithContext() {
   return CIWithContext->finalize();
 }