[clang][DependencyScanning] Remove dependency on clangDriver from clangDependencyScanning

This follows PR llvm#169962 and removes the dependency on clangDriver from
clangDependencyScanning.

DependencyScanningWorker now only supports -cc1 command line inputs,
and all functionality related to driver-level commands has been moved
into clangTooling (DependencyScanningTool.cpp).

Because DependencyScanningWorker now only accepts -cc1 inputs, this
patch enables the use of -cc1 commands with the by-name scanning API
introduced in llvm#164345.

This is part of a broader effort to support driver-managed builds for
compilations using C++ named modules and/or Clang modules. It is
required for linking the dependency scanning tooling against the
driver without introducing cyclic dependencies, which would otherwise
cause build failures when dynamic linking is enabled.

The RFC for this change can be found here:
https://discourse.llvm.org/t/rfc-new-clangoptions-library-remove-dependency-on-clangdriver-from-clangfrontend-and-flangfrontend/88773?u=naveen-seth

Author: naveen-seth

clang/include/clang/DependencyScanning/DependencyScannerImpl.h

@@ -16,7 +16,6 @@
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
-#include "clang/Serialization/ObjectFilePCHContainerReader.h"
 
 namespace clang {
 class DiagnosticConsumer;
@@ -88,27 +87,10 @@ struct TextDiagnosticsPrinterWithOutput {
         DiagPrinter(DiagnosticsOS, *DiagOpts) {}
 };
 
-std::pair<std::unique_ptr<driver::Driver>, std::unique_ptr<driver::Compilation>>
-buildCompilation(ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags,
-                 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
-                 llvm::BumpPtrAllocator &Alloc);
-
 std::unique_ptr<CompilerInvocation>
 createCompilerInvocation(ArrayRef<std::string> CommandLine,
                          DiagnosticsEngine &Diags);
 
-std::pair<IntrusiveRefCntPtr<llvm::vfs::FileSystem>, std::vector<std::string>>
-initVFSForTUBufferScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
-                           ArrayRef<std::string> CommandLine,
-                           StringRef WorkingDirectory,
-                           llvm::MemoryBufferRef TUBuffer);
-
-std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>,
-          std::vector<std::string>>
-initVFSForByNameScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
-                         ArrayRef<std::string> CommandLine,
-                         StringRef WorkingDirectory, StringRef ModuleName);
-
 bool initializeScanCompilerInstance(
     CompilerInstance &ScanInstance,
     IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
@@ -143,22 +125,11 @@ class CompilerInstanceWithContext {
   llvm::StringRef CWD;
   std::vector<std::string> CommandLine;
 
-  // Context - file systems
-  llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS;
-
   // Context - Diagnostics engine.
-  std::unique_ptr<TextDiagnosticsPrinterWithOutput> DiagPrinterWithOS;
-  // DiagConsumer may points to DiagPrinterWithOS->DiagPrinter, or a custom
-  // DiagnosticConsumer passed in from initialize.
   DiagnosticConsumer *DiagConsumer = nullptr;
   std::unique_ptr<DignosticsEngineWithDiagOpts> DiagEngineWithCmdAndOpts;
 
   // Context - compiler invocation
-  // Compilation's command's arguments may be owned by Alloc when expanded from
-  // response files, so we need to keep Alloc alive in the context.
-  llvm::BumpPtrAllocator Alloc;
-  std::unique_ptr<clang::driver::Driver> Driver;
-  std::unique_ptr<clang::driver::Compilation> Compilation;
   std::unique_ptr<CompilerInvocation> OriginalInvocation;
 
   // Context - output options
@@ -180,15 +151,13 @@ class CompilerInstanceWithContext {
       : Worker(Worker), CWD(CWD), CommandLine(CMD) {};
 
   // The three methods below returns false when they fail, with the detail
-  // accumulated in DiagConsumer.
-  bool initialize(DiagnosticConsumer *DC);
+  // accumulated in \c DiagEngineWithDiagOpts's diagnostic consumer.
+  bool initialize(
+      std::unique_ptr<DignosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
+      IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
   bool computeDependencies(StringRef ModuleName, DependencyConsumer &Consumer,
                            DependencyActionController &Controller);
   bool finalize();
-
-  // The method below turns the return status from the above methods
-  // into an llvm::Error using a default DiagnosticConsumer.
-  llvm::Error handleReturnStatus(bool Success);
 };
 } // namespace dependencies
 } // namespace clang

clang/include/clang/DependencyScanning/DependencyScanningWorker.h

@@ -9,9 +9,11 @@
 #ifndef LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H
 #define LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H
 
+#include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/LLVM.h"
+#include "clang/DependencyScanning/DependencyScannerImpl.h"
 #include "clang/DependencyScanning/DependencyScanningService.h"
 #include "clang/DependencyScanning/ModuleDepCollector.h"
 #include "clang/Frontend/PCHContainerOperations.h"
@@ -91,70 +93,72 @@ class DependencyScanningWorker {
 
   ~DependencyScanningWorker();
 
-  /// Run the dependency scanning tool for a given clang driver command-line,
-  /// and report the discovered dependencies to the provided consumer. If
-  /// TUBuffer is not nullopt, it is used as TU input for the dependency
-  /// scanning. Otherwise, the input should be included as part of the
-  /// command-line.
+  /// Run the dependency scanning tool for a given clang -cc1 command-line,
+  /// and report the discovered dependencies to the provided consumer.
   ///
-  /// \returns false if clang errors occurred (with diagnostics reported to
+  /// @return false if clang errors occurred (with diagnostics reported to
   /// \c DiagConsumer), true otherwise.
   bool computeDependencies(
       StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
       DependencyConsumer &DepConsumer, DependencyActionController &Controller,
       DiagnosticConsumer &DiagConsumer,
-      std::optional<llvm::MemoryBufferRef> TUBuffer = std::nullopt);
+      llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> ScanFS = nullptr);
 
-  /// Run the dependency scanning tool for a given clang driver command-line
-  /// for a specific translation unit via file system or memory buffer.
+  /// Run the dependency scanning tool for all given clang -cc1 command-lines,
+  /// and report the discovered dependencies to the provided consumer.
   ///
-  /// \returns A \c StringError with the diagnostic output if clang errors
-  /// occurred, success otherwise.
-  llvm::Error computeDependencies(
-      StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
+  /// \returns false if clang errors occurred (with diagnostics reported to
+  /// \c Diags), true otherwise.
+  bool computeDependencies(
+      StringRef WorkingDirectory,
+      ArrayRef<std::vector<std::string>> CommandLines,
       DependencyConsumer &Consumer, DependencyActionController &Controller,
-      std::optional<llvm::MemoryBufferRef> TUBuffer = std::nullopt);
+      DiagnosticsEngine &Diags,
+      llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> ScanFS = nullptr);
 
   /// The three method below implements a new interface for by name
   /// dependency scanning. They together enable the dependency scanning worker
   /// to more effectively perform scanning for a sequence of modules
   /// by name when the CWD and CommandLine do not change across the queries.
+  /// The initialization function asks the client for a DiagnosticsConsumer
+  /// that it direct the diagnostics to.
 
   /// @brief Initializing the context and the compiler instance.
   /// @param CWD The current working directory used during the scan.
   /// @param CommandLine The commandline used for the scan.
-  /// @return Error if the initializaiton fails.
-  llvm::Error initializeCompilerInstanceWithContextOrError(
-      StringRef CWD, const std::vector<std::string> &CommandLine);
+  /// @return False if the initializaiton fails.
+  bool initializeCompilerInstanceWithContext(StringRef CWD,
+                                             ArrayRef<std::string> CommandLine,
+                                             DiagnosticConsumer &DC);
+
+  /// @brief Initializing the context and the compiler instance.
+  /// @param CWD The current working directory used during the scan.
+  /// @param CommandLine The commandline used for the scan.
+  /// @param DiagEngineWithCmdAndOpts Preconfigured diagnostics engine and
+  /// options associated with the cc1 command line.
+  /// @param FS The file system (typically an overlay) to use for this compiler
+  /// instance.
+  /// @return False if the initializaiton fails.
+  bool initializeCompilerInstanceWithContext(
+      StringRef CWD, ArrayRef<std::string> CommandLine,
+      std::unique_ptr<DignosticsEngineWithDiagOpts> DiagEngineWithCmdAndOpts,
+      IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
 
   /// @brief Performaces dependency scanning for the module whose name is
   ///        specified.
   /// @param ModuleName  The name of the module whose dependency will be
   ///                    scanned.
   /// @param Consumer The dependency consumer that stores the results.
   /// @param Controller The controller for the dependency scanning action.
-  /// @return Error if the scanner incurs errors.
-  llvm::Error computeDependenciesByNameWithContextOrError(
-      StringRef ModuleName, DependencyConsumer &Consumer,
-      DependencyActionController &Controller);
-
-  /// @brief Finalizes the diagnostics engine and deletes the compiler instance.
-  /// @return Error if errors occur during finalization.
-  llvm::Error finalizeCompilerInstanceWithContextOrError();
-
-  /// The three methods below provides the same functionality as the
-  /// three methods above. Instead of returning `llvm::Error`s, these
-  /// three methods return a flag to indicate if the call is successful.
-  /// The initialization function asks the client for a DiagnosticsConsumer
-  /// that it direct the diagnostics to.
-  bool initializeCompilerInstanceWithContext(
-      StringRef CWD, const std::vector<std::string> &CommandLine,
-      DiagnosticConsumer *DC = nullptr);
+  /// @return False if the scanner incurs errors.
   bool
   computeDependenciesByNameWithContext(StringRef ModuleName,
                                        DependencyConsumer &Consumer,
                                        DependencyActionController &Controller);
-  bool finalizeCompilerInstance();
+
+  /// @brief Finalizes the diagnostics engine and deletes the compiler instance.
+  /// @return False if errors occur during finalization.
+  bool finalizeCompilerInstanceWithContext();
 
   llvm::vfs::FileSystem &getVFS() const { return *DepFS; }
 
@@ -169,13 +173,21 @@ class DependencyScanningWorker {
   friend CompilerInstanceWithContext;
   std::unique_ptr<CompilerInstanceWithContext> CIWithContext;
 
-  /// Actually carries out the scan. If \c OverlayFS is provided, it must be
-  /// based on top of DepFS.
-  bool scanDependencies(
-      StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
-      DependencyConsumer &Consumer, DependencyActionController &Controller,
-      DiagnosticConsumer &DC,
-      IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS = nullptr);
+  /// Private helper functions to actually carry out the scan. If \c OverlayFS
+  /// is provided, it must be based on top of DepFS.
+  bool scanDependencies(StringRef WorkingDirectory,
+                        const std::vector<std::string> &CommandLine,
+                        DependencyConsumer &Consumer,
+                        DependencyActionController &Controller,
+                        DiagnosticConsumer &DC,
+                        llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
+
+  bool scanDependencies(StringRef WorkingDirectory,
+                        ArrayRef<std::vector<std::string>> CommandLine,
+                        DependencyConsumer &Consumer,
+                        DependencyActionController &Controller,
+                        DiagnosticsEngine &Diags,
+                        llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
 };
 
 } // end namespace dependencies

clang/include/clang/Tooling/DependencyScanningTool.h

@@ -9,6 +9,7 @@
 #ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H
 #define LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H
 
+#include "clang/DependencyScanning/DependencyScannerImpl.h"
 #include "clang/DependencyScanning/DependencyScanningService.h"
 #include "clang/DependencyScanning/DependencyScanningUtils.h"
 #include "clang/DependencyScanning/DependencyScanningWorker.h"
@@ -127,8 +128,8 @@ class DependencyScanningTool {
   /// @param CWD The current working directory used during the scan.
   /// @param CommandLine The commandline used for the scan.
   /// @return Error if the initializaiton fails.
-  llvm::Error initializeCompilerInstanceWithContext(
-      StringRef CWD, const std::vector<std::string> &CommandLine);
+  llvm::Error initializeCompilerInstanceWithContextOrError(
+      StringRef CWD, ArrayRef<std::string> CommandLine);
 
   /// @brief Computes the dependeny for the module named ModuleName.
   /// @param ModuleName The name of the module for which this method computes
@@ -145,7 +146,7 @@ class DependencyScanningTool {
   /// @return An instance of \c TranslationUnitDeps if the scan is successful.
   ///         Otherwise it returns an error.
   llvm::Expected<clang::dependencies::TranslationUnitDeps>
-  computeDependenciesByNameWithContext(
+  computeDependenciesByNameWithContextOrError(
       StringRef ModuleName,
       const llvm::DenseSet<clang::dependencies::ModuleID> &AlreadySeen,
       clang::dependencies::LookupModuleOutputCallback LookupModuleOutput);
@@ -154,7 +155,7 @@ class DependencyScanningTool {
   ///        diagnostics and deletes the compiler instance. Call this method
   ///        once all names for a same commandline are scanned.
   /// @return Error if an error occured during finalization.
-  llvm::Error finalizeCompilerInstanceWithContext();
+  llvm::Error finalizeCompilerInstanceWithContextOrError();
 
   llvm::vfs::FileSystem &getWorkerVFS() const { return Worker.getVFS(); }
 

clang/lib/DependencyScanning/CMakeLists.txt

@@ -18,9 +18,7 @@ add_clang_library(clangDependencyScanning
   ClangDriverOptions
 
   LINK_LIBS
-  clangAST
   clangBasic
-  clangDriver
   clangFrontend
   clangLex
   clangSerialization