[clang][Dependency Scanning] Implementing CompilerInstanceWithContext to Speedup By Name Scans

clang/include/clang/Frontend/CompilerInstance.h

@@ -948,6 +948,12 @@ class CompilerInstance : public ModuleLoader {
     DependencyCollectors.push_back(std::move(Listener));
   }
 
+  void clearDependencyCollectors() { DependencyCollectors.clear(); }
+
+  std::vector<std::shared_ptr<DependencyCollector>> &getDependencyCollectors() {
+    return DependencyCollectors;
+  }
+
   void setExternalSemaSource(IntrusiveRefCntPtr<ExternalSemaSource> ESS);
 
   ModuleCache &getModuleCache() const { return *ModCache; }

clang/include/clang/Frontend/Utils.h

@@ -40,6 +40,7 @@ class DiagnosticsEngine;
 class ExternalSemaSource;
 class FrontendOptions;
 class PCHContainerReader;
+class PPCallbacks;
 class Preprocessor;
 class PreprocessorOptions;
 class PreprocessorOutputOptions;
@@ -87,6 +88,9 @@ class DependencyCollector {
                                   bool IsSystem, bool IsModuleFile,
                                   bool IsMissing);
 
+  /// @return the PPCallback this collector added to the Preprocessor.
+  virtual PPCallbacks *getPPCallback() { return nullptr; };
+
 protected:
   /// Return true if the filename was added to the list of dependencies, false
   /// otherwise.

clang/include/clang/Lex/Preprocessor.h

@@ -1327,6 +1327,7 @@ class Preprocessor {
                                                 std::move(Callbacks));
     Callbacks = std::move(C);
   }
+  void removePPCallbacks() { Callbacks.reset(); }
   /// \}
 
   /// Get the number of tokens processed so far.

clang/include/clang/Tooling/DependencyScanning/CompilerInstanceWithContext.h

@@ -0,0 +1,90 @@
+//===- CompilerInstanceWithContext.h - clang scanning compiler instance ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_COMPILERINSTANCEWITHCONTEXT_H
+#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_COMPILERINSTANCEWITHCONTEXT_H
+
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/Driver/Compilation.h"
+#include "clang/Driver/Driver.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/TextDiagnosticPrinter.h"
+#include "clang/Serialization/ModuleCache.h"
+#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
+#include <string>
+#include <vector>
+
+namespace clang {
+namespace tooling {
+namespace dependencies {
+
+// Forward declarations.
+class DependencyScanningWorker;
+class DependencyConsumer;
+class DependencyActionController;
+
+class CompilerInstanceWithContext {
+  // Context
+  DependencyScanningWorker &Worker;
+  llvm::StringRef CWD;
+  std::vector<std::string> CommandLine;
+  static const uint64_t MAX_NUM_NAMES = (1 << 12);
+  static const std::string FakeFileBuffer;
+
+  // Context - file systems
+  llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFS;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay;
+
+  // Context - Diagnostics engine, file manager and source mamanger.
+  std::string DiagnosticOutput;
+  llvm::raw_string_ostream DiagnosticsOS;
+  std::unique_ptr<TextDiagnosticPrinter> DiagPrinter;
+  IntrusiveRefCntPtr<DiagnosticsEngine> Diags;
+  std::unique_ptr<FileManager> FileMgr;
+  std::unique_ptr<SourceManager> SrcMgr;
+
+  // Context - compiler invocation
+  std::unique_ptr<clang::driver::Driver> Driver;
+  std::unique_ptr<clang::driver::Compilation> Compilation;
+  std::unique_ptr<CompilerInvocation> Invocation;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFSFromCompilerInvocation;
+
+  // Context - output options
+  std::unique_ptr<DependencyOutputOptions> OutputOpts;
+
+  // Context - stable directory handling
+  llvm::SmallVector<StringRef> StableDirs;
+  PrebuiltModulesAttrsMap PrebuiltModuleVFSMap;
+
+  // Compiler Instance
+  IntrusiveRefCntPtr<ModuleCache> ModCache;
+  std::unique_ptr<CompilerInstance> CIPtr;
+
+  //   // Source location offset.
+  int32_t SrcLocOffset = 0;
+
+public:
+  CompilerInstanceWithContext(DependencyScanningWorker &Worker, StringRef CWD,
+                              const std::vector<std::string> &CMD)
+      : Worker(Worker), CWD(CWD), CommandLine(CMD),
+        DiagnosticsOS(DiagnosticOutput) {};
+
+  llvm::Error initialize();
+  llvm::Error computeDependencies(StringRef ModuleName,
+                                  DependencyConsumer &Consumer,
+                                  DependencyActionController &Controller);
+  llvm::Error finalize();
+};
+} // namespace dependencies
+} // namespace tooling
+} // namespace clang
+
+#endif

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

@@ -161,6 +161,43 @@ class DependencyScanningTool {
 
   llvm::vfs::FileSystem &getWorkerVFS() const { return Worker.getVFS(); }
 
+  /// The following three methods provides a new interface to perform
+  /// by name dependency scan. The new interface's intention is to improve
+  /// dependency scanning performance when a sequence of name is looked up
+  /// with the same current working directory and the command line.
+
+  /// @brief Initializing the context and the compiler instance to perform.
+  ///        This method must be called before performing scanning.
+  /// @param CWD The current working directory used during the scan.
+  /// @param CommandLine The commandline used for the scan.
+  /// @return Error if the initializaiton fails.
+  llvm::Error initializeCompilerInstacneWithContext(
+      StringRef CWD, const std::vector<std::string> &CommandLine);
+
+  /// @brief Computes the dependeny for the module named ModuleName.
+  /// @param ModuleName The name of the module for which this method computes
+  ///.                  dependencies.
+  /// @param AlreadySeen This stores modules which have previously been
+  ///                    reported. Use the same instance for all calls to this
+  ///                    function for a single \c DependencyScanningTool in a
+  ///                    single build. Note that this parameter is not part of
+  ///                    the context because it can be shared across different
+  ///                    worker threads and each worker thread may update it.
+  /// @param LookupModuleOutput This function is called to fill in
+  ///                           "-fmodule-file=", "-o" and other output
+  ///                           arguments for dependencies.
+  /// @return An instance of \c TranslationUnitDeps if the scan is successful.
+  ///         Otherwise it returns an error.
+  llvm::Expected<TranslationUnitDeps> computeDependenciesByNameWithContext(
+      StringRef ModuleName, const llvm::DenseSet<ModuleID> &AlreadySeen,
+      LookupModuleOutputCallback LookupModuleOutput);
+
+  /// @brief This method finializes the compiler instance. It finalizes the
+  ///        diagnostics and deletes the compiler instance. Call this method
+  ///        once all names for a same commandline are scanned.
+  /// @return Error if an error occured during finalization.
+  llvm::Error finalizeCompilerInstanceWithContext();
+
 private:
   DependencyScanningWorker Worker;
 };

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

@@ -13,6 +13,7 @@
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Frontend/PCHContainerOperations.h"
+#include "clang/Tooling/DependencyScanning/CompilerInstanceWithContext.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "llvm/Support/Error.h"
@@ -74,6 +75,22 @@ class DependencyActionController {
                                          ModuleOutputKind Kind) = 0;
 };
 
+/// Some helper functions for the dependency scanning worker.
+std::string
+deduceDepTarget(const std::string &OutputFile,
+                const SmallVectorImpl<FrontendInputFile> &InputFiles);
+void canonicalizeDefines(PreprocessorOptions &PPOpts);
+void sanitizeDiagOpts(DiagnosticOptions &DiagOpts);
+std::unique_ptr<DiagnosticOptions>
+createDiagOptions(const std::vector<std::string> &CommandLine);
+
+using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
+bool visitPrebuiltModule(StringRef PrebuiltModuleFilename, CompilerInstance &CI,
+                         PrebuiltModuleFilesT &ModuleFiles,
+                         PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
+                         DiagnosticsEngine &Diags,
+                         const ArrayRef<StringRef> StableDirs);
+
 /// An individual dependency scanning worker that is able to run on its own
 /// thread.
 ///
@@ -136,6 +153,34 @@ class DependencyScanningWorker {
                                   DependencyActionController &Controller,
                                   StringRef ModuleName);
 
+  /// The three method below implements a new interface for by name
+  /// dependency scanning. They together enable the dependency scanning worker
+  /// to more effectively perform scanning for a sequence of modules
+  /// by name when the CWD and CommandLine are holding constant.
+
+  /// @brief Initializing the context and the compiler instance to perform.
+  /// @param CWD The current working directory used during the scan.
+  /// @param CommandLine The commandline used for the scan.
+  /// @return Error if the initializaiton fails.
+  llvm::Error initializeCompierInstanceWithContext(
+      StringRef CWD, const std::vector<std::string> &CommandLine);
+
+  /// @brief Performaces dependency scanning for the module whose name is
+  ///        specified.
+  /// @param ModuleName  The name of the module whose dependency will be
+  ///                    scanned.
+  /// @param Consumer The dependency consumer that stores the results.
+  /// @param Controller The controller for the dependency scanning action.
+  /// @return Error of the scanner incurs errors.
+  llvm::Error
+  computeDependenciesByNameWithContext(StringRef ModuleName,
+                                       DependencyConsumer &Consumer,
+                                       DependencyActionController &Controller);
+
+  /// @brief Finalizes the diagnostics engine and deletes the compiler instance.
+  /// @return Error if errors occur during finalization.
+  llvm::Error finalizeCompilerInstanceWithContext();
+
   llvm::vfs::FileSystem &getVFS() const { return *BaseFS; }
 
 private:
@@ -151,6 +196,9 @@ class DependencyScanningWorker {
   /// (passed in the constructor).
   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
 
+  friend class CompilerInstanceWithContext;
+  std::unique_ptr<CompilerInstanceWithContext> CIWithContext;
+
   /// Private helper functions.
   bool scanDependencies(StringRef WorkingDirectory,
                         const std::vector<std::string> &CommandLine,
@@ -161,6 +209,32 @@ class DependencyScanningWorker {
                         std::optional<StringRef> ModuleName);
 };
 
+class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
+  DependencyScanningWorkerFilesystem *DepFS;
+
+public:
+  ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {
+    FileMgr.getVirtualFileSystem().visit([&](llvm::vfs::FileSystem &FS) {
+      auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(&FS);
+      if (DFS) {
+        assert(!DepFS && "Found multiple scanning VFSs");
+        DepFS = DFS;
+      }
+    });
+    assert(DepFS && "Did not find scanning VFS");
+  }
+
+  std::unique_ptr<DependencyDirectivesGetter>
+  cloneFor(FileManager &FileMgr) override {
+    return std::make_unique<ScanningDependencyDirectivesGetter>(FileMgr);
+  }
+
+  std::optional<ArrayRef<dependency_directives_scan::Directive>>
+  operator()(FileEntryRef File) override {
+    return DepFS->getDirectiveTokens(File.getName());
+  }
+};
+
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -282,11 +282,12 @@ class ModuleDepCollector final : public DependencyCollector {
                      CompilerInstance &ScanInstance, DependencyConsumer &C,
                      DependencyActionController &Controller,
                      CompilerInvocation OriginalCI,
-                     const PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
+                     const PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
                      const ArrayRef<StringRef> StableDirs);
 
   void attachToPreprocessor(Preprocessor &PP) override;
   void attachToASTReader(ASTReader &R) override;
+  PPCallbacks *getPPCallback() override { return CollectorPPPtr; }
 
   /// Apply any changes implied by the discovered dependencies to the given
   /// invocation, (e.g. disable implicit modules, add explicit module paths).
@@ -305,7 +306,7 @@ class ModuleDepCollector final : public DependencyCollector {
   DependencyActionController &Controller;
   /// Mapping from prebuilt AST filepaths to their attributes referenced during
   /// dependency collecting.
-  const PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
+  const PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
   /// Directory paths known to be stable through an active development and build
   /// cycle.
   const ArrayRef<StringRef> StableDirs;
@@ -339,6 +340,10 @@ class ModuleDepCollector final : public DependencyCollector {
   std::optional<P1689ModuleInfo> ProvidedStdCXXModule;
   std::vector<P1689ModuleInfo> RequiredStdCXXModules;
 
+  /// A pointer to the preprocessor callback so we can invoke it directly
+  /// if needed.
+  ModuleDepCollectorPP *CollectorPPPtr = nullptr;
+
   /// Checks whether the module is known as being prebuilt.
   bool isPrebuiltModule(const Module *M);
 

clang/lib/Tooling/DependencyScanning/CMakeLists.txt

@@ -6,6 +6,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_clang_library(clangDependencyScanning
+  CompilerInstanceWithContext.cpp
   DependencyScanningFilesystem.cpp
   DependencyScanningService.cpp
   DependencyScanningWorker.cpp