[clang][InstallAPI] Introduce basic driver to write out tbd files (#…

…81571)

This introduces a basic outline of installapi as a clang driver option.
It captures relevant information as cc1 args, which are common arguments
already passed to the linker to encode into TBD file outputs. This is
effectively an upstream for what already exists as `tapi installapi` in
Xcode toolchains, but directly in Clang. This patch does not handle any
AST traversing on input yet.

InstallAPI is broadly an operation that takes a series of header files
that represent a single dynamic library and generates a TBD file out of
it which represents all the linkable symbols and necessary attributes
for statically linking in clients. It is the linkable object in all
Apple SDKs and when building dylibs in Xcode. `clang -installapi` also
will support verification where it compares all the information recorded
for the TBD files against the already built binary, to catch possible
mismatches like when a declaration is missing a definition for an
exported symbol.

clang/include/clang/Basic/DiagnosticDriverKinds.td

@@ -804,4 +804,7 @@ def warn_android_unversioned_fallback : Warning<
 
 def err_drv_triple_version_invalid : Error<
   "version '%0' in target triple '%1' is invalid">;
+
+def err_drv_installapi_unsupported : Error<
+  "InstallAPI is not supported for '%0'">;
 }

clang/include/clang/Driver/Action.h

@@ -59,6 +59,7 @@ class Action {
     PreprocessJobClass,
     PrecompileJobClass,
     ExtractAPIJobClass,
+    InstallAPIJobClass,
     AnalyzeJobClass,
     MigrateJobClass,
     CompileJobClass,
@@ -448,6 +449,17 @@ class ExtractAPIJobAction : public JobAction {
   void addHeaderInput(Action *Input) { getInputs().push_back(Input); }
 };
 
+class InstallAPIJobAction : public JobAction {
+  void anchor() override;
+
+public:
+  InstallAPIJobAction(Action *Input, types::ID OutputType);
+
+  static bool classof(const Action *A) {
+    return A->getKind() == InstallAPIJobClass;
+  }
+};
+
 class AnalyzeJobAction : public JobAction {
   void anchor() override;
 

clang/include/clang/Driver/Options.td

@@ -309,6 +309,8 @@ class AnalyzerOpts<string base>
   : KeyPathAndMacro<"AnalyzerOpts->", base, "ANALYZER_"> {}
 class MigratorOpts<string base>
   : KeyPathAndMacro<"MigratorOpts.", base, "MIGRATOR_"> {}
+class InstallAPIOpts<string base>
+  : KeyPathAndMacro<"InstallAPIOpts.", base, "INSTALLAPI_"> {}
 
 // A boolean option which is opt-in in CC1. The positive option exists in CC1 and
 // Args.hasArg(OPT_ffoo) can be used to check that the flag is enabled.
@@ -1114,7 +1116,8 @@ def config_user_dir_EQ : Joined<["--"], "config-user-dir=">,
 def coverage : Flag<["-", "--"], "coverage">, Group<Link_Group>,
   Visibility<[ClangOption, CLOption]>;
 def cpp_precomp : Flag<["-"], "cpp-precomp">, Group<clang_ignored_f_Group>;
-def current__version : JoinedOrSeparate<["-"], "current_version">;
+def current__version : JoinedOrSeparate<["-"], "current_version">,
+  Visibility<[ClangOption, CC1Option]>;
 def cxx_isystem : JoinedOrSeparate<["-"], "cxx-isystem">, Group<clang_i_Group>,
   HelpText<"Add directory to the C++ SYSTEM include search path">,
   Visibility<[ClangOption, CC1Option]>,
@@ -1529,6 +1532,9 @@ def static_libsan : Flag<["-"], "static-libsan">,
   HelpText<"Statically link the sanitizer runtime (Not supported for ASan, TSan or UBSan on darwin)">;
 def : Flag<["-"], "shared-libasan">, Alias<shared_libsan>;
 def fasm : Flag<["-"], "fasm">, Group<f_Group>;
+def installapi : Flag<["-"], "installapi">,
+  Visibility<[ClangOption, CC1Option]>, Group<Action_Group>,
+  HelpText<"Create a text-based stub file by scanning header files">;
 
 defm assume_unique_vtables : BoolFOption<"assume-unique-vtables",
   CodeGenOpts<"AssumeUniqueVTables">, DefaultTrue,
@@ -4291,7 +4297,9 @@ def verify_pch : Flag<["-"], "verify-pch">, Group<Action_Group>,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Load and verify that a pre-compiled header file is not stale">;
 def init : Separate<["-"], "init">;
-def install__name : Separate<["-"], "install_name">;
+def install__name : Separate<["-"], "install_name">,
+  Visibility<[ClangOption, CC1Option]>,
+  MarshallingInfoString<InstallAPIOpts<"InstallName">>;
 def iprefix : JoinedOrSeparate<["-"], "iprefix">, Group<clang_i_Group>,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Set the -iwithprefix/-iwithprefixbefore prefix">, MetaVarName<"<dir>">;

clang/include/clang/Driver/Types.def

@@ -94,6 +94,7 @@ TYPE("lto-bc",                   LTO_BC,       INVALID,         "o",      phases
 TYPE("ast",                      AST,          INVALID,         "ast",    phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("ifs",                      IFS,          INVALID,         "ifs",    phases::IfsMerge)
 TYPE("ifs-cpp",                  IFS_CPP,      INVALID,         "ifs",    phases::Compile, phases::IfsMerge)
+TYPE("tbd",                      TextAPI,      INVALID,         "tbd",    phases::Precompile)
 TYPE("pcm",                      ModuleFile,   INVALID,         "pcm",    phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("header-unit",              HeaderUnit,   INVALID,         "pcm",    phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("plist",                    Plist,        INVALID,         "plist",  phases::Compile, phases::Backend, phases::Assemble, phases::Link)

clang/include/clang/Frontend/CompilerInstance.h

@@ -294,6 +294,13 @@ class CompilerInstance : public ModuleLoader {
     return Invocation->getFrontendOpts();
   }
 
+  InstallAPIOptions &getInstallAPIOpts() {
+    return Invocation->getInstallAPIOpts();
+  }
+  const InstallAPIOptions &getInstallAPIOpts() const {
+    return Invocation->getInstallAPIOpts();
+  }
+
   HeaderSearchOptions &getHeaderSearchOpts() {
     return Invocation->getHeaderSearchOpts();
   }

clang/include/clang/Frontend/CompilerInvocation.h

@@ -18,11 +18,12 @@
 #include "clang/Basic/LangStandard.h"
 #include "clang/Frontend/DependencyOutputOptions.h"
 #include "clang/Frontend/FrontendOptions.h"
+#include "clang/Frontend/InstallAPIOptions.h"
 #include "clang/Frontend/MigratorOptions.h"
 #include "clang/Frontend/PreprocessorOutputOptions.h"
 #include "clang/StaticAnalyzer/Core/AnalyzerOptions.h"
-#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include <memory>
 #include <string>
 
@@ -111,6 +112,9 @@ class CompilerInvocationBase {
   /// Options controlling preprocessed output.
   std::shared_ptr<PreprocessorOutputOptions> PreprocessorOutputOpts;
 
+  /// Options controlling InstallAPI operations and output.
+  std::shared_ptr<InstallAPIOptions> InstallAPIOpts;
+
   /// Dummy tag type whose instance can be passed into the constructor to
   /// prevent creation of the reference-counted option objects.
   struct EmptyConstructor {};
@@ -145,6 +149,7 @@ class CompilerInvocationBase {
   const PreprocessorOutputOptions &getPreprocessorOutputOpts() const {
     return *PreprocessorOutputOpts;
   }
+  const InstallAPIOptions &getInstallAPIOpts() const { return *InstallAPIOpts; }
   /// @}
 
   /// Command line generation.
@@ -237,6 +242,7 @@ class CompilerInvocation : public CompilerInvocationBase {
   using CompilerInvocationBase::getFrontendOpts;
   using CompilerInvocationBase::getDependencyOutputOpts;
   using CompilerInvocationBase::getPreprocessorOutputOpts;
+  using CompilerInvocationBase::getInstallAPIOpts;
   /// @}
 
   /// Mutable getters.
@@ -258,6 +264,7 @@ class CompilerInvocation : public CompilerInvocationBase {
   PreprocessorOutputOptions &getPreprocessorOutputOpts() {
     return *PreprocessorOutputOpts;
   }
+  InstallAPIOptions &getInstallAPIOpts() { return *InstallAPIOpts; }
   /// @}
 
   /// Base class internals.

clang/include/clang/Frontend/FrontendActions.h

@@ -130,6 +130,16 @@ class GenerateModuleAction : public ASTFrontendAction {
   bool shouldEraseOutputFiles() override;
 };
 
+class InstallAPIAction : public ASTFrontendAction {
+protected:
+  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
+                                                 StringRef InFile) override;
+
+public:
+  static std::unique_ptr<llvm::raw_pwrite_stream>
+  CreateOutputFile(CompilerInstance &CI, StringRef InFile);
+};
+
 class GenerateInterfaceStubsAction : public ASTFrontendAction {
 protected:
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,

clang/include/clang/Frontend/FrontendOptions.h

@@ -100,6 +100,9 @@ enum ActionKind {
   /// Only execute frontend initialization.
   InitOnly,
 
+  // Create TextAPI stub.
+  InstallAPI,
+
   /// Dump information about a module file.
   ModuleFileInfo,
 

clang/include/clang/Frontend/InstallAPIOptions.h

@@ -0,0 +1,28 @@
+//===--- InstallAPIOptions.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_FRONTEND_INSTALLAPIOPTIONS_H
+#define LLVM_CLANG_FRONTEND_INSTALLAPIOPTIONS_H
+
+#include "llvm/TextAPI/PackedVersion.h"
+
+namespace clang {
+
+/// InstallAPIOptions - Options for controlling InstallAPI verification and
+/// TextAPI output.
+class InstallAPIOptions {
+public:
+  /// The install name which is apart of the library's ID.
+  std::string InstallName;
+
+  /// The current version which is apart of the library's ID.
+  llvm::MachO::PackedVersion CurrentVersion;
+};
+} // namespace clang
+
+#endif

clang/include/clang/InstallAPI/Context.h

@@ -0,0 +1,65 @@
+//===- InstallAPI/Context.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Top level types for interacting with the generic clang driver and frontend
+// for InstallAPI operations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_INSTALLAPI_CONTEXT_H
+#define LLVM_CLANG_INSTALLAPI_CONTEXT_H
+
+#include "clang/AST/ASTConsumer.h"
+#include "clang/Basic/Diagnostic.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/TextAPI/InterfaceFile.h"
+#include "llvm/TextAPI/RecordVisitor.h"
+#include "llvm/TextAPI/RecordsSlice.h"
+
+namespace clang {
+namespace installapi {
+
+/// Struct used for generating validating InstallAPI.
+/// The attributes captured represent all necessary information
+/// to generate TextAPI output.
+struct InstallAPIContext {
+
+  /// Library attributes that are typically passed as linker inputs.
+  llvm::MachO::RecordsSlice::BinaryAttrs BA;
+
+  /// Active target triple to parse.
+  llvm::Triple TargetTriple{};
+
+  /// Output stream to write TextAPI file to.
+  std::unique_ptr<llvm::raw_pwrite_stream> OS = nullptr;
+
+  /// DiagnosticsEngine to report errors.
+  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags = nullptr;
+
+  /// File Path of output location.
+  StringRef OutputLoc{};
+
+  /// What encoding to write output as.
+  llvm::MachO::FileType FT = llvm::MachO::FileType::TBD_V5;
+};
+
+class InstallAPIConsumer : public ASTConsumer {
+public:
+  InstallAPIConsumer(InstallAPIContext InstallAPICtx)
+      : Ctx(std::move(InstallAPICtx)) {}
+
+  void HandleTranslationUnit(ASTContext &ASTContext) override;
+
+private:
+  InstallAPIContext Ctx;
+};
+
+} // namespace installapi
+} // namespace clang
+
+#endif // LLVM_CLANG_INSTALLAPI_CONTEXT_H

clang/lib/CMakeLists.txt

@@ -23,6 +23,7 @@ add_subdirectory(Tooling)
 add_subdirectory(DirectoryWatcher)
 add_subdirectory(Index)
 add_subdirectory(IndexSerialization)
+add_subdirectory(InstallAPI)
 add_subdirectory(StaticAnalyzer)
 add_subdirectory(Format)
 if(CLANG_INCLUDE_TESTS)

clang/lib/Driver/Action.cpp

@@ -32,6 +32,8 @@ const char *Action::getClassName(ActionClass AC) {
   case CompileJobClass: return "compiler";
   case BackendJobClass: return "backend";
   case AssembleJobClass: return "assembler";
+  case InstallAPIJobClass:
+    return "installapi";
   case IfsMergeJobClass: return "interface-stub-merger";
   case LinkJobClass: return "linker";
   case LipoJobClass: return "lipo";
@@ -362,6 +364,11 @@ void ExtractAPIJobAction::anchor() {}
 ExtractAPIJobAction::ExtractAPIJobAction(Action *Inputs, types::ID OutputType)
     : JobAction(ExtractAPIJobClass, Inputs, OutputType) {}
 
+void InstallAPIJobAction::anchor() {}
+
+InstallAPIJobAction::InstallAPIJobAction(Action *Inputs, types::ID OutputType)
+    : JobAction(InstallAPIJobClass, Inputs, OutputType) {}
+
 void AnalyzeJobAction::anchor() {}
 
 AnalyzeJobAction::AnalyzeJobAction(Action *Input, types::ID OutputType)

clang/lib/Driver/Driver.cpp

@@ -4189,6 +4189,11 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
         break;
       }
 
+      if (auto *IAA = dyn_cast<InstallAPIJobAction>(Current)) {
+        Current = nullptr;
+        break;
+      }
+
       // FIXME: Should we include any prior module file outputs as inputs of
       // later actions in the same command line?
 
@@ -4319,6 +4324,13 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
     if (!MergerInputs.empty())
       Actions.push_back(
           C.MakeAction<IfsMergeJobAction>(MergerInputs, types::TY_Image));
+  } else if (Args.hasArg(options::OPT_installapi)) {
+    // TODO: Lift restriction once operation can handle multiple inputs.
+    assert(Inputs.size() == 1 && "InstallAPI action can only handle 1 input");
+    const auto [InputType, InputArg] = Inputs.front();
+    Action *Current = C.MakeAction<InputAction>(*InputArg, InputType);
+    Actions.push_back(
+        C.MakeAction<InstallAPIJobAction>(Current, types::TY_TextAPI));
   }
 
   for (auto Opt : {options::OPT_print_supported_cpus,
@@ -4762,6 +4774,8 @@ Action *Driver::ConstructPhaseAction(
       return C.MakeAction<VerifyPCHJobAction>(Input, types::TY_Nothing);
     if (Args.hasArg(options::OPT_extract_api))
       return C.MakeAction<ExtractAPIJobAction>(Input, types::TY_API_INFO);
+    if (Args.hasArg(options::OPT_installapi))
+      return C.MakeAction<InstallAPIJobAction>(Input, types::TY_TextAPI);
     return C.MakeAction<CompileJobAction>(Input, types::TY_LLVM_BC);
   }
   case phases::Backend: {
@@ -6441,7 +6455,7 @@ bool Driver::ShouldUseClangCompiler(const JobAction &JA) const {
   // And say "no" if this is not a kind of action clang understands.
   if (!isa<PreprocessJobAction>(JA) && !isa<PrecompileJobAction>(JA) &&
       !isa<CompileJobAction>(JA) && !isa<BackendJobAction>(JA) &&
-      !isa<ExtractAPIJobAction>(JA))
+      !isa<ExtractAPIJobAction>(JA) && !isa<InstallAPIJobAction>(JA))
     return false;
 
   return true;

clang/lib/Driver/ToolChain.cpp

@@ -532,6 +532,7 @@ Tool *ToolChain::getTool(Action::ActionClass AC) const {
   case Action::PrecompileJobClass:
   case Action::PreprocessJobClass:
   case Action::ExtractAPIJobClass:
+  case Action::InstallAPIJobClass:
   case Action::AnalyzeJobClass:
   case Action::MigrateJobClass:
   case Action::VerifyPCHJobClass:

clang/lib/Driver/ToolChains/Clang.cpp

@@ -4939,6 +4939,17 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     if (Arg *ExtractAPIIgnoresFileArg =
             Args.getLastArg(options::OPT_extract_api_ignores_EQ))
       ExtractAPIIgnoresFileArg->render(Args, CmdArgs);
+  } else if (isa<InstallAPIJobAction>(JA)) {
+    if (!Triple.isOSDarwin())
+      D.Diag(diag::err_drv_installapi_unsupported) << Triple.str();
+
+    CmdArgs.push_back("-installapi");
+    // Add necessary library arguments for InstallAPI.
+    if (const Arg *A = Args.getLastArg(options::OPT_install__name))
+      A->render(Args, CmdArgs);
+    if (const Arg *A = Args.getLastArg(options::OPT_current__version))
+      A->render(Args, CmdArgs);
+
   } else {
     assert((isa<CompileJobAction>(JA) || isa<BackendJobAction>(JA)) &&
            "Invalid action for clang tool.");

clang/lib/Frontend/CMakeLists.txt

@@ -7,6 +7,7 @@ set(LLVM_LINK_COMPONENTS
   ProfileData
   Support
   TargetParser
+  TextAPI
   )
 
 add_clang_library(clangFrontend
@@ -27,6 +28,7 @@ add_clang_library(clangFrontend
   HeaderIncludeGen.cpp
   InitPreprocessor.cpp
   LayoutOverrideSource.cpp
+  InstallAPIConsumer.cpp
   LogDiagnosticPrinter.cpp
   ModuleDependencyCollector.cpp
   MultiplexConsumer.cpp
@@ -53,6 +55,7 @@ add_clang_library(clangFrontend
   clangBasic
   clangDriver
   clangEdit
+  clangInstallAPI
   clangLex
   clangParse
   clangSema