refactor: incremental update for compilation database and introduce query toolchain

.github/workflows/cmake.yml

@@ -103,5 +103,9 @@ jobs:
             EXE_EXT=".exe"
           fi
 
+          if [[ "${{ runner.os }}" == "macOS" ]]; then
+            export PATH="/opt/homebrew/opt/llvm@20/bin:/opt/homebrew/opt/lld@20/bin:$PATH"
+          fi
+
           ./build/bin/unit_tests${EXE_EXT} --test-dir="./tests/data"
           uv run pytest -s --log-cli-level=INFO tests/integration --executable=./build/bin/clice${EXE_EXT}

CMakeLists.txt

@@ -27,8 +27,12 @@ include("${PROJECT_SOURCE_DIR}/cmake/package.cmake")
 
 add_library(clice_options INTERFACE)
 
+if(CLICE_ENABLE_TEST)
+    target_compile_definitions(clice_options INTERFACE CLICE_ENABLE_TEST=1)
+endif()
+
 if(CLICE_CI_ENVIRONMENT)
-    target_compile_definitions(clice_options INTERFACE CLICE_CI_ENVIRONMENT)
+    target_compile_definitions(clice_options INTERFACE CLICE_CI_ENVIRONMENT=1)
 endif()
 
 if(WIN32)

bin/clice.cc

@@ -94,11 +94,11 @@ int main(int argc, const char** argv) {
     logging::stderr_logger("clice", logging::options);
 
     if(auto result = fs::init_resource_dir(argv[0]); !result) {
-        LOGGING_FATAL("Cannot find default resource directory, because {}", result.error());
+        LOG_FATAL("Cannot find default resource directory, because {}", result.error());
     }
 
     for(int i = 0; i < argc; ++i) {
-        LOGGING_INFO("argv[{}] = {}", i, argv[i]);
+        LOG_INFO("argv[{}] = {}", i, argv[i]);
     }
 
     async::init();
@@ -112,13 +112,13 @@ int main(int argc, const char** argv) {
     switch(mode) {
         case Mode::Pipe: {
             async::net::listen(loop);
-            LOGGING_INFO("Server starts listening on stdin/stdout");
+            LOG_INFO("Server starts listening on stdin/stdout");
             break;
         }
 
         case Mode::Socket: {
             async::net::listen(host.c_str(), port, loop);
-            LOGGING_INFO("Server starts listening on {}:{}", host.getValue(), port.getValue());
+            LOG_INFO("Server starts listening on {}:{}", host.getValue(), port.getValue());
             break;
         }
 
@@ -130,7 +130,7 @@ int main(int argc, const char** argv) {
 
     async::run();
 
-    LOGGING_INFO("clice exit normally!");
+    LOG_INFO("clice exit normally!");
 
     return 0;
 }

include/Compiler/Command.h

@@ -21,7 +21,7 @@ struct CommandOptions {
     bool resource_dir = false;
 
     /// Query the compiler driver for additional information, such as system includes and target.
-    bool query_driver = false;
+    bool query_toolchain = false;
 
     /// Suppress the warning log if failed to query driver info.
     /// Set true in unittests to avoid cluttering test output.
@@ -35,36 +35,34 @@ struct CommandOptions {
 };
 
 enum class UpdateKind : std::uint8_t {
-    Unchange,
-    Create,
-    Update,
-    Delete,
-};
-
-struct DriverInfo {
-    /// The target of this driver.
-    llvm::StringRef target;
-
-    /// The default system includes of this driver.
-    llvm::ArrayRef<const char*> system_includes;
+    Unchanged,
+    Inserted,
+    Deleted,
 };
 
 struct UpdateInfo {
     /// The kind of update.
     UpdateKind kind;
 
-    llvm::StringRef file;
+    /// The updated file.
+    std::uint32_t path_id;
+
+    /// The compilation context of this file command, which could
+    /// be used to identity the same file with different compilation
+    /// contexts.
+    const void* context;
 };
 
-struct LookupInfo {
+struct CompilationContext {
+    /// The working directory of compilation.
     llvm::StringRef directory;
 
+    /// The compilation arguments.
     std::vector<const char*> arguments;
-
-    /// The include arguments indices in the arguments list.
-    std::vector<std::uint32_t> include_indices;
 };
 
+std::string print_argv(llvm::ArrayRef<const char*> args);
+
 class CompilationDatabase {
 public:
     CompilationDatabase();
@@ -79,45 +77,48 @@ class CompilationDatabase {
 
     ~CompilationDatabase();
 
-private:
-    struct Impl;
-
-    using Self = CompilationDatabase;
-
 public:
+    /// Read the compilation database on the give file and return the
+    /// incremental update infos.
+    std::vector<UpdateInfo> load_compile_database(llvm::StringRef file);
+
+    /// Lookup the compilation context of specific file. If the context
+    /// param is provided, we will return the compilation context corresponding
+    /// to the handle. Otherwise we just return the first one(if the file have)
+    /// multiple compilation contexts.
+    CompilationContext lookup(llvm::StringRef file,
+                              const CommandOptions& options = {},
+                              const void* context = nullptr);
+
+    /// TODO: list all compilation context of the file, this is useful to show
+    /// all contexts and let user choose one.
+    /// std::vector<CompilationContext> fetch_all(llvm::StringRef file);
+
     /// Get an the option for specific argument.
     static std::optional<std::uint32_t> get_option_id(llvm::StringRef argument);
 
+    /// FIXME: bad interface design ...
+    std::vector<const char*> files();
+
+    /// FIXME: remove this api?
     auto save_string(llvm::StringRef string) -> llvm::StringRef;
 
-    /// Query the compiler driver and return its driver info.
-    auto query_driver(llvm::StringRef driver)
-        -> std::expected<DriverInfo, toolchain::QueryDriverError>;
+#ifdef CLICE_ENABLE_TEST
 
-    /// Update with arguments.
-    auto update_command(llvm::StringRef directory,
-                        llvm::StringRef file,
-                        llvm::ArrayRef<const char*> arguments) -> UpdateInfo;
+    void add_command(llvm::StringRef directory,
+                     llvm::StringRef file,
+                     llvm::ArrayRef<const char*> arguments);
 
-    /// Update with full command.
-    auto update_command(llvm::StringRef directory, llvm::StringRef file, llvm::StringRef command)
-        -> UpdateInfo;
+    void add_command(llvm::StringRef directory, llvm::StringRef file, llvm::StringRef command);
 
+    /// FIXME: remove this
     /// Update commands from json file and return all updated file.
-    auto load_commands(llvm::StringRef json_content, llvm::StringRef workspace)
-        -> std::expected<std::vector<UpdateInfo>, std::string>;
-
-    /// Load compile commands from given directories. If no valid commands are found,
-    /// search recursively from the workspace directory.
-    auto load_compile_database(llvm::ArrayRef<std::string> compile_commands_dirs,
-                               llvm::StringRef workspace) -> void;
-
-    /// Get compile command from database. `file` should has relative path of workspace.
-    auto lookup(llvm::StringRef file, CommandOptions options = {}) -> LookupInfo;
-
-    std::vector<const char*> files();
+    std::expected<std::vector<UpdateInfo>, std::string> load_commands(llvm::StringRef json_content,
+                                                                      llvm::StringRef workspace);
+#endif
 
 private:
+    struct Impl;
     std::unique_ptr<Impl> self;
 };
 

include/Compiler/Compilation.h

@@ -23,6 +23,8 @@ struct CompilationParams {
 
     std::string directory;
 
+    bool arguments_from_database = false;
+
     /// Responsible for storing the arguments.
     std::vector<const char*> arguments;
 

include/Compiler/Toolchain.h

@@ -1,51 +1,49 @@
 #pragma once
 
-#include <expected>
-#include "Support/Enum.h"
-#include "Support/Format.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/STLExtras.h"
 
 namespace clice::toolchain {
 
-struct QueryDriverError {
-    struct ErrorKind : refl::Enum<ErrorKind> {
-        enum Kind : std::uint8_t {
-            NotFoundInPATH,
-            NotImplemented,
-            FailToCreateTempFile,
-            InvokeDriverFail,
-            OutputFileNotReadable,
-            InvalidOutputFormat,
-        };
-
-        using Enum::Enum;
-    };
-
-    ErrorKind kind;
-    std::string detail;
+enum class CompilerFamily {
+    Unknown,
+    GCC,      // Covers gcc, g++, cc, c++, and versioned/arch variants
+    Clang,    // Covers clang, clang++, and versioned variants (excluding clang-cl)
+    MSVC,     // Covers cl
+    ClangCL,  // Covers clang-cl explicitly
+    NVCC,     // Covers nvcc
+    Intel,    // Covers icc, icpc, icx, dpcpp
+    Zig,      // Covers zig cc / zig c++ (assumed GCC/Clang compatible for query)
 };
 
-struct QueryResult {
-    std::string target;
-    llvm::SmallVector<std::string, 8> includes;
+struct QueryParams {
+    llvm::StringRef file;
+    llvm::StringRef directory;
+    llvm::ArrayRef<const char*> arguments;
+    llvm::function_ref<const char*(const char*)> callback;
 };
 
-enum class Kind {};
+/// Query the toolchain info and return the full arguments, the returned arguments should be
+/// converted to `clang::CompilerInvocation::CreateFromArgs` directly.
+std::vector<const char*> query_toolchain(const QueryParams& params);
 
-struct Toolchain {};
+CompilerFamily driver_family(llvm::StringRef driver);
 
-/// Query toolchain info according to the original compilation arguments.
-Toolchain query_toolchain(llvm::ArrayRef<const char*> arguments);
+/// Query g++ or mingw toolchain info. We detect the target and corresponding
+/// gcc toolchain install path as default behavior.
+std::vector<const char*> query_gcc_toolchain(const QueryParams& params);
 
-auto query_driver(llvm::StringRef driver) -> std::expected<QueryResult, QueryDriverError>;
+/// Query clang++ or any clang based toolchain, e.g. zig cc/c++. We query
+/// the full cc1 command of clang toolchain as default.
+/// TODO: Is armclang also compatible?
+std::vector<const char*> query_clang_toolchain(const QueryParams& params);
 
-}  // namespace clice::toolchain
+/// Query the msvc or clang-cl toolchain, default behavior only adds the
+/// target and includes info.
+std::vector<const char*> query_msvc_toolchain(const QueryParams& params);
 
-template <>
-struct std::formatter<clice::toolchain::QueryDriverError> : std::formatter<std::string_view> {
-    template <typename FormatContext>
-    auto format(const clice::toolchain::QueryDriverError& e, FormatContext& ctx) const {
-        return std::format_to(ctx.out(), "{} {}", e.kind.name(), e.detail);
-    }
-};
+/// FIXME: To be implemented
+std::vector<const char*> query_nvcc_toolchain(const QueryParams& params);
+
+}  // namespace clice::toolchain

include/Support/FileSystem.h

@@ -50,6 +50,8 @@ inline std::expected<void, std::string> init_resource_dir(llvm::StringRef execut
     return std::expected<void, std::string>();
 }
 
+using llvm::sys::fs::createTemporaryFile;
+
 inline std::expected<std::string, std::error_code> createTemporaryFile(llvm::StringRef prefix,
                                                                        llvm::StringRef suffix) {
     llvm::SmallString<128> path;

include/Support/Logging.h

@@ -92,14 +92,26 @@ void critical [[noreturn]] (logging_format<Args...> fmt, Args&&... args) {
 
 }  // namespace clice::logging
 
-#define LOGGING_MESSAGE(name, fmt, ...)                                                            \
+#define LOG_MESSAGE(name, fmt, ...)                                                                \
     if(clice::logging::options.level <= clice::logging::Level::name) {                             \
         clice::logging::name(fmt __VA_OPT__(, ) __VA_ARGS__);                                      \
     }
 
-#define LOGGING_TRACE(fmt, ...) LOGGING_MESSAGE(trace, fmt, __VA_ARGS__)
-#define LOGGING_DEBUG(fmt, ...) LOGGING_MESSAGE(debug, fmt, __VA_ARGS__)
-#define LOGGING_INFO(fmt, ...) LOGGING_MESSAGE(info, fmt, __VA_ARGS__)
-#define LOGGING_WARN(fmt, ...) LOGGING_MESSAGE(warn, fmt, __VA_ARGS__)
-#define LOGGING_ERROR(fmt, ...) LOGGING_MESSAGE(err, fmt, __VA_ARGS__)
-#define LOGGING_FATAL(fmt, ...) clice::logging::critical(fmt __VA_OPT__(, ) __VA_ARGS__);
+#define LOG_TRACE(fmt, ...) LOG_MESSAGE(trace, fmt, __VA_ARGS__)
+#define LOG_DEBUG(fmt, ...) LOG_MESSAGE(debug, fmt, __VA_ARGS__)
+#define LOG_INFO(fmt, ...) LOG_MESSAGE(info, fmt, __VA_ARGS__)
+#define LOG_WARN(fmt, ...) LOG_MESSAGE(warn, fmt, __VA_ARGS__)
+#define LOG_ERROR(fmt, ...) LOG_MESSAGE(err, fmt, __VA_ARGS__)
+#define LOG_FATAL(fmt, ...) clice::logging::critical(fmt __VA_OPT__(, ) __VA_ARGS__);
+
+#define LOG_MESSAGE_RET(ret, name, fmt, ...)                                                       \
+    do {                                                                                           \
+        LOG_MESSAGE(name, fmt, __VA_ARGS__);                                                       \
+        return ret;                                                                                \
+    } while(0);
+
+#define LOG_TRACE_RET(ret, fmt, ...) LOG_MESSAGE_RET(ret, trace, fmt, __VA_ARGS__)
+#define LOG_DEBUG_RET(ret, fmt, ...) LOG_MESSAGE_RET(ret, debug, fmt, __VA_ARGS__)
+#define LOG_INFO_RET(ret, fmt, ...) LOG_MESSAGE_RET(ret, info, fmt, __VA_ARGS__)
+#define LOG_WARN_RET(ret, fmt, ...) LOG_MESSAGE_RET(ret, warn, fmt, __VA_ARGS__)
+#define LOG_ERROR_RET(ret, fmt, ...) LOG_MESSAGE_RET(ret, err, fmt, __VA_ARGS__)