From 1ab8908b3dd2453fdd75af13398ffdef6bba1966 Mon Sep 17 00:00:00 2001 From: ykiko Date: Sun, 8 Mar 2026 21:33:52 +0800 Subject: [PATCH 1/8] refactor: add syntax/scan module and remove server code - Introduce `syntax/scan.h` with lexer-based `scan()` for fast module name and include extraction, and `scan_with_preprocessor()` for full preprocessing-based dependency scanning with hooked VFS - Move `compute_preamble_bound(s)` from `compile/preamble` to `syntax/scan` - Consolidate `PCHInfo`, `ModuleInfo`, `PCMInfo` into `compile/compilation.h`, removing `compile/preamble.h`, `compile/preamble.cpp`, `compile/module.h` - Remove server code (`src/server/`) pending redesign; replace `clice.cc` with placeholder --- CMakeLists.txt | 6 +- src/clice.cc | 185 +------- src/compile/compilation.h | 42 +- src/compile/module.h | 44 -- src/compile/preamble.cpp | 61 --- src/compile/preamble.h | 41 -- src/server/protocol.h | 103 ----- src/server/runtime.h | 32 -- src/server/server.cpp | 914 ------------------------------------- src/server/worker.cpp | 228 --------- src/syntax/scan.cpp | 398 ++++++++++++++++ src/syntax/scan.h | 51 +++ tests/unit/test/tester.cpp | 2 +- 13 files changed, 493 insertions(+), 1614 deletions(-) delete mode 100644 src/compile/module.h delete mode 100644 src/compile/preamble.cpp delete mode 100644 src/compile/preamble.h delete mode 100644 src/server/protocol.h delete mode 100644 src/server/runtime.h delete mode 100644 src/server/server.cpp delete mode 100644 src/server/worker.cpp create mode 100644 src/syntax/scan.cpp create mode 100644 src/syntax/scan.h diff --git a/CMakeLists.txt b/CMakeLists.txt index df9d5a815..f8124be2a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,15 +131,12 @@ add_custom_target(generate_flatbuffers_schema DEPENDS "${GENERATED_HEADER}") # Temporary migration-only build graph. add_library(clice-core STATIC - "${PROJECT_SOURCE_DIR}/src/server/server.cpp" - "${PROJECT_SOURCE_DIR}/src/server/worker.cpp" "${PROJECT_SOURCE_DIR}/src/compile/command.cpp" "${PROJECT_SOURCE_DIR}/src/compile/toolchain.cpp" "${PROJECT_SOURCE_DIR}/src/compile/compilation.cpp" "${PROJECT_SOURCE_DIR}/src/compile/compilation_unit.cpp" "${PROJECT_SOURCE_DIR}/src/compile/diagnostic.cpp" "${PROJECT_SOURCE_DIR}/src/compile/directive.cpp" - "${PROJECT_SOURCE_DIR}/src/compile/preamble.cpp" "${PROJECT_SOURCE_DIR}/src/compile/tidy.cpp" "${PROJECT_SOURCE_DIR}/src/support/doxygen.cpp" "${PROJECT_SOURCE_DIR}/src/support/structed_text.cpp" @@ -147,6 +144,7 @@ add_library(clice-core STATIC "${PROJECT_SOURCE_DIR}/src/support/glob_pattern.cpp" "${PROJECT_SOURCE_DIR}/src/support/logging.cpp" "${PROJECT_SOURCE_DIR}/src/syntax/lexer.cpp" + "${PROJECT_SOURCE_DIR}/src/syntax/scan.cpp" "${PROJECT_SOURCE_DIR}/src/feature/semantic_tokens.cpp" "${PROJECT_SOURCE_DIR}/src/feature/document_links.cpp" "${PROJECT_SOURCE_DIR}/src/feature/document_symbols.cpp" @@ -185,7 +183,7 @@ target_link_libraries(clice-core PUBLIC ) add_executable(clice "${PROJECT_SOURCE_DIR}/src/clice.cc") -target_link_libraries(clice PRIVATE clice::core eventide::deco) +target_link_libraries(clice PRIVATE clice::core) install(TARGETS clice RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) message(STATUS "Copying resource directory for development build") diff --git a/src/clice.cc b/src/clice.cc index d31ab0967..120fcc999 100644 --- a/src/clice.cc +++ b/src/clice.cc @@ -1,186 +1,3 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "eventide/deco/macro.h" -#include "eventide/deco/runtime.h" -#include "eventide/serde/config.h" -#include "server/protocol.h" -#include "server/runtime.h" - -namespace { - -using clice::server::Mode; -using clice::server::Options; - -struct ModeOption { - Mode value = Mode::Pipe; - - auto into(std::string_view text) -> std::optional { - if(text == "pipe") { - value = Mode::Pipe; - return std::nullopt; - } - if(text == "socket") { - value = Mode::Socket; - return std::nullopt; - } - if(text == "worker") { - value = Mode::Worker; - return std::nullopt; - } - return "invalid --mode, expected: pipe|socket|worker"; - } -}; - -struct CliOptions { - DECO_CFG_START(required = false;); - - DecoFlag(names = {"-h", "--help"}; help = "Show this help message and exit"; required = false;) - help = false; - - DecoFlag(names = {std::string_view(clice::server::k_worker_mode)}; - help = "Run as worker process"; - required = false;) - worker_mode = false; - - DecoKVStyled(deco::decl::KVStyle::Joined, names = {"--mode=", "--mode"}; meta_var = "MODE"; - help = "Server mode: pipe|socket|worker"; - required = false;) - mode; - - DecoKVStyled(deco::decl::KVStyle::Joined, names = {"--host=", "--host"}; meta_var = "HOST"; - help = "Socket host (default: 127.0.0.1)"; - required = false;) - host; - - DecoKVStyled(deco::decl::KVStyle::Joined, names = {"--port=", "--port"}; meta_var = "PORT"; - help = "Socket port (default: 50051)"; - required = false) - port; - - DecoKVStyled(deco::decl::KVStyle::Joined, names = {"--worker-count=", "--worker-count"}; - meta_var = "N"; - help = "Worker process count (default: 2)"; - required = false) - worker_count; - - DecoKVStyled(deco::decl::KVStyle::Joined, - names = {"--worker-doc-capacity=", "--worker-doc-capacity"}; - meta_var = "N"; - help = "Per-worker AST cache capacity (default: 32)"; - required = false;) - worker_document_capacity; - - DecoKVStyled(deco::decl::KVStyle::Joined, - names = {"--master-doc-capacity=", "--master-doc-capacity"}; - meta_var = "N"; - help = "Master ownership cache capacity (default: 256)"; - required = false;) - master_document_capacity; - - DECO_CFG_END(); -}; - -auto resolve_self_path(int argc, const char** argv) -> std::string { - if(argc <= 0 || argv == nullptr || argv[0] == nullptr) { - return "clice"; - } - - std::error_code ec; - auto absolute = std::filesystem::absolute(argv[0], ec); - if(ec) { - return std::string(argv[0]); - } - return absolute.string(); -} - -auto build_options(const CliOptions& cli_options, int argc, const char** argv) - -> std::expected { - Options options; - options.self_path = resolve_self_path(argc, argv); - - if(cli_options.mode.value.has_value()) { - options.mode = cli_options.mode->value; - } else if(cli_options.worker_mode.value.value_or(false)) { - options.mode = Mode::Worker; - } - - if(cli_options.host.value.has_value()) { - options.host = *cli_options.host; - } - if(cli_options.port.value.has_value()) { - if(*cli_options.port <= 0) { - return std::unexpected("--port must be a positive integer"); - } - options.port = *cli_options.port; - } - if(cli_options.worker_count.value.has_value()) { - if(*cli_options.worker_count == 0) { - return std::unexpected("--worker-count must be a positive integer"); - } - options.worker_count = *cli_options.worker_count; - } - if(cli_options.worker_document_capacity.value.has_value()) { - if(*cli_options.worker_document_capacity == 0) { - return std::unexpected("--worker-doc-capacity must be a positive integer"); - } - options.worker_document_capacity = *cli_options.worker_document_capacity; - } - if(cli_options.master_document_capacity.value.has_value()) { - if(*cli_options.master_document_capacity == 0) { - return std::unexpected("--master-doc-capacity must be a positive integer"); - } - options.master_document_capacity = *cli_options.master_document_capacity; - } - - return options; -} - -auto print_usage() -> void { - deco::cli::Dispatcher dispatcher("clice [OPTIONS]"); - dispatcher.usage(std::cerr, true); -} - -auto run_with_options(const Options& options) -> int { - switch(options.mode) { - case Mode::Pipe: return clice::server::run_pipe_mode(options); - case Mode::Socket: return clice::server::run_socket_mode(options); - case Mode::Worker: return clice::server::run_worker_mode(options); - } - return 1; -} - -} // namespace - int main(int argc, const char** argv) { - eventide::serde::config::set_field_rename_policy(); - - auto args = deco::util::argvify(argc, argv); - auto parsed = deco::cli::parse(args); - if(!parsed) { - std::fprintf(stderr, "%s\n", parsed.error().message.c_str()); - print_usage(); - return 1; - } - - const auto& cli_options = parsed->options; - if(cli_options.help.value.value_or(false)) { - print_usage(); - return 0; - } - - auto options = build_options(cli_options, argc, argv); - if(!options) { - std::fprintf(stderr, "%s\n", options.error().c_str()); - print_usage(); - return 1; - } - - return run_with_options(*options); + return 0; } diff --git a/src/compile/compilation.h b/src/compile/compilation.h index eacd7b84d..2c3e86b1a 100644 --- a/src/compile/compilation.h +++ b/src/compile/compilation.h @@ -10,8 +10,6 @@ #include #include "compile/compilation_unit.h" -#include "compile/module.h" -#include "compile/preamble.h" #include "support/filesystem.h" #include "llvm/ADT/StringMap.h" @@ -25,6 +23,46 @@ class CodeCompleteConsumer; namespace clice { +struct PCHInfo { + /// The path of the output PCH file. + std::string path; + + /// The building time of this PCH. + std::int64_t mtime; + + /// The content used to build this PCH. + std::string preamble; + + /// All files involved in building this PCH. + std::vector deps; + + /// The command arguments used to build this PCH. + std::vector arguments; +}; + +struct ModuleInfo { + /// Whether this module is an interface unit. + /// i.e. has export module declaration. + bool isInterfaceUnit = false; + + /// Module name. + std::string name; + + /// Dependent modules of this module. + std::vector mods; +}; + +struct PCMInfo : ModuleInfo { + /// PCM file path. + std::string path; + + /// Source file path. + std::string srcPath; + + /// Files involved in building this PCM(not include module). + std::vector deps; +}; + struct CompilationParams { /// The kind of this compilation. CompilationKind kind; diff --git a/src/compile/module.h b/src/compile/module.h deleted file mode 100644 index 57db8f07c..000000000 --- a/src/compile/module.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace clice { - -class CompilationUnit; - -struct CompilationParams; - -struct ModuleInfo { - /// Whether this module is an interface unit. - /// i.e. has export module declaration. - bool isInterfaceUnit = false; - - /// Module name. - std::string name; - - /// Dependent modules of this module. - std::vector mods; -}; - -struct PCMInfo : ModuleInfo { - /// PCM file path. - std::string path; - - /// Source file path. - std::string srcPath; - - /// Files involved in building this PCM(not include module). - std::vector deps; -}; - -/// If input file is module interface unit, return its module name. -/// Otherwise, return an empty string. -std::string scanModuleName(CompilationParams& params); - -/// Run the preprocessor to scan the given module unit to -/// collect its module name and dependencies. -std::expected scanModule(CompilationParams& params); - -} // namespace clice diff --git a/src/compile/preamble.cpp b/src/compile/preamble.cpp deleted file mode 100644 index 10f625ccc..000000000 --- a/src/compile/preamble.cpp +++ /dev/null @@ -1,61 +0,0 @@ -#include "compile/preamble.h" - -#include "support/format.h" -#include "support/logging.h" -#include "syntax/lexer.h" - -namespace clice { - -std::uint32_t compute_preamble_bound(llvm::StringRef content) { - auto result = compute_preamble_bounds(content); - if(result.empty()) { - return 0; - } else { - return result.back(); - } -} - -std::vector compute_preamble_bounds(llvm::StringRef content) { - std::vector result; - - Lexer lexer(content, true, nullptr, false); - - while(true) { - auto token = lexer.advance(); - if(token.is_eof()) { - break; - } - - if(token.is_at_start_of_line) { - if(token.kind == clang::tok::hash) { - /// For preprocessor directive, consume the whole directive. - lexer.advance_until(clang::tok::eod); - auto last = lexer.last(); - - /// Append the token before the eod. - result.push_back(last.range.end); - } else if(token.is_identifier() && token.text(content) == "module") { - /// If we encounter a module keyword at the start of a line, it may be - /// a module declaration or global module fragment. - auto next = lexer.next(); - - if(next.kind == clang::tok::semi) { - /// If next token is `;`, it is a global module fragment. - /// we just continue. - lexer.advance(); - - /// Append it to bounds. - result.push_back(next.range.end); - } else { - break; - } - } else { - break; - } - } - } - - return result; -} - -} // namespace clice diff --git a/src/compile/preamble.h b/src/compile/preamble.h deleted file mode 100644 index fbfb6809a..000000000 --- a/src/compile/preamble.h +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include "llvm/ADT/StringRef.h" - -namespace clice { - -class CompilationUnit; - -struct CompilationParams; - -struct PCHInfo { - /// The path of the output PCH file. - std::string path; - - /// The building time of this PCH. - std::int64_t mtime; - - /// The content used to build this PCH. - std::string preamble; - - /// All files involved in building this PCH. - std::vector deps; - - /// The command arguments used to build this PCH. - std::vector arguments; -}; - -/// Compute the preamble bound of given content. We just -/// run lex until we find first not directive. -std::uint32_t compute_preamble_bound(llvm::StringRef content); - -/// Same as above, but return a group of bounds for chained PCH -/// building. -std::vector compute_preamble_bounds(llvm::StringRef content); - -} // namespace clice diff --git a/src/server/protocol.h b/src/server/protocol.h deleted file mode 100644 index 286efb49f..000000000 --- a/src/server/protocol.h +++ /dev/null @@ -1,103 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "eventide/jsonrpc/protocol.h" -#include "eventide/language/protocol.h" - -namespace clice::server { - -namespace et = eventide; -namespace jsonrpc = et::jsonrpc; -namespace rpc = jsonrpc::protocol; - -constexpr inline std::string_view k_worker_mode = "--worker"; - -struct WorkerCompileParams { - std::string uri; - int version = 0; - std::string text; -}; - -struct WorkerCompileResult { - std::string uri; - int version = 0; - std::vector diagnostics; -}; - -struct WorkerHoverParams { - std::string uri; - int version = 0; - std::string text; - int line = 0; - int character = 0; -}; - -struct WorkerHoverResult { - rpc::RequestTraits::Result result = std::nullopt; -}; - -struct WorkerCompletionParams { - std::string uri; - int version = 0; - std::string text; - int line = 0; - int character = 0; -}; - -struct WorkerCompletionResult { - rpc::RequestTraits::Result result = nullptr; -}; - -struct WorkerSignatureHelpParams { - std::string uri; - int version = 0; - std::string text; - int line = 0; - int character = 0; -}; - -struct WorkerSignatureHelpResult { - rpc::RequestTraits::Result result = std::nullopt; -}; - -struct WorkerEvictParams { - std::string uri; -}; - -} // namespace clice::server - -namespace eventide::jsonrpc::protocol { - -template <> -struct RequestTraits { - using Result = clice::server::WorkerCompileResult; - constexpr inline static std::string_view method = "clice/worker/compile"; -}; - -template <> -struct RequestTraits { - using Result = clice::server::WorkerHoverResult; - constexpr inline static std::string_view method = "clice/worker/hover"; -}; - -template <> -struct RequestTraits { - using Result = clice::server::WorkerCompletionResult; - constexpr inline static std::string_view method = "clice/worker/completion"; -}; - -template <> -struct RequestTraits { - using Result = clice::server::WorkerSignatureHelpResult; - constexpr inline static std::string_view method = "clice/worker/signatureHelp"; -}; - -template <> -struct NotificationTraits { - constexpr inline static std::string_view method = "clice/worker/evict"; -}; - -} // namespace eventide::jsonrpc::protocol diff --git a/src/server/runtime.h b/src/server/runtime.h deleted file mode 100644 index ec22461c9..000000000 --- a/src/server/runtime.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include -#include - -namespace clice::server { - -enum class Mode { - Pipe, - Socket, - Worker, -}; - -struct Options { - Mode mode = Mode::Pipe; - std::string host = "127.0.0.1"; - int port = 50051; - - std::string self_path; - - std::size_t worker_count = 2; - std::size_t worker_document_capacity = 32; - std::size_t master_document_capacity = 256; -}; - -auto run_pipe_mode(const Options& options) -> int; - -auto run_socket_mode(const Options& options) -> int; - -auto run_worker_mode(const Options& options) -> int; - -} // namespace clice::server diff --git a/src/server/server.cpp b/src/server/server.cpp deleted file mode 100644 index b404a0c19..000000000 --- a/src/server/server.cpp +++ /dev/null @@ -1,914 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "eventide/async/loop.h" -#include "eventide/async/process.h" -#include "eventide/async/stream.h" -#include "eventide/jsonrpc/peer.h" -#include "eventide/jsonrpc/transport.h" -#include "eventide/language/protocol.h" -#include "server/protocol.h" -#include "server/runtime.h" - -namespace clice::server { - -namespace { - -auto print_error_message(std::string_view prefix, std::string_view message) -> void { - std::fprintf(stderr, - "%.*s%.*s\n", - static_cast(prefix.size()), - prefix.data(), - static_cast(message.size()), - message.data()); -} - -auto make_initialize_result() -> rpc::InitializeResult { - rpc::InitializeResult result; - - rpc::TextDocumentSyncOptions sync; - sync.open_close = true; - sync.change = rpc::TextDocumentSyncKind::Full; - sync.save = true; - - rpc::CompletionOptions completion; - completion.resolve_provider = false; - - rpc::SignatureHelpOptions signature_help; - signature_help.trigger_characters = std::vector{"(", ","}; - - result.capabilities.text_document_sync = std::move(sync); - result.capabilities.hover_provider = true; - result.capabilities.completion_provider = std::move(completion); - result.capabilities.signature_help_provider = std::move(signature_help); - result.server_info = rpc::ServerInfo{ - .name = "clice", - .version = std::string("0.1.0"), - }; - - return result; -} - -auto make_publish_diagnostics(std::string uri, - std::optional version, - std::vector diagnostics = {}) - -> rpc::PublishDiagnosticsParams { - rpc::PublishDiagnosticsParams payload{ - .uri = std::move(uri), - .diagnostics = std::move(diagnostics), - }; - if(version) { - payload.version = static_cast(*version); - } - return payload; -} - -class WorkerPool { -public: - WorkerPool(et::event_loop& loop, const Options& options) : loop(loop), options(options) {} - - auto start() -> std::expected { - if(started) { - return {}; - } - started = true; - - if(options.worker_count == 0) { - return std::unexpected("worker_count cannot be 0"); - } - if(options.self_path.empty()) { - return std::unexpected("worker executable path is empty"); - } - - workers.reserve(options.worker_count); - for(std::size_t index = 0; index < options.worker_count; ++index) { - auto spawned = spawn_worker(); - if(!spawned) { - return std::unexpected(std::move(spawned.error())); - } - workers.push_back(std::move(*spawned)); - } - - return {}; - } - - auto compile(WorkerCompileParams params) -> et::task> { - if(workers.empty()) { - co_return std::unexpected("worker pool is empty"); - } - - const auto worker_index = assign_worker(params.uri); - auto response = co_await workers[worker_index]->send_request(params); - if(response) { - co_return response; - } - - auto restarted = restart_worker(worker_index); - if(!restarted) { - co_return std::unexpected("worker request failed: " + response.error() + - "; worker restart failed: " + restarted.error()); - } - - co_return co_await workers[worker_index]->send_request(params); - } - - auto hover(WorkerHoverParams params) -> et::task> { - if(workers.empty()) { - co_return std::unexpected("worker pool is empty"); - } - - const auto worker_index = assign_worker(params.uri); - auto response = co_await workers[worker_index]->send_request(params); - if(response) { - co_return response; - } - - auto restarted = restart_worker(worker_index); - if(!restarted) { - co_return std::unexpected("worker request failed: " + response.error() + - "; worker restart failed: " + restarted.error()); - } - - co_return co_await workers[worker_index]->send_request(params); - } - - auto completion(WorkerCompletionParams params) - -> et::task> { - if(workers.empty()) { - co_return std::unexpected("worker pool is empty"); - } - - const auto worker_index = assign_worker(params.uri); - auto response = co_await workers[worker_index]->send_request(params); - if(response) { - co_return response; - } - - auto restarted = restart_worker(worker_index); - if(!restarted) { - co_return std::unexpected("worker request failed: " + response.error() + - "; worker restart failed: " + restarted.error()); - } - - co_return co_await workers[worker_index]->send_request(params); - } - - auto signature_help(WorkerSignatureHelpParams params) - -> et::task> { - if(workers.empty()) { - co_return std::unexpected("worker pool is empty"); - } - - const auto worker_index = assign_worker(params.uri); - auto response = co_await workers[worker_index]->send_request(params); - if(response) { - co_return response; - } - - auto restarted = restart_worker(worker_index); - if(!restarted) { - co_return std::unexpected("worker request failed: " + response.error() + - "; worker restart failed: " + restarted.error()); - } - - co_return co_await workers[worker_index]->send_request(params); - } - - void release_document(std::string_view uri) { - auto key = std::string(uri); - auto owner_iter = owner.find(key); - if(owner_iter != owner.end()) { - auto worker_id = owner_iter->second; - auto& worker = workers[worker_id]; - if(worker.owned_documents > 0) { - worker.owned_documents -= 1; - } - evict_from_worker(worker_id, key); - owner.erase(owner_iter); - } - - auto lru_iter = owner_lru_index.find(key); - if(lru_iter != owner_lru_index.end()) { - owner_lru.erase(lru_iter->second); - owner_lru_index.erase(lru_iter); - } - } - - auto shutdown() -> et::task<> { - if(!started) { - co_return; - } - started = false; - - for(auto& worker: workers) { - if(worker.peer) { - auto status = worker.peer->close_output(); - (void)status; - } - } - - for(auto& worker: workers) { - auto waited = co_await worker.process.wait(); - if(waited) { - continue; - } - - auto kill_status = worker.process.kill(SIGTERM); - (void)kill_status; - auto waited_after_kill = co_await worker.process.wait(); - (void)waited_after_kill; - } - - workers.clear(); - owner.clear(); - owner_lru.clear(); - owner_lru_index.clear(); - } - -private: - struct WorkerClient { - et::process process; - std::shared_ptr peer; - std::size_t owned_documents = 0; - - auto operator->() -> jsonrpc::Peer* { - return peer.get(); - } - }; - - static auto run_worker_peer(std::shared_ptr peer) -> et::task<> { - co_await peer->run(); - } - - auto spawn_worker() -> std::expected { - et::process::options process_options; - process_options.file = options.self_path; - process_options.args = { - options.self_path, - std::string(k_worker_mode), - "--worker-doc-capacity=" + std::to_string(options.worker_document_capacity), - }; - process_options.streams = { - et::process::stdio::pipe(true, false), - et::process::stdio::pipe(false, true), - et::process::stdio::inherit(), - }; - - auto spawned = et::process::spawn(process_options, loop); - if(!spawned) { - return std::unexpected(std::string(spawned.error().message())); - } - - auto transport = std::make_unique(std::move(spawned->stdout_pipe), - std::move(spawned->stdin_pipe)); - auto peer = std::make_shared(loop, std::move(transport)); - loop.schedule(run_worker_peer(peer)); - - WorkerClient client; - client.process = std::move(spawned->proc); - client.peer = std::move(peer); - return client; - } - - auto restart_worker(std::size_t worker_index) -> std::expected { - if(worker_index >= workers.size()) { - return std::unexpected("worker index out of range"); - } - - auto replacement = spawn_worker(); - if(!replacement) { - return std::unexpected(std::move(replacement.error())); - } - - auto old_process = std::move(workers[worker_index].process); - auto old_peer = std::move(workers[worker_index].peer); - - if(old_peer) { - auto status = old_peer->close_output(); - (void)status; - } - - if(old_process.pid() > 0) { - auto kill_status = old_process.kill(SIGTERM); - (void)kill_status; - loop.schedule(reap_worker_process(std::move(old_process))); - } - - workers[worker_index].process = std::move(replacement->process); - workers[worker_index].peer = std::move(replacement->peer); - return {}; - } - - auto reap_worker_process(et::process process) -> et::task<> { - auto waited = co_await process.wait(); - (void)waited; - } - - auto assign_worker(std::string_view uri) -> std::size_t { - auto key = std::string(uri); - auto owner_iter = owner.find(key); - if(owner_iter != owner.end()) { - touch_owner_lru(key); - return owner_iter->second; - } - - shrink_owner(); - const auto selected = pick_worker(); - owner.emplace(key, selected); - workers[selected].owned_documents += 1; - touch_owner_lru(key); - return selected; - } - - void touch_owner_lru(const std::string& key) { - auto lru_iter = owner_lru_index.find(key); - if(lru_iter != owner_lru_index.end()) { - owner_lru.splice(owner_lru.begin(), owner_lru, lru_iter->second); - lru_iter->second = owner_lru.begin(); - return; - } - - owner_lru.push_front(key); - owner_lru_index.emplace(key, owner_lru.begin()); - } - - void shrink_owner() { - while(owner.size() >= options.master_document_capacity && !owner_lru.empty()) { - auto victim = std::move(owner_lru.back()); - owner_lru.pop_back(); - owner_lru_index.erase(victim); - - auto owner_iter = owner.find(victim); - if(owner_iter == owner.end()) { - continue; - } - - auto& worker = workers[owner_iter->second]; - auto worker_id = owner_iter->second; - if(worker.owned_documents > 0) { - worker.owned_documents -= 1; - } - evict_from_worker(worker_id, victim); - owner.erase(owner_iter); - } - } - - auto pick_worker() const -> std::size_t { - if(workers.empty()) { - return 0; - } - - std::size_t selected = 0; - for(std::size_t index = 1; index < workers.size(); ++index) { - if(workers[index].owned_documents < workers[selected].owned_documents) { - selected = index; - } - } - return selected; - } - - void evict_from_worker(std::size_t worker_id, const std::string& uri) { - if(worker_id >= workers.size()) { - return; - } - - auto status = workers[worker_id]->send_notification(WorkerEvictParams{ - .uri = uri, - }); - (void)status; - } - -private: - et::event_loop& loop; - const Options& options; - bool started = false; - - std::vector workers; - std::unordered_map owner; - std::list owner_lru; - std::unordered_map::iterator> owner_lru_index; -}; - -class MasterServer { -public: - MasterServer(et::event_loop& loop, jsonrpc::Peer& peer, const Options& options) : - loop(loop), peer(peer), workers(loop, options) { - register_callbacks(); - } - - auto start() -> std::expected { - return workers.start(); - } - - [[nodiscard]] auto exit_code() const -> int { - return requested_exit_code; - } - -private: - struct DocumentState { - int version = 0; - std::string text; - std::uint64_t generation = 0; - bool build_running = false; - bool build_requested = false; - }; - - struct HoverRequestSnapshot { - std::string uri; - int version = 0; - std::uint64_t generation = 0; - std::string text; - int line = 0; - int character = 0; - }; - - using CompletionRequestSnapshot = HoverRequestSnapshot; - using SignatureHelpRequestSnapshot = HoverRequestSnapshot; - - void register_callbacks() { - peer.on_request( - [this](jsonrpc::RequestContext& context, const rpc::InitializeParams& params) - -> jsonrpc::RequestResult { - return on_initialize(context, params); - }); - - peer.on_request([this](jsonrpc::RequestContext& context, const rpc::ShutdownParams& params) - -> jsonrpc::RequestResult { - return on_shutdown(context, params); - }); - - peer.on_request( - [this](jsonrpc::RequestContext& context, - const rpc::HoverParams& params) -> jsonrpc::RequestResult { - return on_hover(context, params); - }); - - peer.on_request( - [this](jsonrpc::RequestContext& context, const rpc::CompletionParams& params) - -> jsonrpc::RequestResult { - return on_completion(context, params); - }); - - peer.on_request( - [this](jsonrpc::RequestContext& context, const rpc::SignatureHelpParams& params) - -> jsonrpc::RequestResult { - return on_signature_help(context, params); - }); - - peer.on_notification([this](const rpc::InitializedParams&) { - if(!initialize_request || shutdown_request) { - return; - } - initialized_notification = true; - }); - - peer.on_notification([this](const rpc::ExitParams&) { - if(exiting) { - return; - } - exiting = true; - requested_exit_code = shutdown_request ? 0 : 1; - loop.schedule(stop()); - }); - - peer.on_notification( - [this](const rpc::DidOpenTextDocumentParams& params) { on_did_open(params); }); - - peer.on_notification( - [this](const rpc::DidChangeTextDocumentParams& params) { on_did_change(params); }); - - peer.on_notification( - [this](const rpc::DidSaveTextDocumentParams& params) { on_did_save(params); }); - - peer.on_notification( - [this](const rpc::DidCloseTextDocumentParams& params) { on_did_close(params); }); - } - - auto on_initialize(jsonrpc::RequestContext&, const rpc::InitializeParams&) - -> jsonrpc::RequestResult { - if(initialize_request) { - co_return std::unexpected("initialize can only be requested once"); - } - if(shutdown_request) { - co_return std::unexpected("server is shutting down"); - } - - initialize_request = true; - co_return make_initialize_result(); - } - - auto on_shutdown(jsonrpc::RequestContext&, const rpc::ShutdownParams&) - -> jsonrpc::RequestResult { - if(!initialize_request) { - co_return std::unexpected("server is not initialized"); - } - if(shutdown_request) { - co_return std::unexpected("shutdown has already been requested"); - } - - shutdown_request = true; - co_return nullptr; - } - - auto on_hover(jsonrpc::RequestContext&, const rpc::HoverParams& params) - -> jsonrpc::RequestResult { - if(!initialize_request) { - co_return std::unexpected("server is not initialized"); - } - if(shutdown_request) { - co_return std::unexpected("server is shutting down"); - } - - const auto& tdpp = params.text_document_position_params; - const auto& uri = tdpp.text_document.uri; - auto line = static_cast(tdpp.position.line); - auto character = static_cast(tdpp.position.character); - - auto doc_iter = documents.find(uri); - if(doc_iter == documents.end()) { - co_return std::nullopt; - } - - auto snapshot = HoverRequestSnapshot{ - .uri = std::string(uri), - .version = doc_iter->second.version, - .generation = doc_iter->second.generation, - .text = doc_iter->second.text, - .line = line, - .character = character, - }; - - co_return co_await run_hover(std::move(snapshot)); - } - - auto on_completion(jsonrpc::RequestContext&, const rpc::CompletionParams& params) - -> jsonrpc::RequestResult { - if(!initialize_request) { - co_return std::unexpected("server is not initialized"); - } - if(shutdown_request) { - co_return std::unexpected("server is shutting down"); - } - - const auto& tdpp = params.text_document_position_params; - const auto& uri = tdpp.text_document.uri; - auto line = static_cast(tdpp.position.line); - auto character = static_cast(tdpp.position.character); - - auto doc_iter = documents.find(uri); - if(doc_iter == documents.end()) { - co_return nullptr; - } - - auto snapshot = CompletionRequestSnapshot{ - .uri = std::string(uri), - .version = doc_iter->second.version, - .generation = doc_iter->second.generation, - .text = doc_iter->second.text, - .line = line, - .character = character, - }; - - co_return co_await run_completion(std::move(snapshot)); - } - - auto on_signature_help(jsonrpc::RequestContext&, const rpc::SignatureHelpParams& params) - -> jsonrpc::RequestResult { - if(!initialize_request) { - co_return std::unexpected("server is not initialized"); - } - if(shutdown_request) { - co_return std::unexpected("server is shutting down"); - } - - const auto& tdpp = params.text_document_position_params; - const auto& uri = tdpp.text_document.uri; - auto line = static_cast(tdpp.position.line); - auto character = static_cast(tdpp.position.character); - - auto doc_iter = documents.find(uri); - if(doc_iter == documents.end()) { - co_return std::nullopt; - } - - auto snapshot = SignatureHelpRequestSnapshot{ - .uri = std::string(uri), - .version = doc_iter->second.version, - .generation = doc_iter->second.generation, - .text = doc_iter->second.text, - .line = line, - .character = character, - }; - - co_return co_await run_signature_help(std::move(snapshot)); - } - - void on_did_open(const rpc::DidOpenTextDocumentParams& params) { - if(!accept_document_notifications()) { - return; - } - - auto& document = documents[params.text_document.uri]; - document.version = static_cast(params.text_document.version); - document.text = params.text_document.text; - document.generation += 1; - - schedule_build(params.text_document.uri); - } - - void on_did_change(const rpc::DidChangeTextDocumentParams& params) { - if(!accept_document_notifications()) { - return; - } - - std::optional latest_text; - for(const auto& change: params.content_changes) { - if(auto whole = std::get_if(&change)) { - latest_text = whole->text; - continue; - } - if(auto partial = std::get_if(&change)) { - latest_text = partial->text; - } - } - if(!latest_text) { - return; - } - - auto& document = documents[params.text_document.uri]; - document.version = static_cast(params.text_document.version); - document.text = std::move(*latest_text); - document.generation += 1; - - schedule_build(params.text_document.uri); - } - - void on_did_save(const rpc::DidSaveTextDocumentParams& params) { - if(!accept_document_notifications()) { - return; - } - - auto doc_iter = documents.find(params.text_document.uri); - if(doc_iter == documents.end()) { - return; - } - - auto& document = doc_iter->second; - if(params.text) { - document.text = *params.text; - } - document.generation += 1; - schedule_build(params.text_document.uri); - } - - void on_did_close(const rpc::DidCloseTextDocumentParams& params) { - if(!accept_document_notifications()) { - return; - } - - auto uri = std::string(params.text_document.uri); - documents.erase(uri); - workers.release_document(uri); - - auto status = - peer.send_notification(make_publish_diagnostics(std::move(uri), std::nullopt)); - (void)status; - } - - auto run_hover(HoverRequestSnapshot snapshot) -> jsonrpc::RequestResult { - WorkerHoverParams params{ - .uri = snapshot.uri, - .version = snapshot.version, - .text = snapshot.text, - .line = snapshot.line, - .character = snapshot.character, - }; - - auto hover_result = co_await workers.hover(std::move(params)); - if(!hover_result) { - co_return std::nullopt; - } - - auto latest_iter = documents.find(snapshot.uri); - if(latest_iter == documents.end() || - latest_iter->second.generation != snapshot.generation) { - co_return std::nullopt; - } - - co_return std::move(hover_result->result); - } - - auto run_completion(CompletionRequestSnapshot snapshot) - -> jsonrpc::RequestResult { - WorkerCompletionParams params{ - .uri = snapshot.uri, - .version = snapshot.version, - .text = snapshot.text, - .line = snapshot.line, - .character = snapshot.character, - }; - - auto completion_result = co_await workers.completion(std::move(params)); - if(!completion_result) { - co_return nullptr; - } - - auto latest_iter = documents.find(snapshot.uri); - if(latest_iter == documents.end() || - latest_iter->second.generation != snapshot.generation) { - co_return nullptr; - } - - co_return std::move(completion_result->result); - } - - auto run_signature_help(SignatureHelpRequestSnapshot snapshot) - -> jsonrpc::RequestResult { - WorkerSignatureHelpParams params{ - .uri = snapshot.uri, - .version = snapshot.version, - .text = snapshot.text, - .line = snapshot.line, - .character = snapshot.character, - }; - - auto signature_help_result = co_await workers.signature_help(std::move(params)); - if(!signature_help_result) { - co_return std::nullopt; - } - - auto latest_iter = documents.find(snapshot.uri); - if(latest_iter == documents.end() || - latest_iter->second.generation != snapshot.generation) { - co_return std::nullopt; - } - - co_return std::move(signature_help_result->result); - } - - [[nodiscard]] auto accept_document_notifications() const -> bool { - return initialize_request && !shutdown_request; - } - - void schedule_build(std::string uri) { - auto doc_iter = documents.find(uri); - if(doc_iter == documents.end()) { - return; - } - - auto& document = doc_iter->second; - document.build_requested = true; - - if(document.build_running) { - return; - } - document.build_running = true; - - loop.schedule(run_build_drain(std::move(uri))); - } - - auto run_build_drain(std::string uri) -> et::task<> { - while(true) { - auto doc_iter = documents.find(uri); - if(doc_iter == documents.end()) { - co_return; - } - - auto& document = doc_iter->second; - if(!document.build_requested) { - document.build_running = false; - co_return; - } - - document.build_requested = false; - const auto generation = document.generation; - WorkerCompileParams params{ - .uri = uri, - .version = document.version, - .text = document.text, - }; - - auto compile_result = co_await workers.compile(std::move(params)); - if(!compile_result) { - continue; - } - - auto latest_iter = documents.find(uri); - if(latest_iter == documents.end()) { - co_return; - } - if(latest_iter->second.generation != generation) { - continue; - } - - auto status = peer.send_notification( - make_publish_diagnostics(compile_result->uri, - compile_result->version, - std::move(compile_result->diagnostics))); - (void)status; - } - } - - auto stop() -> et::task<> { - if(stopping) { - co_return; - } - stopping = true; - - co_await workers.shutdown(); - loop.stop(); - } - -private: - et::event_loop& loop; - jsonrpc::Peer& peer; - WorkerPool workers; - - bool initialize_request = false; - bool initialized_notification = false; - bool shutdown_request = false; - bool exiting = false; - bool stopping = false; - int requested_exit_code = 0; - - std::unordered_map documents; -}; - -auto run_master_session(et::event_loop& loop, - std::unique_ptr transport, - const Options& options) -> int { - jsonrpc::Peer peer(loop, std::move(transport)); - MasterServer server(loop, peer, options); - - auto started = server.start(); - if(!started) { - std::fprintf(stderr, "failed to start worker pool: %s\n", started.error().c_str()); - return 1; - } - - loop.schedule(peer.run()); - auto loop_status = loop.run(); - if(loop_status != 0) { - return loop_status; - } - return server.exit_code(); -} - -} // namespace - -auto run_pipe_mode(const Options& options) -> int { - et::event_loop loop; - auto stdio = jsonrpc::StreamTransport::open_stdio(loop); - if(!stdio) { - std::fprintf(stderr, "failed to open stdio transport: %s\n", stdio.error().c_str()); - return 1; - } - - std::unique_ptr transport = std::move(*stdio); - return run_master_session(loop, std::move(transport), options); -} - -auto run_socket_mode(const Options& options) -> int { - et::event_loop loop; - - auto listener_result = et::tcp_socket::listen(options.host, options.port, {}, loop); - if(!listener_result) { - print_error_message("failed to listen: ", listener_result.error().message()); - return 1; - } - - auto listener = std::move(*listener_result); - auto accept_task = listener.accept(); - loop.schedule(accept_task); - auto loop_status = loop.run(); - if(loop_status != 0) { - return loop_status; - } - - auto accepted = accept_task.value(); - listener = {}; - if(!accepted || !accepted->has_value()) { - if(accepted && !accepted->has_value()) { - print_error_message("failed to accept connection: ", accepted->error().message()); - } else { - std::fprintf(stderr, "failed to accept connection: unknown error\n"); - } - return 1; - } - - auto socket = std::move(**accepted); - auto stream = et::stream(std::move(socket)); - auto transport = std::make_unique(std::move(stream)); - return run_master_session(loop, std::move(transport), options); -} - -} // namespace clice::server diff --git a/src/server/worker.cpp b/src/server/worker.cpp deleted file mode 100644 index 0f68ec09a..000000000 --- a/src/server/worker.cpp +++ /dev/null @@ -1,228 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "eventide/async/loop.h" -#include "eventide/jsonrpc/peer.h" -#include "eventide/jsonrpc/transport.h" -#include "server/protocol.h" -#include "server/runtime.h" - -namespace clice::server { - -namespace { - -auto make_hover_result(std::string_view uri, int version, int line, int character) - -> rpc::RequestTraits::Result { - rpc::Hover hover; - hover.contents = rpc::MarkupContent{ - .kind = rpc::MarkupKind::Plaintext, - .value = "clice hover snapshot: uri=" + std::string(uri) + - ", version=" + std::to_string(version) + ", line=" + std::to_string(line) + - ", character=" + std::to_string(character), - }; - return hover; -} - -auto make_completion_result(std::string_view uri, int version, int line, int character) - -> rpc::RequestTraits::Result { - rpc::CompletionItem item; - item.label = "clice::completion"; - item.kind = rpc::CompletionItemKind::Text; - item.detail = "uri=" + std::string(uri) + ", version=" + std::to_string(version) + - ", line=" + std::to_string(line) + ", character=" + std::to_string(character); - - rpc::CompletionList completion{ - .is_incomplete = false, - .items = {std::move(item)}, - }; - return completion; -} - -auto make_signature_help_result(std::string_view uri, int version, int line, int character) - -> rpc::RequestTraits::Result { - rpc::ParameterInformation parameter; - parameter.label = std::string("int value"); - - rpc::SignatureInformation signature; - signature.label = "void clice_signature(" + std::string(uri) + ", " + std::to_string(version) + - ", " + std::to_string(line) + ", " + std::to_string(character) + ")"; - signature.parameters = std::vector{std::move(parameter)}; - - rpc::SignatureHelp help{ - .signatures = {std::move(signature)}, - .active_signature = static_cast(0), - }; - return help; -} - -class WorkerRuntime { -public: - WorkerRuntime(jsonrpc::Peer& peer, std::size_t document_capacity) : - peer(peer), document_capacity(std::max(1, document_capacity)) { - register_callbacks(); - } - -private: - struct CachedDocument { - int version = 0; - std::string text; - }; - - void register_callbacks() { - peer.on_request([this](jsonrpc::RequestContext& context, const WorkerCompileParams& params) - -> jsonrpc::RequestResult { - return on_compile(context, params); - }); - - peer.on_request([this](jsonrpc::RequestContext& context, const WorkerHoverParams& params) - -> jsonrpc::RequestResult { - return on_hover(context, params); - }); - - peer.on_request( - [this](jsonrpc::RequestContext& context, const WorkerCompletionParams& params) - -> jsonrpc::RequestResult { - return on_completion(context, params); - }); - - peer.on_request( - [this](jsonrpc::RequestContext& context, const WorkerSignatureHelpParams& params) - -> jsonrpc::RequestResult { - return on_signature_help(context, params); - }); - - peer.on_notification( - [this](const WorkerEvictParams& params) { evict_document(params.uri); }); - } - - auto on_compile(jsonrpc::RequestContext& context, const WorkerCompileParams& params) - -> jsonrpc::RequestResult { - upsert_document(params.uri, params.version, params.text); - (void)context; - - co_return WorkerCompileResult{ - .uri = params.uri, - .version = params.version, - .diagnostics = {}, - }; - } - - auto on_hover(jsonrpc::RequestContext& context, const WorkerHoverParams& params) - -> jsonrpc::RequestResult { - upsert_document(params.uri, params.version, params.text); - (void)context; - - co_return WorkerHoverResult{ - .result = make_hover_result(params.uri, params.version, params.line, params.character), - }; - } - - auto on_completion(jsonrpc::RequestContext& context, const WorkerCompletionParams& params) - -> jsonrpc::RequestResult { - upsert_document(params.uri, params.version, params.text); - (void)context; - - co_return WorkerCompletionResult{ - .result = - make_completion_result(params.uri, params.version, params.line, params.character), - }; - } - - auto on_signature_help(jsonrpc::RequestContext& context, - const WorkerSignatureHelpParams& params) - -> jsonrpc::RequestResult { - upsert_document(params.uri, params.version, params.text); - (void)context; - - co_return WorkerSignatureHelpResult{ - .result = make_signature_help_result(params.uri, - params.version, - params.line, - params.character), - }; - } - - void upsert_document(std::string_view uri, int version, std::string_view text) { - auto key = std::string(uri); - auto doc_iter = documents.find(key); - if(doc_iter == documents.end()) { - documents.emplace(key, - CachedDocument{ - .version = version, - .text = std::string(text), - }); - } else { - doc_iter->second.version = version; - doc_iter->second.text.assign(text); - } - - touch_document(key); - shrink_to_capacity(); - } - - void touch_document(const std::string& key) { - auto lru_iter = lru_index.find(key); - if(lru_iter != lru_index.end()) { - lru.splice(lru.begin(), lru, lru_iter->second); - lru_iter->second = lru.begin(); - return; - } - - lru.push_front(key); - lru_index.emplace(key, lru.begin()); - } - - void shrink_to_capacity() { - while(documents.size() > document_capacity && !lru.empty()) { - auto victim = std::move(lru.back()); - lru.pop_back(); - lru_index.erase(victim); - documents.erase(victim); - } - } - - void evict_document(std::string_view uri) { - auto key = std::string(uri); - documents.erase(key); - - auto lru_iter = lru_index.find(key); - if(lru_iter != lru_index.end()) { - lru.erase(lru_iter->second); - lru_index.erase(lru_iter); - } - } - -private: - jsonrpc::Peer& peer; - std::size_t document_capacity = 1; - std::unordered_map documents; - std::list lru; - std::unordered_map::iterator> lru_index; -}; - -} // namespace - -auto run_worker_mode(const Options& options) -> int { - et::event_loop loop; - auto stdio = jsonrpc::StreamTransport::open_stdio(loop); - if(!stdio) { - std::fprintf(stderr, "failed to open worker stdio transport: %s\n", stdio.error().c_str()); - return 1; - } - - jsonrpc::Peer peer(loop, std::move(*stdio)); - WorkerRuntime runtime(peer, options.worker_document_capacity); - (void)runtime; - - loop.schedule(peer.run()); - return loop.run(); -} - -} // namespace clice::server diff --git a/src/syntax/scan.cpp b/src/syntax/scan.cpp new file mode 100644 index 000000000..53cf597e4 --- /dev/null +++ b/src/syntax/scan.cpp @@ -0,0 +1,398 @@ +#include "syntax/scan.h" + +#include "syntax/lexer.h" + +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/MemoryBuffer.h" +#include "clang/Basic/DiagnosticOptions.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Lex/PPCallbacks.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Tooling/CompilationDatabase.h" + +namespace clice { + +ScanResult scan(llvm::StringRef content) { + ScanResult result; + + Lexer lexer(content, true, nullptr, false); + + int conditional_depth = 0; + + while(true) { + auto token = lexer.advance(); + if(token.is_eof()) { + break; + } + + if(!token.is_at_start_of_line) { + continue; + } + + if(token.kind == clang::tok::hash) { + auto directive = lexer.advance(); + if(directive.is_eof()) { + break; + } + + auto spelling = directive.text(content); + + if(spelling == "if" || spelling == "ifdef" || spelling == "ifndef") { + conditional_depth++; + lexer.advance_until(clang::tok::eod); + } else if(spelling == "endif") { + if(conditional_depth > 0) { + conditional_depth--; + } + lexer.advance_until(clang::tok::eod); + } else if(spelling == "elif" || spelling == "elifdef" || spelling == "elifndef" || + spelling == "else") { + lexer.advance_until(clang::tok::eod); + } else if(spelling == "include") { + auto header = lexer.advance(); + if(header.is_header_name()) { + auto name = header.text(content); + // Strip <> or "" delimiters + result.includes.emplace_back(name.substr(1, name.size() - 2)); + } + lexer.advance_until(clang::tok::eod); + } else { + lexer.advance_until(clang::tok::eod); + } + } else if(token.is_identifier()) { + auto spelling = token.text(content); + bool is_export = false; + + if(spelling == "export") { + is_export = true; + auto next = lexer.advance(); + if(next.is_eof()) { + break; + } + if(!next.is_identifier() || next.text(content) != "module") { + continue; + } + spelling = "module"; + } + + if(spelling == "module") { + auto next = lexer.next(); + if(next.is_eof()) { + break; + } + + // `module;` is global module fragment, skip it + if(next.kind == clang::tok::semi) { + lexer.advance(); + continue; + } + + // Module declaration inside conditional directive + if(conditional_depth > 0) { + result.need_preprocess = true; + return result; + } + + // Collect module name: identifiers, '.', ':' + std::string module_name; + while(true) { + auto tok = lexer.advance(); + if(tok.is_eof() || tok.kind == clang::tok::semi || tok.is_eod()) { + break; + } + if(tok.is_identifier()) { + module_name += tok.text(content); + } else if(tok.kind == clang::tok::period) { + module_name += '.'; + } else if(tok.kind == clang::tok::colon) { + module_name += ':'; + } + } + + result.module_name = std::move(module_name); + result.is_interface_unit = is_export; + } + } + } + + return result; +} + +namespace { + +class InMemoryFile : public llvm::vfs::File { +public: + explicit InMemoryFile(std::unique_ptr buffer, llvm::vfs::Status status) : + buffer(std::move(buffer)), file_status(std::move(status)) {} + + llvm::ErrorOr status() override { + return file_status; + } + + llvm::ErrorOr> + getBuffer(const llvm::Twine&, int64_t, bool, bool) override { + return llvm::MemoryBuffer::getMemBufferCopy(buffer->getBuffer(), + buffer->getBufferIdentifier()); + } + + std::error_code close() override { + return {}; + } + +private: + std::unique_ptr buffer; + llvm::vfs::Status file_status; +}; + +/// Strip file content to only #include lines for fast preprocessing. +std::string strip_to_includes(llvm::StringRef content) { + std::string result; + + Lexer lexer(content, true, nullptr, false); + + while(true) { + auto token = lexer.advance(); + if(token.is_eof()) { + break; + } + + if(token.is_at_start_of_line && token.kind == clang::tok::hash) { + auto directive = lexer.advance(); + if(directive.is_eof()) { + break; + } + + auto spelling = directive.text(content); + if(spelling == "include") { + // Keep the whole directive line + auto start = token.range.begin; + auto eod = lexer.advance_until(clang::tok::eod); + auto end = eod.range.begin; + result += content.substr(start, end - start); + result += '\n'; + } else { + lexer.advance_until(clang::tok::eod); + } + } + } + + return result; +} + +class IncludeOnlyVFS : public llvm::vfs::ProxyFileSystem { +public: + explicit IncludeOnlyVFS(llvm::IntrusiveRefCntPtr base) : + ProxyFileSystem(std::move(base)) {} + + llvm::ErrorOr> + openFileForRead(const llvm::Twine& path) override { + llvm::SmallString<256> storage; + llvm::StringRef path_str = path.toStringRef(storage); + + // If already visited, return empty buffer to cut include chains + if(!visited.insert(path_str).second) { + auto status_or = getUnderlyingFS().status(path); + if(!status_or) { + return status_or.getError(); + } + auto empty = llvm::MemoryBuffer::getMemBuffer("", path_str); + return std::make_unique(std::move(empty), *status_or); + } + + // First visit: read real file, strip to includes only + auto file = getUnderlyingFS().openFileForRead(path); + if(!file) { + return file; + } + + auto status_or = (*file)->status(); + if(!status_or) { + return status_or.getError(); + } + + auto buffer = (*file)->getBuffer(path_str, -1, true, false); + if(!buffer) { + return buffer.getError(); + } + + auto stripped = strip_to_includes((*buffer)->getBuffer()); + auto new_buffer = llvm::MemoryBuffer::getMemBufferCopy(stripped, path_str); + return std::make_unique(std::move(new_buffer), *status_or); + } + +private: + llvm::StringSet<> visited; +}; + +class ScanPPCallbacks : public clang::PPCallbacks { +public: + explicit ScanPPCallbacks(ScanResult& result) : result(result) {} + + void InclusionDirective(clang::SourceLocation, + const clang::Token&, + llvm::StringRef, + bool, + clang::CharSourceRange, + clang::OptionalFileEntryRef file, + llvm::StringRef, + llvm::StringRef, + const clang::Module*, + bool, + clang::SrcMgr::CharacteristicKind) override { + if(file) { + result.includes.emplace_back(file->getFileEntry().tryGetRealPathName().str()); + } + } + + void moduleImport(clang::SourceLocation, + clang::ModuleIdPath names, + const clang::Module*) override { + std::string name; + for(auto& part: names) { + if(!name.empty()) { + name += '.'; + } + name += part.getIdentifierInfo()->getName(); + } + result.modules.emplace_back(std::move(name)); + } + +private: + ScanResult& result; +}; + +} // namespace + +ScanResult scan_with_preprocessor(llvm::ArrayRef arguments, + llvm::StringRef directory, + bool arguments_from_database, + llvm::IntrusiveRefCntPtr vfs) { + ScanResult result; + + if(!vfs) { + vfs = llvm::vfs::createPhysicalFileSystem(); + } + + auto hooked_vfs = llvm::makeIntrusiveRefCnt(std::move(vfs)); + + // Create compiler invocation + clang::DiagnosticOptions diag_opts; + auto diag_engine = clang::CompilerInstance::createDiagnostics(*hooked_vfs, + diag_opts, + new clang::IgnoringDiagConsumer(), + true); + + std::unique_ptr invocation; + + if(arguments_from_database) { + invocation = std::make_unique(); + if(!clang::CompilerInvocation::CreateFromArgs(*invocation, + llvm::ArrayRef(arguments).drop_front(), + *diag_engine, + arguments[0])) { + return result; + } + } else { + clang::CreateInvocationOptions options = { + .Diags = diag_engine, + .VFS = hooked_vfs, + .ProbePrecompiled = false, + }; + invocation = clang::createInvocation(arguments, options); + if(!invocation) { + return result; + } + } + + invocation->getFrontendOpts().DisableFree = false; + + // Set working directory + invocation->getFileSystemOpts().WorkingDir = directory.str(); + + auto instance = std::make_unique(std::move(invocation)); + instance->createDiagnostics(*hooked_vfs, new clang::IgnoringDiagConsumer(), true); + instance->createFileManager(hooked_vfs); + + if(!instance->createTarget()) { + return result; + } + + auto action = std::make_unique(); + + if(!action->BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) { + return result; + } + + instance->getPreprocessor().addPPCallbacks(std::make_unique(result)); + + if(auto error = action->Execute()) { + llvm::consumeError(std::move(error)); + } + + action->EndSourceFile(); + + // Get module name from preprocessor + auto& pp = instance->getPreprocessor(); + auto module_name = pp.getNamedModuleName(); + if(!module_name.empty()) { + result.module_name = module_name; + result.is_interface_unit = pp.isInNamedInterfaceUnit(); + } + + return result; +} + +std::uint32_t compute_preamble_bound(llvm::StringRef content) { + auto result = compute_preamble_bounds(content); + if(result.empty()) { + return 0; + } else { + return result.back(); + } +} + +std::vector compute_preamble_bounds(llvm::StringRef content) { + std::vector result; + + Lexer lexer(content, true, nullptr, false); + + while(true) { + auto token = lexer.advance(); + if(token.is_eof()) { + break; + } + + if(token.is_at_start_of_line) { + if(token.kind == clang::tok::hash) { + /// For preprocessor directive, consume the whole directive. + lexer.advance_until(clang::tok::eod); + auto last = lexer.last(); + + /// Append the token before the eod. + result.push_back(last.range.end); + } else if(token.is_identifier() && token.text(content) == "module") { + /// If we encounter a module keyword at the start of a line, it may be + /// a module declaration or global module fragment. + auto next = lexer.next(); + + if(next.kind == clang::tok::semi) { + /// If next token is `;`, it is a global module fragment. + /// we just continue. + lexer.advance(); + + /// Append it to bounds. + result.push_back(next.range.end); + } else { + break; + } + } else { + break; + } + } + } + + return result; +} + +} // namespace clice diff --git a/src/syntax/scan.h b/src/syntax/scan.h new file mode 100644 index 000000000..af9f6f1e8 --- /dev/null +++ b/src/syntax/scan.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/VirtualFileSystem.h" + +namespace clice { + +struct ScanResult { + /// Module name (empty if not a module unit). + std::string module_name; + + /// Whether this is an interface unit (has `export module`). + bool is_interface_unit = false; + + /// Whether module declaration is inside conditional directive, + /// signaling caller should fall back to scan_with_preprocessor. + bool need_preprocess = false; + + /// Include file names (spelling without <>/quotes, e.g. "vector", "foo/bar.h"). + /// From lexer scan these are the raw header names; + /// from preprocessor scan these are resolved file paths. + std::vector includes; + + /// Dependent module names (only populated by scan_with_preprocessor). + std::vector modules; +}; + +/// Quick lexer-based scan for module name and include file names. +/// If module declaration is inside #if/#ifdef, sets need_preprocess=true +/// and module_name will be empty. +ScanResult scan(llvm::StringRef content); + +/// Full preprocessing-based scan. Uses hooked VFS to rapidly traverse +/// include chains. Returns module name, resolved include paths, and +/// module dependencies. +ScanResult scan_with_preprocessor(llvm::ArrayRef arguments, + llvm::StringRef directory, + bool arguments_from_database, + llvm::IntrusiveRefCntPtr vfs = nullptr); + +/// Compute preamble bound (moved from compile/preamble). +std::uint32_t compute_preamble_bound(llvm::StringRef content); +std::vector compute_preamble_bounds(llvm::StringRef content); + +} // namespace clice diff --git a/tests/unit/test/tester.cpp b/tests/unit/test/tester.cpp index 552be26af..9e4612b83 100644 --- a/tests/unit/test/tester.cpp +++ b/tests/unit/test/tester.cpp @@ -3,7 +3,7 @@ #include #include -#include "compile/preamble.h" +#include "syntax/scan.h" namespace clice::testing { From 201b834d476c5d35dda1bb5c6f072a5ce1634a7f Mon Sep 17 00:00:00 2001 From: ykiko Date: Mon, 9 Mar 2026 00:02:03 +0800 Subject: [PATCH 2/8] refactor: replace IncludeOnlyVFS with DependencyDirectivesGetter in scan module Replace the custom VFS hook (IncludeOnlyVFS/strip_to_includes) with clang's native DependencyDirectivesGetter + scanSourceForDependencyDirectives API. - Rewrite scan() to use scanSourceForDependencyDirectives instead of custom Lexer - Split scan_with_preprocessor into scan_fuzzy (strips conditionals/#define, processes all #includes unconditionally) and scan_precise (keeps all directives) - scan_fuzzy returns StringMap with per-file include results - Add SharedScanCache for cross-file directive caching with include_is_conditional flags computed from raw directive structure before filtering - Add IncludeInfo struct with conditional and not_found fields - Add content parameter to scan_fuzzy/scan_precise for remapped file support via OverlayFileSystem - Override FileNotFound in fuzzy mode to suppress errors and continue scanning - Use std::deque for local_entries to avoid Directive::Tokens ArrayRef invalidation - Add 19 unit tests covering scan, scan_fuzzy, and scan_precise Co-Authored-By: Claude Opus 4.6 (1M context) --- src/syntax/scan.cpp | 606 ++++++++++++++++++++++--------- src/syntax/scan.h | 70 +++- tests/unit/syntax/scan_tests.cpp | 367 +++++++++++++++++++ 3 files changed, 852 insertions(+), 191 deletions(-) create mode 100644 tests/unit/syntax/scan_tests.cpp diff --git a/src/syntax/scan.cpp b/src/syntax/scan.cpp index 53cf597e4..69bf1cf72 100644 --- a/src/syntax/scan.cpp +++ b/src/syntax/scan.cpp @@ -2,9 +2,15 @@ #include "syntax/lexer.h" +#include + #include "llvm/ADT/StringSet.h" #include "llvm/Support/MemoryBuffer.h" #include "clang/Basic/DiagnosticOptions.h" +#include "clang/Basic/FileEntry.h" +#include "clang/Basic/FileManager.h" +#include "clang/Basic/SourceManager.h" +#include "clang/Lex/PreprocessorOptions.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Lex/PPCallbacks.h" @@ -14,104 +20,95 @@ namespace clice { ScanResult scan(llvm::StringRef content) { - ScanResult result; + namespace dds = clang::dependency_directives_scan; - Lexer lexer(content, true, nullptr, false); + ScanResult result; - int conditional_depth = 0; + llvm::SmallVector tokens; + llvm::SmallVector directives; - while(true) { - auto token = lexer.advance(); - if(token.is_eof()) { - break; - } + if(clang::scanSourceForDependencyDirectives(content, tokens, directives)) { + return result; + } - if(!token.is_at_start_of_line) { - continue; - } + int conditional_depth = 0; - if(token.kind == clang::tok::hash) { - auto directive = lexer.advance(); - if(directive.is_eof()) { + for(auto& dir: directives) { + switch(dir.Kind) { + case dds::pp_if: + case dds::pp_ifdef: + case dds::pp_ifndef: { + conditional_depth++; break; } - - auto spelling = directive.text(content); - - if(spelling == "if" || spelling == "ifdef" || spelling == "ifndef") { - conditional_depth++; - lexer.advance_until(clang::tok::eod); - } else if(spelling == "endif") { + case dds::pp_endif: { if(conditional_depth > 0) { conditional_depth--; } - lexer.advance_until(clang::tok::eod); - } else if(spelling == "elif" || spelling == "elifdef" || spelling == "elifndef" || - spelling == "else") { - lexer.advance_until(clang::tok::eod); - } else if(spelling == "include") { - auto header = lexer.advance(); - if(header.is_header_name()) { - auto name = header.text(content); - // Strip <> or "" delimiters - result.includes.emplace_back(name.substr(1, name.size() - 2)); - } - lexer.advance_until(clang::tok::eod); - } else { - lexer.advance_until(clang::tok::eod); + break; } - } else if(token.is_identifier()) { - auto spelling = token.text(content); - bool is_export = false; - - if(spelling == "export") { - is_export = true; - auto next = lexer.advance(); - if(next.is_eof()) { - break; - } - if(!next.is_identifier() || next.text(content) != "module") { - continue; - } - spelling = "module"; + case dds::pp_elif: + case dds::pp_elifdef: + case dds::pp_elifndef: + case dds::pp_else: { + break; } - - if(spelling == "module") { - auto next = lexer.next(); - if(next.is_eof()) { - break; - } - - // `module;` is global module fragment, skip it - if(next.kind == clang::tok::semi) { - lexer.advance(); - continue; + case dds::pp_include: + case dds::pp_include_next: + case dds::pp___include_macros: { + // Find the header token (string_literal or header_name). + for(auto& tok: dir.Tokens) { + if(tok.is(clang::tok::header_name) || tok.is(clang::tok::string_literal)) { + auto name = content.substr(tok.Offset, tok.Length); + // Strip <> or "" delimiters. + if(name.size() >= 2) { + result.includes.push_back({ + std::string(name.substr(1, name.size() - 2)), + conditional_depth > 0, + false, + }); + } + break; + } } - - // Module declaration inside conditional directive + break; + } + case dds::cxx_module_decl: + case dds::cxx_export_module_decl: { if(conditional_depth > 0) { result.need_preprocess = true; return result; } - // Collect module name: identifiers, '.', ':' + // Collect module name from tokens: skip keywords, then + // collect identifiers, '.', ':'. std::string module_name; - while(true) { - auto tok = lexer.advance(); - if(tok.is_eof() || tok.kind == clang::tok::semi || tok.is_eod()) { - break; + bool seen_module_keyword = false; + for(auto& tok: dir.Tokens) { + if(!seen_module_keyword) { + if(tok.is(clang::tok::raw_identifier)) { + auto spelling = content.substr(tok.Offset, tok.Length); + if(spelling == "module") { + seen_module_keyword = true; + } + } + continue; } - if(tok.is_identifier()) { - module_name += tok.text(content); - } else if(tok.kind == clang::tok::period) { + if(tok.is(clang::tok::raw_identifier)) { + module_name += content.substr(tok.Offset, tok.Length); + } else if(tok.is(clang::tok::period)) { module_name += '.'; - } else if(tok.kind == clang::tok::colon) { + } else if(tok.is(clang::tok::colon)) { module_name += ':'; } } result.module_name = std::move(module_name); - result.is_interface_unit = is_export; + result.is_interface_unit = (dir.Kind == dds::cxx_export_module_decl); + break; + } + default: { + break; } } } @@ -121,117 +118,255 @@ ScanResult scan(llvm::StringRef content) { namespace { -class InMemoryFile : public llvm::vfs::File { -public: - explicit InMemoryFile(std::unique_ptr buffer, llvm::vfs::Status status) : - buffer(std::move(buffer)), file_status(std::move(status)) {} +enum class ScanMode { Fuzzy, Precise }; - llvm::ErrorOr status() override { - return file_status; - } +/// Compute include_is_conditional from raw directives: for each pp_include +/// (and pp_include_next, pp___include_macros, pp_import), record whether +/// it is nested inside any conditional block. +void compute_include_conditionals(SharedScanCache::CachedEntry& entry) { + using namespace clang::dependency_directives_scan; + + entry.include_is_conditional.clear(); + int cond_depth = 0; - llvm::ErrorOr> - getBuffer(const llvm::Twine&, int64_t, bool, bool) override { - return llvm::MemoryBuffer::getMemBufferCopy(buffer->getBuffer(), - buffer->getBufferIdentifier()); + for(auto& dir: entry.directives) { + switch(dir.Kind) { + case pp_if: + case pp_ifdef: + case pp_ifndef: { + cond_depth++; + break; + } + case pp_endif: { + if(cond_depth > 0) { + cond_depth--; + } + break; + } + case pp_include: + case pp_include_next: + case pp___include_macros: + case pp_import: { + entry.include_is_conditional.push_back(cond_depth > 0); + break; + } + default: { + break; + } + } } +} + +class ScanDirectivesGetter : public clang::DependencyDirectivesGetter { +public: + ScanDirectivesGetter(ScanMode mode, SharedScanCache* cache, clang::FileManager& file_mgr) : + mode(mode), cache(cache), file_mgr(&file_mgr) {} - std::error_code close() override { - return {}; + std::unique_ptr + cloneFor(clang::FileManager& new_file_mgr) override { + return std::make_unique(mode, cache, new_file_mgr); } -private: - std::unique_ptr buffer; - llvm::vfs::Status file_status; -}; + std::optional> + operator()(clang::FileEntryRef file) override { + auto path = file.getFileEntry().tryGetRealPathName(); + if(path.empty()) { + path = file.getName(); + } -/// Strip file content to only #include lines for fast preprocessing. -std::string strip_to_includes(llvm::StringRef content) { - std::string result; + // Check cache first. + if(cache) { + auto it = cache->entries.find(path); + if(it != cache->entries.end()) { + return get_directives(it->second); + } + } - Lexer lexer(content, true, nullptr, false); + // Read the file content. + auto buffer = file_mgr->getBufferForFile(file); + if(!buffer) { + return std::nullopt; + } - while(true) { - auto token = lexer.advance(); - if(token.is_eof()) { - break; + auto source = (*buffer)->getBuffer().str(); + + // Create entry in its final location first, then scan into it. + // Directive::Tokens are ArrayRefs pointing into the tokens SmallVector, + // so the entry must not be moved after scanning. + SharedScanCache::CachedEntry* entry_ptr; + if(cache) { + auto [it, _] = cache->entries.try_emplace(path); + entry_ptr = &it->second; + } else { + local_entries.emplace_back(); + entry_ptr = &local_entries.back(); } - if(token.is_at_start_of_line && token.kind == clang::tok::hash) { - auto directive = lexer.advance(); - if(directive.is_eof()) { - break; - } + entry_ptr->source = std::move(source); - auto spelling = directive.text(content); - if(spelling == "include") { - // Keep the whole directive line - auto start = token.range.begin; - auto eod = lexer.advance_until(clang::tok::eod); - auto end = eod.range.begin; - result += content.substr(start, end - start); - result += '\n'; + if(clang::scanSourceForDependencyDirectives(entry_ptr->source, + entry_ptr->tokens, + entry_ptr->directives)) { + // Scan failed — remove the entry. + if(cache) { + cache->entries.erase(path); } else { - lexer.advance_until(clang::tok::eod); + local_entries.pop_back(); } + return std::nullopt; } + + compute_include_conditionals(*entry_ptr); + return get_directives(*entry_ptr); } - return result; -} +private: + llvm::ArrayRef + get_directives(SharedScanCache::CachedEntry& entry) { + if(mode == ScanMode::Precise) { + return entry.directives; + } -class IncludeOnlyVFS : public llvm::vfs::ProxyFileSystem { -public: - explicit IncludeOnlyVFS(llvm::IntrusiveRefCntPtr base) : - ProxyFileSystem(std::move(base)) {} - - llvm::ErrorOr> - openFileForRead(const llvm::Twine& path) override { - llvm::SmallString<256> storage; - llvm::StringRef path_str = path.toStringRef(storage); - - // If already visited, return empty buffer to cut include chains - if(!visited.insert(path_str).second) { - auto status_or = getUnderlyingFS().status(path); - if(!status_or) { - return status_or.getError(); - } - auto empty = llvm::MemoryBuffer::getMemBuffer("", path_str); - return std::make_unique(std::move(empty), *status_or); + // Fuzzy mode: strip #define/#undef and ALL conditional directives, + // so every #include is processed unconditionally by the preprocessor. + auto& filtered = filtered_directives[&entry]; + if(!filtered.empty()) { + return filtered; } - // First visit: read real file, strip to includes only - auto file = getUnderlyingFS().openFileForRead(path); - if(!file) { - return file; + using namespace clang::dependency_directives_scan; + for(auto& dir: entry.directives) { + switch(dir.Kind) { + case pp_define: + case pp_undef: + case pp_if: + case pp_ifdef: + case pp_ifndef: + case pp_elif: + case pp_elifdef: + case pp_elifndef: + case pp_else: + case pp_endif: + case pp_pragma_push_macro: + case pp_pragma_pop_macro: { + break; + } + default: { + filtered.push_back(dir); + break; + } + } } - auto status_or = (*file)->status(); - if(!status_or) { - return status_or.getError(); + return filtered; + } + + ScanMode mode; + SharedScanCache* cache; + clang::FileManager* file_mgr; + std::deque local_entries; + llvm::DenseMap> + filtered_directives; +}; + +/// PPCallbacks for fuzzy mode: tracks per-file includes with conditional +/// flags looked up from the SharedScanCache. +class FuzzyScanPPCallbacks : public clang::PPCallbacks { +public: + FuzzyScanPPCallbacks(llvm::StringMap& results, + SharedScanCache& cache, + clang::SourceManager& source_mgr) : + results(results), cache(cache), source_mgr(source_mgr) {} + + void FileChanged(clang::SourceLocation loc, + FileChangeReason reason, + clang::SrcMgr::CharacteristicKind, + clang::FileID) override { + if(reason == EnterFile) { + current_file = get_file_path(source_mgr.getFileID(loc)); } + } - auto buffer = (*file)->getBuffer(path_str, -1, true, false); - if(!buffer) { - return buffer.getError(); + bool FileNotFound(llvm::StringRef file_name) override { + // Record the not-found include and consume the include counter + // so conditional flag correlation stays in sync. + record_include(current_file, file_name.str(), true); + // Return true to suppress the diagnostic and continue scanning. + return true; + } + + void InclusionDirective(clang::SourceLocation hash_loc, + const clang::Token&, + llvm::StringRef file_name, + bool, + clang::CharSourceRange, + clang::OptionalFileEntryRef file, + llvm::StringRef, + llvm::StringRef, + const clang::Module*, + bool, + clang::SrcMgr::CharacteristicKind) override { + // Determine which file this include is from via HashLoc. + auto from_file = get_file_path(source_mgr.getFileID(hash_loc)); + + std::string resolved_path; + if(file) { + resolved_path = file->getFileEntry().tryGetRealPathName().str(); + if(resolved_path.empty()) { + resolved_path = file->getName().str(); + } + } else { + resolved_path = file_name.str(); } - auto stripped = strip_to_includes((*buffer)->getBuffer()); - auto new_buffer = llvm::MemoryBuffer::getMemBufferCopy(stripped, path_str); - return std::make_unique(std::move(new_buffer), *status_or); + record_include(from_file, std::move(resolved_path), !file.has_value()); } private: - llvm::StringSet<> visited; + llvm::StringRef get_file_path(clang::FileID fid) { + auto fe = source_mgr.getFileEntryRefForID(fid); + if(fe) { + auto path = fe->getFileEntry().tryGetRealPathName(); + return path.empty() ? fe->getName() : path; + } + return ""; + } + + void record_include(llvm::StringRef from_file, std::string path, bool not_found) { + // Look up conditional flag from cache. + bool conditional = false; + auto cache_it = cache.entries.find(from_file); + if(cache_it != cache.entries.end()) { + unsigned idx = include_counters[from_file]++; + if(idx < cache_it->second.include_is_conditional.size()) { + conditional = cache_it->second.include_is_conditional[idx]; + } + } + + results[from_file].includes.push_back({ + std::move(path), + conditional, + not_found, + }); + } + + llvm::StringMap& results; + SharedScanCache& cache; + clang::SourceManager& source_mgr; + llvm::StringRef current_file; + llvm::StringMap include_counters; }; -class ScanPPCallbacks : public clang::PPCallbacks { +/// PPCallbacks for precise mode: single ScanResult with accurate +/// conditional tracking via preprocessor callbacks. +class PreciseScanPPCallbacks : public clang::PPCallbacks { public: - explicit ScanPPCallbacks(ScanResult& result) : result(result) {} + explicit PreciseScanPPCallbacks(ScanResult& result) : result(result) {} void InclusionDirective(clang::SourceLocation, const clang::Token&, - llvm::StringRef, + llvm::StringRef file_name, bool, clang::CharSourceRange, clang::OptionalFileEntryRef file, @@ -240,8 +375,36 @@ class ScanPPCallbacks : public clang::PPCallbacks { const clang::Module*, bool, clang::SrcMgr::CharacteristicKind) override { + bool not_found = !file.has_value(); + std::string resolved_path; if(file) { - result.includes.emplace_back(file->getFileEntry().tryGetRealPathName().str()); + resolved_path = file->getFileEntry().tryGetRealPathName().str(); + } else { + resolved_path = file_name.str(); + } + + result.includes.push_back({ + std::move(resolved_path), + conditional_depth > 0, + not_found, + }); + } + + void If(clang::SourceLocation, clang::SourceRange, ConditionValueKind) override { + conditional_depth++; + } + + void Ifdef(clang::SourceLocation, const clang::Token&, const clang::MacroDefinition&) override { + conditional_depth++; + } + + void Ifndef(clang::SourceLocation, const clang::Token&, const clang::MacroDefinition&) override { + conditional_depth++; + } + + void Endif(clang::SourceLocation, clang::SourceLocation) override { + if(conditional_depth > 0) { + conditional_depth--; } } @@ -260,28 +423,23 @@ class ScanPPCallbacks : public clang::PPCallbacks { private: ScanResult& result; + int conditional_depth = 0; }; -} // namespace - -ScanResult scan_with_preprocessor(llvm::ArrayRef arguments, - llvm::StringRef directory, - bool arguments_from_database, - llvm::IntrusiveRefCntPtr vfs) { - ScanResult result; - - if(!vfs) { - vfs = llvm::vfs::createPhysicalFileSystem(); - } - - auto hooked_vfs = llvm::makeIntrusiveRefCnt(std::move(vfs)); - - // Create compiler invocation +/// Create and configure a CompilerInstance for scanning. +/// If content is non-empty, it is used as remapped source for the main file. +std::unique_ptr + create_scan_instance(llvm::ArrayRef arguments, + llvm::StringRef directory, + bool arguments_from_database, + llvm::StringRef content, + llvm::IntrusiveRefCntPtr vfs) { clang::DiagnosticOptions diag_opts; - auto diag_engine = clang::CompilerInstance::createDiagnostics(*hooked_vfs, - diag_opts, - new clang::IgnoringDiagConsumer(), - true); + auto diag_engine = + clang::CompilerInstance::createDiagnostics(*vfs, + diag_opts, + new clang::IgnoringDiagConsumer(), + true); std::unique_ptr invocation; @@ -291,28 +449,117 @@ ScanResult scan_with_preprocessor(llvm::ArrayRef arguments, llvm::ArrayRef(arguments).drop_front(), *diag_engine, arguments[0])) { - return result; + return nullptr; } } else { clang::CreateInvocationOptions options = { .Diags = diag_engine, - .VFS = hooked_vfs, + .VFS = vfs, .ProbePrecompiled = false, }; invocation = clang::createInvocation(arguments, options); if(!invocation) { - return result; + return nullptr; } } invocation->getFrontendOpts().DisableFree = false; - - // Set working directory invocation->getFileSystemOpts().WorkingDir = directory.str(); + if(!content.empty()) { + auto& inputs = invocation->getFrontendOpts().Inputs; + if(!inputs.empty()) { + auto main_file = inputs[0].getFile(); + // Use an overlay VFS to inject the remapped content. This ensures + // both the preprocessor and the DependencyDirectivesGetter see it. + auto overlay = llvm::makeIntrusiveRefCnt(vfs); + auto mem_fs = llvm::makeIntrusiveRefCnt(); + mem_fs->addFile(main_file, 0, llvm::MemoryBuffer::getMemBufferCopy(content, main_file)); + overlay->pushOverlay(std::move(mem_fs)); + vfs = std::move(overlay); + } + } + auto instance = std::make_unique(std::move(invocation)); - instance->createDiagnostics(*hooked_vfs, new clang::IgnoringDiagConsumer(), true); - instance->createFileManager(hooked_vfs); + instance->createDiagnostics(*vfs, new clang::IgnoringDiagConsumer(), true); + instance->getDiagnostics().setSuppressAllDiagnostics(true); + instance->createFileManager(vfs); + + return instance; +} + +} // namespace + +llvm::StringMap + scan_fuzzy(llvm::ArrayRef arguments, + llvm::StringRef directory, + bool arguments_from_database, + llvm::StringRef content, + SharedScanCache* cache, + llvm::IntrusiveRefCntPtr vfs) { + llvm::StringMap results; + + if(!vfs) { + vfs = llvm::vfs::createPhysicalFileSystem(); + } + + auto instance = create_scan_instance(arguments, directory, arguments_from_database, content, vfs); + if(!instance) { + return results; + } + + // Use a local cache if none provided, so we always have conditional flags. + SharedScanCache local_cache; + if(!cache) { + cache = &local_cache; + } + + auto getter = + std::make_unique(ScanMode::Fuzzy, cache, instance->getFileManager()); + instance->setDependencyDirectivesGetter(std::move(getter)); + + if(!instance->createTarget()) { + return results; + } + + auto action = std::make_unique(); + + if(!action->BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) { + return results; + } + + instance->getPreprocessor().addPPCallbacks( + std::make_unique(results, *cache, instance->getSourceManager())); + + if(auto error = action->Execute()) { + llvm::consumeError(std::move(error)); + } + + action->EndSourceFile(); + + return results; +} + +ScanResult scan_precise(llvm::ArrayRef arguments, + llvm::StringRef directory, + bool arguments_from_database, + llvm::StringRef content, + SharedScanCache* cache, + llvm::IntrusiveRefCntPtr vfs) { + ScanResult result; + + if(!vfs) { + vfs = llvm::vfs::createPhysicalFileSystem(); + } + + auto instance = create_scan_instance(arguments, directory, arguments_from_database, content, vfs); + if(!instance) { + return result; + } + + auto getter = + std::make_unique(ScanMode::Precise, cache, instance->getFileManager()); + instance->setDependencyDirectivesGetter(std::move(getter)); if(!instance->createTarget()) { return result; @@ -324,7 +571,7 @@ ScanResult scan_with_preprocessor(llvm::ArrayRef arguments, return result; } - instance->getPreprocessor().addPPCallbacks(std::make_unique(result)); + instance->getPreprocessor().addPPCallbacks(std::make_unique(result)); if(auto error = action->Execute()) { llvm::consumeError(std::move(error)); @@ -332,11 +579,10 @@ ScanResult scan_with_preprocessor(llvm::ArrayRef arguments, action->EndSourceFile(); - // Get module name from preprocessor + // Get module name from preprocessor. auto& pp = instance->getPreprocessor(); - auto module_name = pp.getNamedModuleName(); - if(!module_name.empty()) { - result.module_name = module_name; + if(pp.isInNamedModule()) { + result.module_name = pp.getNamedModuleName(); result.is_interface_unit = pp.isInNamedInterfaceUnit(); } diff --git a/src/syntax/scan.h b/src/syntax/scan.h index af9f6f1e8..0f0da8854 100644 --- a/src/syntax/scan.h +++ b/src/syntax/scan.h @@ -6,8 +6,10 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/VirtualFileSystem.h" +#include "clang/Lex/DependencyDirectivesScanner.h" namespace clice { @@ -19,33 +21,79 @@ struct ScanResult { bool is_interface_unit = false; /// Whether module declaration is inside conditional directive, - /// signaling caller should fall back to scan_with_preprocessor. + /// signaling caller should fall back to preprocessor-based scan. bool need_preprocess = false; - /// Include file names (spelling without <>/quotes, e.g. "vector", "foo/bar.h"). + struct IncludeInfo { + /// Resolved file path (fuzzy/precise scan) or raw header name (lexer scan). + std::string path; + + /// Whether this include is inside a conditional directive context. + bool conditional = false; + + /// Whether the included file was not found during resolution. + bool not_found = false; + }; + + /// Include file names. /// From lexer scan these are the raw header names; /// from preprocessor scan these are resolved file paths. - std::vector includes; + std::vector includes; - /// Dependent module names (only populated by scan_with_preprocessor). + /// Dependent module names. std::vector modules; }; +/// Shared cache for dependency directives across multiple scan invocations. +struct SharedScanCache { + struct CachedEntry { + /// The source content of the file (kept alive for token references). + std::string source; + + /// Scanned tokens. + llvm::SmallVector tokens; + + /// Scanned directives (referencing tokens above). + llvm::SmallVector directives; + + /// Whether each pp_include directive is inside a conditional block, + /// computed from the raw directive structure before filtering. + std::vector include_is_conditional; + }; + + /// path -> cached scan result. + llvm::StringMap entries; +}; + /// Quick lexer-based scan for module name and include file names. /// If module declaration is inside #if/#ifdef, sets need_preprocess=true /// and module_name will be empty. ScanResult scan(llvm::StringRef content); -/// Full preprocessing-based scan. Uses hooked VFS to rapidly traverse -/// include chains. Returns module name, resolved include paths, and -/// module dependencies. -ScanResult scan_with_preprocessor(llvm::ArrayRef arguments, - llvm::StringRef directory, - bool arguments_from_database, - llvm::IntrusiveRefCntPtr vfs = nullptr); +/// Fuzzy preprocessing-based scan. Strips #define and conditional directives +/// so ALL #include are processed unconditionally. Each include is marked +/// with its structural conditional status from the raw directive scan. +/// Returns per-file results (main file + all transitively included files). +llvm::StringMap + scan_fuzzy(llvm::ArrayRef arguments, + llvm::StringRef directory, + bool arguments_from_database, + llvm::StringRef content = {}, + SharedScanCache* cache = nullptr, + llvm::IntrusiveRefCntPtr vfs = nullptr); + +/// Precise preprocessing-based scan. Keeps all directives including #define +/// and conditionals. Used for lazy module dependency resolution. +ScanResult scan_precise(llvm::ArrayRef arguments, + llvm::StringRef directory, + bool arguments_from_database, + llvm::StringRef content = {}, + SharedScanCache* cache = nullptr, + llvm::IntrusiveRefCntPtr vfs = nullptr); /// Compute preamble bound (moved from compile/preamble). std::uint32_t compute_preamble_bound(llvm::StringRef content); + std::vector compute_preamble_bounds(llvm::StringRef content); } // namespace clice diff --git a/tests/unit/syntax/scan_tests.cpp b/tests/unit/syntax/scan_tests.cpp new file mode 100644 index 000000000..6de5dffea --- /dev/null +++ b/tests/unit/syntax/scan_tests.cpp @@ -0,0 +1,367 @@ +#include "test/test.h" +#include "syntax/scan.h" + +#include "llvm/Support/VirtualFileSystem.h" + +namespace clice::testing { +namespace { + +/// Helper: find entry in StringMap whose key contains the given substring. +template +auto find_by_substr(llvm::StringMap& map, llvm::StringRef substr) { + for(auto it = map.begin(); it != map.end(); ++it) { + if(it->first().contains(substr)) { + return it; + } + } + return map.end(); +} + +TEST_SUITE(Scan) { + +// === scan() tests === + +TEST_CASE(BasicIncludes) { + auto result = scan(R"( +#include +#include "foo/bar.h" +int x = 1; +)"); + + ASSERT_EQ(result.includes.size(), 2u); + EXPECT_EQ(result.includes[0].path, "vector"); + EXPECT_FALSE(result.includes[0].conditional); + EXPECT_EQ(result.includes[1].path, "foo/bar.h"); + EXPECT_FALSE(result.includes[1].conditional); + EXPECT_TRUE(result.module_name.empty()); +} + +TEST_CASE(ConditionalIncludes) { + auto result = scan(R"( +#include +#ifdef FOO +#include +#endif +#include +)"); + + ASSERT_EQ(result.includes.size(), 3u); + EXPECT_EQ(result.includes[0].path, "always.h"); + EXPECT_FALSE(result.includes[0].conditional); + EXPECT_EQ(result.includes[1].path, "conditional.h"); + EXPECT_TRUE(result.includes[1].conditional); + EXPECT_EQ(result.includes[2].path, "after.h"); + EXPECT_FALSE(result.includes[2].conditional); +} + +TEST_CASE(NestedConditionals) { + auto result = scan(R"( +#ifdef A +#ifdef B +#include +#endif +#include +#endif +#include +)"); + + ASSERT_EQ(result.includes.size(), 3u); + EXPECT_EQ(result.includes[0].path, "nested.h"); + EXPECT_TRUE(result.includes[0].conditional); + EXPECT_EQ(result.includes[1].path, "outer.h"); + EXPECT_TRUE(result.includes[1].conditional); + EXPECT_EQ(result.includes[2].path, "top.h"); + EXPECT_FALSE(result.includes[2].conditional); +} + +TEST_CASE(ModuleDeclaration) { + auto result = scan(R"( +module; +#include +export module my.module; +)"); + + EXPECT_EQ(result.module_name, "my.module"); + EXPECT_TRUE(result.is_interface_unit); + EXPECT_FALSE(result.need_preprocess); + ASSERT_EQ(result.includes.size(), 1u); + EXPECT_EQ(result.includes[0].path, "header.h"); +} + +TEST_CASE(ModulePartition) { + auto result = scan(R"( +module my.module:part; +)"); + + EXPECT_EQ(result.module_name, "my.module:part"); + EXPECT_FALSE(result.is_interface_unit); +} + +TEST_CASE(ModuleImplementation) { + auto result = scan(R"( +module my.module; +)"); + + EXPECT_EQ(result.module_name, "my.module"); + EXPECT_FALSE(result.is_interface_unit); +} + +TEST_CASE(ConditionalModule) { + auto result = scan(R"( +#ifdef USE_MODULES +export module foo; +#endif +)"); + + EXPECT_TRUE(result.module_name.empty()); + EXPECT_TRUE(result.need_preprocess); +} + +TEST_CASE(GlobalModuleFragment) { + auto result = scan(R"( +module; +export module test; +)"); + + EXPECT_EQ(result.module_name, "test"); + EXPECT_TRUE(result.is_interface_unit); +} + +TEST_CASE(EmptyContent) { + auto result = scan(""); + EXPECT_TRUE(result.includes.empty()); + EXPECT_TRUE(result.module_name.empty()); + EXPECT_FALSE(result.need_preprocess); +} + +TEST_CASE(NoDirectives) { + auto result = scan(R"( +int main() { + return 0; +} +)"); + + EXPECT_TRUE(result.includes.empty()); + EXPECT_TRUE(result.module_name.empty()); +} + +// === scan_fuzzy() tests === + +TEST_CASE(FuzzyBasic) { + auto vfs = llvm::makeIntrusiveRefCnt(); + vfs->addFile("/test/main.cpp", 0, + llvm::MemoryBuffer::getMemBuffer(R"( +#include "header.h" +int main() {} +)")); + vfs->addFile("/test/header.h", 0, + llvm::MemoryBuffer::getMemBuffer(R"( +#pragma once +int x = 1; +)")); + + const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; + auto results = scan_fuzzy(args, "/test", false, {}, nullptr, vfs); + + auto main_it = find_by_substr(results, "main.cpp"); + ASSERT_TRUE(main_it != results.end()); + ASSERT_EQ(main_it->second.includes.size(), 1u); + EXPECT_FALSE(main_it->second.includes[0].not_found); + EXPECT_FALSE(main_it->second.includes[0].conditional); +} + +TEST_CASE(FuzzyConditionalTracking) { + auto vfs = llvm::makeIntrusiveRefCnt(); + vfs->addFile("/test/main.cpp", 0, + llvm::MemoryBuffer::getMemBuffer(R"( +#include "always.h" +#ifdef FOO +#include "conditional.h" +#endif +#include "after.h" +)")); + vfs->addFile("/test/always.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile("/test/conditional.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile("/test/after.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + + const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; + auto results = scan_fuzzy(args, "/test", false, {}, nullptr, vfs); + + auto main_it = find_by_substr(results, "main.cpp"); + ASSERT_TRUE(main_it != results.end()); + + auto& includes = main_it->second.includes; + ASSERT_EQ(includes.size(), 3u); + EXPECT_FALSE(includes[0].conditional); // always.h + EXPECT_TRUE(includes[1].conditional); // conditional.h + EXPECT_FALSE(includes[2].conditional); // after.h +} + +TEST_CASE(FuzzyNotFound) { + auto vfs = llvm::makeIntrusiveRefCnt(); + vfs->addFile("/test/main.cpp", 0, + llvm::MemoryBuffer::getMemBuffer(R"( +#include "exists.h" +#include "missing.h" +#include "also_exists.h" +)")); + vfs->addFile("/test/exists.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile("/test/also_exists.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + + const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; + auto results = scan_fuzzy(args, "/test", false, {}, nullptr, vfs); + + auto main_it = find_by_substr(results, "main.cpp"); + ASSERT_TRUE(main_it != results.end()); + + auto& includes = main_it->second.includes; + ASSERT_EQ(includes.size(), 3u); + EXPECT_FALSE(includes[0].not_found); // exists.h + EXPECT_TRUE(includes[1].not_found); // missing.h + EXPECT_FALSE(includes[2].not_found); // also_exists.h +} + +TEST_CASE(FuzzyTransitiveIncludes) { + auto vfs = llvm::makeIntrusiveRefCnt(); + vfs->addFile("/test/main.cpp", 0, + llvm::MemoryBuffer::getMemBuffer(R"( +#include "a.h" +)")); + vfs->addFile("/test/a.h", 0, + llvm::MemoryBuffer::getMemBuffer(R"( +#pragma once +#include "b.h" +)")); + vfs->addFile("/test/b.h", 0, + llvm::MemoryBuffer::getMemBuffer(R"( +#pragma once +int b = 1; +)")); + + const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; + auto results = scan_fuzzy(args, "/test", false, {}, nullptr, vfs); + + // main.cpp includes a.h + auto main_it = find_by_substr(results, "main.cpp"); + ASSERT_TRUE(main_it != results.end()); + ASSERT_EQ(main_it->second.includes.size(), 1u); + + // a.h includes b.h + auto a_it = find_by_substr(results, "a.h"); + ASSERT_TRUE(a_it != results.end()); + ASSERT_EQ(a_it->second.includes.size(), 1u); +} + +TEST_CASE(FuzzyWithCache) { + auto vfs = llvm::makeIntrusiveRefCnt(); + vfs->addFile("/test/main.cpp", 0, + llvm::MemoryBuffer::getMemBuffer(R"( +#include "shared.h" +)")); + vfs->addFile("/test/other.cpp", 0, + llvm::MemoryBuffer::getMemBuffer(R"( +#include "shared.h" +)")); + vfs->addFile("/test/shared.h", 0, + llvm::MemoryBuffer::getMemBuffer(R"( +#pragma once +int shared = 1; +)")); + + SharedScanCache cache; + + const char* args1[] = {"clang++", "-std=c++20", "/test/main.cpp"}; + auto results1 = scan_fuzzy(args1, "/test", false, {}, &cache, vfs); + + // shared.h should be cached after first scan. + EXPECT_FALSE(cache.entries.empty()); + + const char* args2[] = {"clang++", "-std=c++20", "/test/other.cpp"}; + auto results2 = scan_fuzzy(args2, "/test", false, {}, &cache, vfs); + + // Both scans should find includes. + ASSERT_TRUE(find_by_substr(results1, "main.cpp") != results1.end()); + ASSERT_TRUE(find_by_substr(results2, "other.cpp") != results2.end()); +} + +TEST_CASE(FuzzyWithContent) { + auto vfs = llvm::makeIntrusiveRefCnt(); + vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile("/test/header.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + + const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; + auto results = scan_fuzzy(args, "/test", false, + R"(#include "header.h")", + nullptr, vfs); + + auto main_it = find_by_substr(results, "main.cpp"); + ASSERT_TRUE(main_it != results.end()); + ASSERT_EQ(main_it->second.includes.size(), 1u); + EXPECT_FALSE(main_it->second.includes[0].not_found); +} + +// === scan_precise() tests === + +TEST_CASE(PreciseBasic) { + auto vfs = llvm::makeIntrusiveRefCnt(); + vfs->addFile("/test/main.cpp", 0, + llvm::MemoryBuffer::getMemBuffer(R"( +#include "header.h" +int main() {} +)")); + vfs->addFile("/test/header.h", 0, + llvm::MemoryBuffer::getMemBuffer(R"( +#pragma once +int x = 1; +)")); + + const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; + auto result = scan_precise(args, "/test", false, {}, nullptr, vfs); + + ASSERT_EQ(result.includes.size(), 1u); + EXPECT_FALSE(result.includes[0].not_found); + EXPECT_FALSE(result.includes[0].conditional); +} + +TEST_CASE(PreciseConditionalWithDefine) { + auto vfs = llvm::makeIntrusiveRefCnt(); + vfs->addFile("/test/main.cpp", 0, + llvm::MemoryBuffer::getMemBuffer(R"( +#define USE_FOO +#ifdef USE_FOO +#include "foo.h" +#endif +#ifndef USE_FOO +#include "bar.h" +#endif +)")); + vfs->addFile("/test/foo.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile("/test/bar.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + + const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; + auto result = scan_precise(args, "/test", false, {}, nullptr, vfs); + + // Precise mode evaluates conditionals: only foo.h should be included. + ASSERT_EQ(result.includes.size(), 1u); + EXPECT_TRUE(result.includes[0].conditional); + EXPECT_TRUE(result.includes[0].path.find("foo.h") != std::string::npos); +} + +TEST_CASE(PreciseWithContent) { + auto vfs = llvm::makeIntrusiveRefCnt(); + vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile("/test/header.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + + const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; + auto result = scan_precise(args, "/test", false, + R"(#include "header.h")", + nullptr, vfs); + + ASSERT_EQ(result.includes.size(), 1u); + EXPECT_FALSE(result.includes[0].not_found); +} + +}; // TEST_SUITE(Scan) + +} // namespace +} // namespace clice::testing From 501d5c4b82acfd938bf31140e6025de1d0d2025b Mon Sep 17 00:00:00 2001 From: ykiko Date: Mon, 9 Mar 2026 00:03:39 +0800 Subject: [PATCH 3/8] ci: expand cmake test matrix to 6 combinations Test all 3 OS (windows, ubuntu, macos) x 2 build types (Debug, RelWithDebInfo) without exclusions, matching the xmake matrix coverage. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/test-cmake.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-cmake.yml b/.github/workflows/test-cmake.yml index 34286459a..778d52209 100644 --- a/.github/workflows/test-cmake.yml +++ b/.github/workflows/test-cmake.yml @@ -8,13 +8,8 @@ jobs: strategy: fail-fast: false matrix: - include: - - os: windows-2025 - build_type: RelWithDebInfo - - os: ubuntu-24.04 - build_type: Debug - - os: macos-15 - build_type: Debug + os: [windows-2025, ubuntu-24.04, macos-15] + build_type: [Debug, RelWithDebInfo] runs-on: ${{ matrix.os }} steps: - name: Checkout repository From 861427586ab41fb48a06bf71d289ca17270dee28 Mon Sep 17 00:00:00 2001 From: ykiko Date: Mon, 9 Mar 2026 00:06:37 +0800 Subject: [PATCH 4/8] test: add preamble bound computation tests Add 8 test cases for compute_preamble_bound and compute_preamble_bounds covering empty input, no directives, single/multiple includes, global module fragment, bounds vector ordering, stopping at code, and conditional directives. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/unit/syntax/scan_tests.cpp | 88 ++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tests/unit/syntax/scan_tests.cpp b/tests/unit/syntax/scan_tests.cpp index 6de5dffea..bd6936a6e 100644 --- a/tests/unit/syntax/scan_tests.cpp +++ b/tests/unit/syntax/scan_tests.cpp @@ -363,5 +363,93 @@ TEST_CASE(PreciseWithContent) { }; // TEST_SUITE(Scan) +TEST_SUITE(PreambleBound) { + +TEST_CASE(Empty) { + EXPECT_EQ(compute_preamble_bound(""), 0u); +} + +TEST_CASE(NoDirectives) { + EXPECT_EQ(compute_preamble_bound("int x = 1;"), 0u); +} + +TEST_CASE(SingleInclude) { + llvm::StringRef src = "#include \nint x;"; + auto bound = compute_preamble_bound(src); + // Bound should be at the end of the #include directive. + EXPECT_TRUE(bound > 0u); + EXPECT_TRUE(bound <= src.find('\n') + 1); +} + +TEST_CASE(MultipleDirectives) { + llvm::StringRef src = + "#include \n" + "#include \n" + "#define FOO 1\n" + "int x;"; + auto bound = compute_preamble_bound(src); + // Bound should include all three directives. + EXPECT_TRUE(bound > src.find("#define")); +} + +TEST_CASE(GlobalModuleFragment) { + llvm::StringRef src = + "module;\n" + "#include \n" + "export module foo;"; + auto bound = compute_preamble_bound(src); + // Bound should include module; and #include, stopping at export module. + EXPECT_TRUE(bound > 0u); + EXPECT_TRUE(bound < src.size()); +} + +TEST_CASE(BoundsVector) { + llvm::StringRef src = + "#include \n" + "#include \n" + "int x;"; + auto bounds = compute_preamble_bounds(src); + // Should have two bounds, one per directive. + ASSERT_EQ(bounds.size(), 2u); + EXPECT_TRUE(bounds[0] < bounds[1]); +} + +TEST_CASE(BoundsWithModuleFragment) { + llvm::StringRef src = + "module;\n" + "#include \n" + "#include \n" + "export module foo;"; + auto bounds = compute_preamble_bounds(src); + // module; + two #include = 3 bounds. + ASSERT_EQ(bounds.size(), 3u); + EXPECT_TRUE(bounds[0] < bounds[1]); + EXPECT_TRUE(bounds[1] < bounds[2]); +} + +TEST_CASE(StopsAtCode) { + llvm::StringRef src = + "#include \n" + "int x;\n" + "#include \n"; + auto bounds = compute_preamble_bounds(src); + // Should stop at "int x;", only one bound for #include . + ASSERT_EQ(bounds.size(), 1u); +} + +TEST_CASE(ConditionalDirectives) { + llvm::StringRef src = + "#ifndef GUARD\n" + "#define GUARD\n" + "#include \n" + "#endif\n" + "int x;"; + auto bound = compute_preamble_bound(src); + // All directives are part of preamble. + EXPECT_TRUE(bound > src.find("#endif")); +} + +}; // TEST_SUITE(PreambleBound) + } // namespace } // namespace clice::testing From 14e1178493a63eb432ec48716f906a0449f94904 Mon Sep 17 00:00:00 2001 From: ykiko Date: Mon, 9 Mar 2026 00:09:25 +0800 Subject: [PATCH 5/8] ci: replace reviewdog suggester with diff check in format workflow Use simple git diff check instead of reviewdog action for format verification. Remove unnecessary PR write permissions from format job. Also apply clang-format and use raw string literals in preamble tests. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/check-format.yml | 15 ++-- .github/workflows/main.yml | 5 -- src/syntax/scan.cpp | 43 +++++----- tests/unit/syntax/scan_tests.cpp | 122 +++++++++++++---------------- 4 files changed, 85 insertions(+), 100 deletions(-) diff --git a/.github/workflows/check-format.yml b/.github/workflows/check-format.yml index f251d7112..26b308d75 100644 --- a/.github/workflows/check-format.yml +++ b/.github/workflows/check-format.yml @@ -24,10 +24,11 @@ jobs: args: --lint ./docs continue-on-error: true - - name: Suggest changes - if: github.event_name == 'pull_request' - uses: reviewdog/action-suggester@v1.24.0 - with: - tool_name: "fmt" - fail_level: any - filter_mode: nofilter + - name: Check diff + run: | + if ! git diff --quiet; then + echo "::error::Formatting changes detected. Please run 'pixi run format' and commit the result." + git --no-pager diff --stat + git --no-pager diff + exit 1 + fi diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9fe224011..d3cec081c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,11 +56,6 @@ jobs: format: needs: changes - permissions: - contents: read - checks: write - issues: write - pull-requests: write if: ${{ needs.changes.outputs.format == 'true' }} uses: ./.github/workflows/check-format.yml diff --git a/src/syntax/scan.cpp b/src/syntax/scan.cpp index 69bf1cf72..ebfb3e719 100644 --- a/src/syntax/scan.cpp +++ b/src/syntax/scan.cpp @@ -1,20 +1,20 @@ #include "syntax/scan.h" -#include "syntax/lexer.h" - #include +#include "syntax/lexer.h" + #include "llvm/ADT/StringSet.h" #include "llvm/Support/MemoryBuffer.h" #include "clang/Basic/DiagnosticOptions.h" #include "clang/Basic/FileEntry.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/SourceManager.h" -#include "clang/Lex/PreprocessorOptions.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Lex/PPCallbacks.h" #include "clang/Lex/Preprocessor.h" +#include "clang/Lex/PreprocessorOptions.h" #include "clang/Tooling/CompilationDatabase.h" namespace clice { @@ -398,7 +398,9 @@ class PreciseScanPPCallbacks : public clang::PPCallbacks { conditional_depth++; } - void Ifndef(clang::SourceLocation, const clang::Token&, const clang::MacroDefinition&) override { + void Ifndef(clang::SourceLocation, + const clang::Token&, + const clang::MacroDefinition&) override { conditional_depth++; } @@ -435,11 +437,10 @@ std::unique_ptr llvm::StringRef content, llvm::IntrusiveRefCntPtr vfs) { clang::DiagnosticOptions diag_opts; - auto diag_engine = - clang::CompilerInstance::createDiagnostics(*vfs, - diag_opts, - new clang::IgnoringDiagConsumer(), - true); + auto diag_engine = clang::CompilerInstance::createDiagnostics(*vfs, + diag_opts, + new clang::IgnoringDiagConsumer(), + true); std::unique_ptr invocation; @@ -490,20 +491,20 @@ std::unique_ptr } // namespace -llvm::StringMap - scan_fuzzy(llvm::ArrayRef arguments, - llvm::StringRef directory, - bool arguments_from_database, - llvm::StringRef content, - SharedScanCache* cache, - llvm::IntrusiveRefCntPtr vfs) { +llvm::StringMap scan_fuzzy(llvm::ArrayRef arguments, + llvm::StringRef directory, + bool arguments_from_database, + llvm::StringRef content, + SharedScanCache* cache, + llvm::IntrusiveRefCntPtr vfs) { llvm::StringMap results; if(!vfs) { vfs = llvm::vfs::createPhysicalFileSystem(); } - auto instance = create_scan_instance(arguments, directory, arguments_from_database, content, vfs); + auto instance = + create_scan_instance(arguments, directory, arguments_from_database, content, vfs); if(!instance) { return results; } @@ -552,13 +553,15 @@ ScanResult scan_precise(llvm::ArrayRef arguments, vfs = llvm::vfs::createPhysicalFileSystem(); } - auto instance = create_scan_instance(arguments, directory, arguments_from_database, content, vfs); + auto instance = + create_scan_instance(arguments, directory, arguments_from_database, content, vfs); if(!instance) { return result; } - auto getter = - std::make_unique(ScanMode::Precise, cache, instance->getFileManager()); + auto getter = std::make_unique(ScanMode::Precise, + cache, + instance->getFileManager()); instance->setDependencyDirectivesGetter(std::move(getter)); if(!instance->createTarget()) { diff --git a/tests/unit/syntax/scan_tests.cpp b/tests/unit/syntax/scan_tests.cpp index bd6936a6e..f56b69d5f 100644 --- a/tests/unit/syntax/scan_tests.cpp +++ b/tests/unit/syntax/scan_tests.cpp @@ -149,13 +149,11 @@ int main() { TEST_CASE(FuzzyBasic) { auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, - llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "header.h" int main() {} )")); - vfs->addFile("/test/header.h", 0, - llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile("/test/header.h", 0, llvm::MemoryBuffer::getMemBuffer(R"( #pragma once int x = 1; )")); @@ -172,8 +170,7 @@ int x = 1; TEST_CASE(FuzzyConditionalTracking) { auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, - llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "always.h" #ifdef FOO #include "conditional.h" @@ -199,8 +196,7 @@ TEST_CASE(FuzzyConditionalTracking) { TEST_CASE(FuzzyNotFound) { auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, - llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "exists.h" #include "missing.h" #include "also_exists.h" @@ -223,17 +219,14 @@ TEST_CASE(FuzzyNotFound) { TEST_CASE(FuzzyTransitiveIncludes) { auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, - llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "a.h" )")); - vfs->addFile("/test/a.h", 0, - llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile("/test/a.h", 0, llvm::MemoryBuffer::getMemBuffer(R"( #pragma once #include "b.h" )")); - vfs->addFile("/test/b.h", 0, - llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile("/test/b.h", 0, llvm::MemoryBuffer::getMemBuffer(R"( #pragma once int b = 1; )")); @@ -254,16 +247,13 @@ int b = 1; TEST_CASE(FuzzyWithCache) { auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, - llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "shared.h" )")); - vfs->addFile("/test/other.cpp", 0, - llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile("/test/other.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "shared.h" )")); - vfs->addFile("/test/shared.h", 0, - llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile("/test/shared.h", 0, llvm::MemoryBuffer::getMemBuffer(R"( #pragma once int shared = 1; )")); @@ -290,9 +280,7 @@ TEST_CASE(FuzzyWithContent) { vfs->addFile("/test/header.h", 0, llvm::MemoryBuffer::getMemBuffer("")); const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto results = scan_fuzzy(args, "/test", false, - R"(#include "header.h")", - nullptr, vfs); + auto results = scan_fuzzy(args, "/test", false, R"(#include "header.h")", nullptr, vfs); auto main_it = find_by_substr(results, "main.cpp"); ASSERT_TRUE(main_it != results.end()); @@ -304,13 +292,11 @@ TEST_CASE(FuzzyWithContent) { TEST_CASE(PreciseBasic) { auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, - llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "header.h" int main() {} )")); - vfs->addFile("/test/header.h", 0, - llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile("/test/header.h", 0, llvm::MemoryBuffer::getMemBuffer(R"( #pragma once int x = 1; )")); @@ -325,8 +311,7 @@ int x = 1; TEST_CASE(PreciseConditionalWithDefine) { auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, - llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( #define USE_FOO #ifdef USE_FOO #include "foo.h" @@ -353,9 +338,7 @@ TEST_CASE(PreciseWithContent) { vfs->addFile("/test/header.h", 0, llvm::MemoryBuffer::getMemBuffer("")); const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto result = scan_precise(args, "/test", false, - R"(#include "header.h")", - nullptr, vfs); + auto result = scan_precise(args, "/test", false, R"(#include "header.h")", nullptr, vfs); ASSERT_EQ(result.includes.size(), 1u); EXPECT_FALSE(result.includes[0].not_found); @@ -374,52 +357,55 @@ TEST_CASE(NoDirectives) { } TEST_CASE(SingleInclude) { - llvm::StringRef src = "#include \nint x;"; + llvm::StringRef src = R"( +#include +int x; +)"; auto bound = compute_preamble_bound(src); - // Bound should be at the end of the #include directive. EXPECT_TRUE(bound > 0u); - EXPECT_TRUE(bound <= src.find('\n') + 1); + EXPECT_TRUE(bound <= src.find("int")); } TEST_CASE(MultipleDirectives) { - llvm::StringRef src = - "#include \n" - "#include \n" - "#define FOO 1\n" - "int x;"; + llvm::StringRef src = R"( +#include +#include +#define FOO 1 +int x; +)"; auto bound = compute_preamble_bound(src); - // Bound should include all three directives. EXPECT_TRUE(bound > src.find("#define")); } TEST_CASE(GlobalModuleFragment) { - llvm::StringRef src = - "module;\n" - "#include \n" - "export module foo;"; + llvm::StringRef src = R"( +module; +#include +export module foo; +)"; auto bound = compute_preamble_bound(src); - // Bound should include module; and #include, stopping at export module. EXPECT_TRUE(bound > 0u); EXPECT_TRUE(bound < src.size()); } TEST_CASE(BoundsVector) { - llvm::StringRef src = - "#include \n" - "#include \n" - "int x;"; + llvm::StringRef src = R"( +#include +#include +int x; +)"; auto bounds = compute_preamble_bounds(src); - // Should have two bounds, one per directive. ASSERT_EQ(bounds.size(), 2u); EXPECT_TRUE(bounds[0] < bounds[1]); } TEST_CASE(BoundsWithModuleFragment) { - llvm::StringRef src = - "module;\n" - "#include \n" - "#include \n" - "export module foo;"; + llvm::StringRef src = R"( +module; +#include +#include +export module foo; +)"; auto bounds = compute_preamble_bounds(src); // module; + two #include = 3 bounds. ASSERT_EQ(bounds.size(), 3u); @@ -428,24 +414,24 @@ TEST_CASE(BoundsWithModuleFragment) { } TEST_CASE(StopsAtCode) { - llvm::StringRef src = - "#include \n" - "int x;\n" - "#include \n"; + llvm::StringRef src = R"( +#include +int x; +#include +)"; auto bounds = compute_preamble_bounds(src); - // Should stop at "int x;", only one bound for #include . ASSERT_EQ(bounds.size(), 1u); } TEST_CASE(ConditionalDirectives) { - llvm::StringRef src = - "#ifndef GUARD\n" - "#define GUARD\n" - "#include \n" - "#endif\n" - "int x;"; + llvm::StringRef src = R"( +#ifndef GUARD +#define GUARD +#include +#endif +int x; +)"; auto bound = compute_preamble_bound(src); - // All directives are part of preamble. EXPECT_TRUE(bound > src.find("#endif")); } From 4efd17724fd12c3a486a2e5c36887b00c32f01fe Mon Sep 17 00:00:00 2001 From: ykiko Date: Mon, 9 Mar 2026 00:28:12 +0800 Subject: [PATCH 6/8] fix: use CreateFromArgs in scan tests to fix Windows failures The clang driver (createInvocation) doesn't work reliably on Windows with a pure InMemoryFileSystem because it tries to resolve system paths and tools. Using CreateFromArgs (arguments_from_database=true) bypasses the driver entirely. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/unit/syntax/scan_tests.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/unit/syntax/scan_tests.cpp b/tests/unit/syntax/scan_tests.cpp index f56b69d5f..cb15fd73e 100644 --- a/tests/unit/syntax/scan_tests.cpp +++ b/tests/unit/syntax/scan_tests.cpp @@ -159,7 +159,7 @@ int x = 1; )")); const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto results = scan_fuzzy(args, "/test", false, {}, nullptr, vfs); + auto results = scan_fuzzy(args, "/test", true, {}, nullptr, vfs); auto main_it = find_by_substr(results, "main.cpp"); ASSERT_TRUE(main_it != results.end()); @@ -182,7 +182,7 @@ TEST_CASE(FuzzyConditionalTracking) { vfs->addFile("/test/after.h", 0, llvm::MemoryBuffer::getMemBuffer("")); const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto results = scan_fuzzy(args, "/test", false, {}, nullptr, vfs); + auto results = scan_fuzzy(args, "/test", true, {}, nullptr, vfs); auto main_it = find_by_substr(results, "main.cpp"); ASSERT_TRUE(main_it != results.end()); @@ -205,7 +205,7 @@ TEST_CASE(FuzzyNotFound) { vfs->addFile("/test/also_exists.h", 0, llvm::MemoryBuffer::getMemBuffer("")); const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto results = scan_fuzzy(args, "/test", false, {}, nullptr, vfs); + auto results = scan_fuzzy(args, "/test", true, {}, nullptr, vfs); auto main_it = find_by_substr(results, "main.cpp"); ASSERT_TRUE(main_it != results.end()); @@ -232,7 +232,7 @@ int b = 1; )")); const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto results = scan_fuzzy(args, "/test", false, {}, nullptr, vfs); + auto results = scan_fuzzy(args, "/test", true, {}, nullptr, vfs); // main.cpp includes a.h auto main_it = find_by_substr(results, "main.cpp"); @@ -261,13 +261,13 @@ int shared = 1; SharedScanCache cache; const char* args1[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto results1 = scan_fuzzy(args1, "/test", false, {}, &cache, vfs); + auto results1 = scan_fuzzy(args1, "/test", true, {}, &cache, vfs); // shared.h should be cached after first scan. EXPECT_FALSE(cache.entries.empty()); const char* args2[] = {"clang++", "-std=c++20", "/test/other.cpp"}; - auto results2 = scan_fuzzy(args2, "/test", false, {}, &cache, vfs); + auto results2 = scan_fuzzy(args2, "/test", true, {}, &cache, vfs); // Both scans should find includes. ASSERT_TRUE(find_by_substr(results1, "main.cpp") != results1.end()); @@ -280,7 +280,7 @@ TEST_CASE(FuzzyWithContent) { vfs->addFile("/test/header.h", 0, llvm::MemoryBuffer::getMemBuffer("")); const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto results = scan_fuzzy(args, "/test", false, R"(#include "header.h")", nullptr, vfs); + auto results = scan_fuzzy(args, "/test", true, R"(#include "header.h")", nullptr, vfs); auto main_it = find_by_substr(results, "main.cpp"); ASSERT_TRUE(main_it != results.end()); @@ -302,7 +302,7 @@ int x = 1; )")); const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto result = scan_precise(args, "/test", false, {}, nullptr, vfs); + auto result = scan_precise(args, "/test", true, {}, nullptr, vfs); ASSERT_EQ(result.includes.size(), 1u); EXPECT_FALSE(result.includes[0].not_found); @@ -324,7 +324,7 @@ TEST_CASE(PreciseConditionalWithDefine) { vfs->addFile("/test/bar.h", 0, llvm::MemoryBuffer::getMemBuffer("")); const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto result = scan_precise(args, "/test", false, {}, nullptr, vfs); + auto result = scan_precise(args, "/test", true, {}, nullptr, vfs); // Precise mode evaluates conditionals: only foo.h should be included. ASSERT_EQ(result.includes.size(), 1u); @@ -338,7 +338,7 @@ TEST_CASE(PreciseWithContent) { vfs->addFile("/test/header.h", 0, llvm::MemoryBuffer::getMemBuffer("")); const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto result = scan_precise(args, "/test", false, R"(#include "header.h")", nullptr, vfs); + auto result = scan_precise(args, "/test", true, R"(#include "header.h")", nullptr, vfs); ASSERT_EQ(result.includes.size(), 1u); EXPECT_FALSE(result.includes[0].not_found); From 2b28e4f7cc0c92f6c91abfd538c1aaf87b7b71db Mon Sep 17 00:00:00 2001 From: ykiko Date: Mon, 9 Mar 2026 20:39:42 +0800 Subject: [PATCH 7/8] fix: use platform-appropriate absolute paths in VFS tests On Windows, paths like `/test/main.cpp` are not truly absolute (no drive letter), so InMemoryFileSystem::makeAbsolute() mangles them. Follow clangd's pattern: add test_root()/test_path() helpers that return `C:\clice-test\...` on Windows and `/clice-test/...` on Unix, and set CWD on InMemoryFileSystem before adding files. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/unit/syntax/scan_tests.cpp | 124 ++++++++++++++++++------------- tests/unit/test/platform.h | 21 ++++++ 2 files changed, 92 insertions(+), 53 deletions(-) diff --git a/tests/unit/syntax/scan_tests.cpp b/tests/unit/syntax/scan_tests.cpp index cb15fd73e..39362b513 100644 --- a/tests/unit/syntax/scan_tests.cpp +++ b/tests/unit/syntax/scan_tests.cpp @@ -17,6 +17,13 @@ auto find_by_substr(llvm::StringMap& map, llvm::StringRef substr) { return map.end(); } +/// Create an InMemoryFileSystem with CWD set to test_root(). +auto make_test_vfs() { + auto vfs = llvm::makeIntrusiveRefCnt(); + vfs->setCurrentWorkingDirectory(test_root()); + return vfs; +} + TEST_SUITE(Scan) { // === scan() tests === @@ -148,18 +155,20 @@ int main() { // === scan_fuzzy() tests === TEST_CASE(FuzzyBasic) { - auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = make_test_vfs(); + auto main_path = test_path("main.cpp"); + auto header_path = test_path("header.h"); + vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "header.h" int main() {} )")); - vfs->addFile("/test/header.h", 0, llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile(header_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( #pragma once int x = 1; )")); - const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto results = scan_fuzzy(args, "/test", true, {}, nullptr, vfs); + auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; + auto results = scan_fuzzy(args, test_root(), true, {}, nullptr, vfs); auto main_it = find_by_substr(results, "main.cpp"); ASSERT_TRUE(main_it != results.end()); @@ -169,20 +178,21 @@ int x = 1; } TEST_CASE(FuzzyConditionalTracking) { - auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = make_test_vfs(); + auto main_path = test_path("main.cpp"); + vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "always.h" #ifdef FOO #include "conditional.h" #endif #include "after.h" )")); - vfs->addFile("/test/always.h", 0, llvm::MemoryBuffer::getMemBuffer("")); - vfs->addFile("/test/conditional.h", 0, llvm::MemoryBuffer::getMemBuffer("")); - vfs->addFile("/test/after.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile(test_path("always.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile(test_path("conditional.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile(test_path("after.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); - const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto results = scan_fuzzy(args, "/test", true, {}, nullptr, vfs); + auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; + auto results = scan_fuzzy(args, test_root(), true, {}, nullptr, vfs); auto main_it = find_by_substr(results, "main.cpp"); ASSERT_TRUE(main_it != results.end()); @@ -195,17 +205,18 @@ TEST_CASE(FuzzyConditionalTracking) { } TEST_CASE(FuzzyNotFound) { - auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = make_test_vfs(); + auto main_path = test_path("main.cpp"); + vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "exists.h" #include "missing.h" #include "also_exists.h" )")); - vfs->addFile("/test/exists.h", 0, llvm::MemoryBuffer::getMemBuffer("")); - vfs->addFile("/test/also_exists.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile(test_path("exists.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile(test_path("also_exists.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); - const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto results = scan_fuzzy(args, "/test", true, {}, nullptr, vfs); + auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; + auto results = scan_fuzzy(args, test_root(), true, {}, nullptr, vfs); auto main_it = find_by_substr(results, "main.cpp"); ASSERT_TRUE(main_it != results.end()); @@ -218,21 +229,22 @@ TEST_CASE(FuzzyNotFound) { } TEST_CASE(FuzzyTransitiveIncludes) { - auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = make_test_vfs(); + auto main_path = test_path("main.cpp"); + vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "a.h" )")); - vfs->addFile("/test/a.h", 0, llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile(test_path("a.h"), 0, llvm::MemoryBuffer::getMemBuffer(R"( #pragma once #include "b.h" )")); - vfs->addFile("/test/b.h", 0, llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile(test_path("b.h"), 0, llvm::MemoryBuffer::getMemBuffer(R"( #pragma once int b = 1; )")); - const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto results = scan_fuzzy(args, "/test", true, {}, nullptr, vfs); + auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; + auto results = scan_fuzzy(args, test_root(), true, {}, nullptr, vfs); // main.cpp includes a.h auto main_it = find_by_substr(results, "main.cpp"); @@ -246,28 +258,30 @@ int b = 1; } TEST_CASE(FuzzyWithCache) { - auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = make_test_vfs(); + auto main_path = test_path("main.cpp"); + auto other_path = test_path("other.cpp"); + vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "shared.h" )")); - vfs->addFile("/test/other.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile(other_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "shared.h" )")); - vfs->addFile("/test/shared.h", 0, llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile(test_path("shared.h"), 0, llvm::MemoryBuffer::getMemBuffer(R"( #pragma once int shared = 1; )")); SharedScanCache cache; - const char* args1[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto results1 = scan_fuzzy(args1, "/test", true, {}, &cache, vfs); + auto args1 = std::vector{"clang++", "-std=c++20", main_path.c_str()}; + auto results1 = scan_fuzzy(args1, test_root(), true, {}, &cache, vfs); // shared.h should be cached after first scan. EXPECT_FALSE(cache.entries.empty()); - const char* args2[] = {"clang++", "-std=c++20", "/test/other.cpp"}; - auto results2 = scan_fuzzy(args2, "/test", true, {}, &cache, vfs); + auto args2 = std::vector{"clang++", "-std=c++20", other_path.c_str()}; + auto results2 = scan_fuzzy(args2, test_root(), true, {}, &cache, vfs); // Both scans should find includes. ASSERT_TRUE(find_by_substr(results1, "main.cpp") != results1.end()); @@ -275,12 +289,13 @@ int shared = 1; } TEST_CASE(FuzzyWithContent) { - auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer("")); - vfs->addFile("/test/header.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + auto vfs = make_test_vfs(); + auto main_path = test_path("main.cpp"); + vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile(test_path("header.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); - const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto results = scan_fuzzy(args, "/test", true, R"(#include "header.h")", nullptr, vfs); + auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; + auto results = scan_fuzzy(args, test_root(), true, R"(#include "header.h")", nullptr, vfs); auto main_it = find_by_substr(results, "main.cpp"); ASSERT_TRUE(main_it != results.end()); @@ -291,18 +306,19 @@ TEST_CASE(FuzzyWithContent) { // === scan_precise() tests === TEST_CASE(PreciseBasic) { - auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = make_test_vfs(); + auto main_path = test_path("main.cpp"); + vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( #include "header.h" int main() {} )")); - vfs->addFile("/test/header.h", 0, llvm::MemoryBuffer::getMemBuffer(R"( + vfs->addFile(test_path("header.h"), 0, llvm::MemoryBuffer::getMemBuffer(R"( #pragma once int x = 1; )")); - const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto result = scan_precise(args, "/test", true, {}, nullptr, vfs); + auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; + auto result = scan_precise(args, test_root(), true, {}, nullptr, vfs); ASSERT_EQ(result.includes.size(), 1u); EXPECT_FALSE(result.includes[0].not_found); @@ -310,8 +326,9 @@ int x = 1; } TEST_CASE(PreciseConditionalWithDefine) { - auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = make_test_vfs(); + auto main_path = test_path("main.cpp"); + vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( #define USE_FOO #ifdef USE_FOO #include "foo.h" @@ -320,11 +337,11 @@ TEST_CASE(PreciseConditionalWithDefine) { #include "bar.h" #endif )")); - vfs->addFile("/test/foo.h", 0, llvm::MemoryBuffer::getMemBuffer("")); - vfs->addFile("/test/bar.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile(test_path("foo.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile(test_path("bar.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); - const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto result = scan_precise(args, "/test", true, {}, nullptr, vfs); + auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; + auto result = scan_precise(args, test_root(), true, {}, nullptr, vfs); // Precise mode evaluates conditionals: only foo.h should be included. ASSERT_EQ(result.includes.size(), 1u); @@ -333,12 +350,13 @@ TEST_CASE(PreciseConditionalWithDefine) { } TEST_CASE(PreciseWithContent) { - auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->addFile("/test/main.cpp", 0, llvm::MemoryBuffer::getMemBuffer("")); - vfs->addFile("/test/header.h", 0, llvm::MemoryBuffer::getMemBuffer("")); + auto vfs = make_test_vfs(); + auto main_path = test_path("main.cpp"); + vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer("")); + vfs->addFile(test_path("header.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); - const char* args[] = {"clang++", "-std=c++20", "/test/main.cpp"}; - auto result = scan_precise(args, "/test", true, R"(#include "header.h")", nullptr, vfs); + auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; + auto result = scan_precise(args, test_root(), true, R"(#include "header.h")", nullptr, vfs); ASSERT_EQ(result.includes.size(), 1u); EXPECT_FALSE(result.includes[0].not_found); diff --git a/tests/unit/test/platform.h b/tests/unit/test/platform.h index 8ba2f91e5..702b37ff8 100644 --- a/tests/unit/test/platform.h +++ b/tests/unit/test/platform.h @@ -1,3 +1,6 @@ +#include "llvm/ADT/SmallString.h" +#include "llvm/Support/Path.h" + namespace clice::testing { #ifdef _WIN32 @@ -42,4 +45,22 @@ constexpr inline bool CIEnvironment = true; constexpr inline bool CIEnvironment = false; #endif +/// Platform-appropriate absolute root for VFS-based tests. +/// Windows requires a drive letter for paths to be truly absolute. +inline const char* test_root() { +#ifdef _WIN32 + return "C:\\clice-test"; +#else + return "/clice-test"; +#endif +} + +/// Build an absolute test path from a relative component. +/// e.g. test_path("main.cpp") → "/clice-test/main.cpp" or "C:\clice-test\main.cpp" +inline std::string test_path(llvm::StringRef relative) { + llvm::SmallString<128> result; + llvm::sys::path::append(result, test_root(), relative); + return std::string(result); +} + } // namespace clice::testing From b02e89b4e01e90993230527fdff9dcb6853cd841 Mon Sep 17 00:00:00 2001 From: ykiko Date: Mon, 9 Mar 2026 21:06:35 +0800 Subject: [PATCH 8/8] refactor: extract TestVFS helper class to simplify VFS test setup Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/unit/syntax/scan_tests.cpp | 142 ++++++++++++++----------------- tests/unit/test/platform.h | 38 ++++++--- 2 files changed, 91 insertions(+), 89 deletions(-) diff --git a/tests/unit/syntax/scan_tests.cpp b/tests/unit/syntax/scan_tests.cpp index 39362b513..05f9952ea 100644 --- a/tests/unit/syntax/scan_tests.cpp +++ b/tests/unit/syntax/scan_tests.cpp @@ -1,8 +1,6 @@ #include "test/test.h" #include "syntax/scan.h" -#include "llvm/Support/VirtualFileSystem.h" - namespace clice::testing { namespace { @@ -17,13 +15,6 @@ auto find_by_substr(llvm::StringMap& map, llvm::StringRef substr) { return map.end(); } -/// Create an InMemoryFileSystem with CWD set to test_root(). -auto make_test_vfs() { - auto vfs = llvm::makeIntrusiveRefCnt(); - vfs->setCurrentWorkingDirectory(test_root()); - return vfs; -} - TEST_SUITE(Scan) { // === scan() tests === @@ -155,20 +146,19 @@ int main() { // === scan_fuzzy() tests === TEST_CASE(FuzzyBasic) { - auto vfs = make_test_vfs(); - auto main_path = test_path("main.cpp"); - auto header_path = test_path("header.h"); - vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = llvm::makeIntrusiveRefCnt(); + auto main_path = TestVFS::path("main.cpp"); + vfs->add("main.cpp", R"( #include "header.h" int main() {} -)")); - vfs->addFile(header_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( +)"); + vfs->add("header.h", R"( #pragma once int x = 1; -)")); +)"); auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; - auto results = scan_fuzzy(args, test_root(), true, {}, nullptr, vfs); + auto results = scan_fuzzy(args, TestVFS::root(), true, {}, nullptr, vfs); auto main_it = find_by_substr(results, "main.cpp"); ASSERT_TRUE(main_it != results.end()); @@ -178,21 +168,21 @@ int x = 1; } TEST_CASE(FuzzyConditionalTracking) { - auto vfs = make_test_vfs(); - auto main_path = test_path("main.cpp"); - vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = llvm::makeIntrusiveRefCnt(); + auto main_path = TestVFS::path("main.cpp"); + vfs->add("main.cpp", R"( #include "always.h" #ifdef FOO #include "conditional.h" #endif #include "after.h" -)")); - vfs->addFile(test_path("always.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); - vfs->addFile(test_path("conditional.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); - vfs->addFile(test_path("after.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); +)"); + vfs->add("always.h"); + vfs->add("conditional.h"); + vfs->add("after.h"); auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; - auto results = scan_fuzzy(args, test_root(), true, {}, nullptr, vfs); + auto results = scan_fuzzy(args, TestVFS::root(), true, {}, nullptr, vfs); auto main_it = find_by_substr(results, "main.cpp"); ASSERT_TRUE(main_it != results.end()); @@ -205,18 +195,18 @@ TEST_CASE(FuzzyConditionalTracking) { } TEST_CASE(FuzzyNotFound) { - auto vfs = make_test_vfs(); - auto main_path = test_path("main.cpp"); - vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = llvm::makeIntrusiveRefCnt(); + auto main_path = TestVFS::path("main.cpp"); + vfs->add("main.cpp", R"( #include "exists.h" #include "missing.h" #include "also_exists.h" -)")); - vfs->addFile(test_path("exists.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); - vfs->addFile(test_path("also_exists.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); +)"); + vfs->add("exists.h"); + vfs->add("also_exists.h"); auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; - auto results = scan_fuzzy(args, test_root(), true, {}, nullptr, vfs); + auto results = scan_fuzzy(args, TestVFS::root(), true, {}, nullptr, vfs); auto main_it = find_by_substr(results, "main.cpp"); ASSERT_TRUE(main_it != results.end()); @@ -229,22 +219,22 @@ TEST_CASE(FuzzyNotFound) { } TEST_CASE(FuzzyTransitiveIncludes) { - auto vfs = make_test_vfs(); - auto main_path = test_path("main.cpp"); - vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = llvm::makeIntrusiveRefCnt(); + auto main_path = TestVFS::path("main.cpp"); + vfs->add("main.cpp", R"( #include "a.h" -)")); - vfs->addFile(test_path("a.h"), 0, llvm::MemoryBuffer::getMemBuffer(R"( +)"); + vfs->add("a.h", R"( #pragma once #include "b.h" -)")); - vfs->addFile(test_path("b.h"), 0, llvm::MemoryBuffer::getMemBuffer(R"( +)"); + vfs->add("b.h", R"( #pragma once int b = 1; -)")); +)"); auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; - auto results = scan_fuzzy(args, test_root(), true, {}, nullptr, vfs); + auto results = scan_fuzzy(args, TestVFS::root(), true, {}, nullptr, vfs); // main.cpp includes a.h auto main_it = find_by_substr(results, "main.cpp"); @@ -258,30 +248,30 @@ int b = 1; } TEST_CASE(FuzzyWithCache) { - auto vfs = make_test_vfs(); - auto main_path = test_path("main.cpp"); - auto other_path = test_path("other.cpp"); - vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = llvm::makeIntrusiveRefCnt(); + auto main_path = TestVFS::path("main.cpp"); + auto other_path = TestVFS::path("other.cpp"); + vfs->add("main.cpp", R"( #include "shared.h" -)")); - vfs->addFile(other_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( +)"); + vfs->add("other.cpp", R"( #include "shared.h" -)")); - vfs->addFile(test_path("shared.h"), 0, llvm::MemoryBuffer::getMemBuffer(R"( +)"); + vfs->add("shared.h", R"( #pragma once int shared = 1; -)")); +)"); SharedScanCache cache; auto args1 = std::vector{"clang++", "-std=c++20", main_path.c_str()}; - auto results1 = scan_fuzzy(args1, test_root(), true, {}, &cache, vfs); + auto results1 = scan_fuzzy(args1, TestVFS::root(), true, {}, &cache, vfs); // shared.h should be cached after first scan. EXPECT_FALSE(cache.entries.empty()); auto args2 = std::vector{"clang++", "-std=c++20", other_path.c_str()}; - auto results2 = scan_fuzzy(args2, test_root(), true, {}, &cache, vfs); + auto results2 = scan_fuzzy(args2, TestVFS::root(), true, {}, &cache, vfs); // Both scans should find includes. ASSERT_TRUE(find_by_substr(results1, "main.cpp") != results1.end()); @@ -289,13 +279,13 @@ int shared = 1; } TEST_CASE(FuzzyWithContent) { - auto vfs = make_test_vfs(); - auto main_path = test_path("main.cpp"); - vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer("")); - vfs->addFile(test_path("header.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); + auto vfs = llvm::makeIntrusiveRefCnt(); + auto main_path = TestVFS::path("main.cpp"); + vfs->add("main.cpp"); + vfs->add("header.h"); auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; - auto results = scan_fuzzy(args, test_root(), true, R"(#include "header.h")", nullptr, vfs); + auto results = scan_fuzzy(args, TestVFS::root(), true, R"(#include "header.h")", nullptr, vfs); auto main_it = find_by_substr(results, "main.cpp"); ASSERT_TRUE(main_it != results.end()); @@ -306,19 +296,19 @@ TEST_CASE(FuzzyWithContent) { // === scan_precise() tests === TEST_CASE(PreciseBasic) { - auto vfs = make_test_vfs(); - auto main_path = test_path("main.cpp"); - vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = llvm::makeIntrusiveRefCnt(); + auto main_path = TestVFS::path("main.cpp"); + vfs->add("main.cpp", R"( #include "header.h" int main() {} -)")); - vfs->addFile(test_path("header.h"), 0, llvm::MemoryBuffer::getMemBuffer(R"( +)"); + vfs->add("header.h", R"( #pragma once int x = 1; -)")); +)"); auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; - auto result = scan_precise(args, test_root(), true, {}, nullptr, vfs); + auto result = scan_precise(args, TestVFS::root(), true, {}, nullptr, vfs); ASSERT_EQ(result.includes.size(), 1u); EXPECT_FALSE(result.includes[0].not_found); @@ -326,9 +316,9 @@ int x = 1; } TEST_CASE(PreciseConditionalWithDefine) { - auto vfs = make_test_vfs(); - auto main_path = test_path("main.cpp"); - vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer(R"( + auto vfs = llvm::makeIntrusiveRefCnt(); + auto main_path = TestVFS::path("main.cpp"); + vfs->add("main.cpp", R"( #define USE_FOO #ifdef USE_FOO #include "foo.h" @@ -336,12 +326,12 @@ TEST_CASE(PreciseConditionalWithDefine) { #ifndef USE_FOO #include "bar.h" #endif -)")); - vfs->addFile(test_path("foo.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); - vfs->addFile(test_path("bar.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); +)"); + vfs->add("foo.h"); + vfs->add("bar.h"); auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; - auto result = scan_precise(args, test_root(), true, {}, nullptr, vfs); + auto result = scan_precise(args, TestVFS::root(), true, {}, nullptr, vfs); // Precise mode evaluates conditionals: only foo.h should be included. ASSERT_EQ(result.includes.size(), 1u); @@ -350,13 +340,13 @@ TEST_CASE(PreciseConditionalWithDefine) { } TEST_CASE(PreciseWithContent) { - auto vfs = make_test_vfs(); - auto main_path = test_path("main.cpp"); - vfs->addFile(main_path, 0, llvm::MemoryBuffer::getMemBuffer("")); - vfs->addFile(test_path("header.h"), 0, llvm::MemoryBuffer::getMemBuffer("")); + auto vfs = llvm::makeIntrusiveRefCnt(); + auto main_path = TestVFS::path("main.cpp"); + vfs->add("main.cpp"); + vfs->add("header.h"); auto args = std::vector{"clang++", "-std=c++20", main_path.c_str()}; - auto result = scan_precise(args, test_root(), true, R"(#include "header.h")", nullptr, vfs); + auto result = scan_precise(args, TestVFS::root(), true, R"(#include "header.h")", nullptr, vfs); ASSERT_EQ(result.includes.size(), 1u); EXPECT_FALSE(result.includes[0].not_found); diff --git a/tests/unit/test/platform.h b/tests/unit/test/platform.h index 702b37ff8..b58dbea89 100644 --- a/tests/unit/test/platform.h +++ b/tests/unit/test/platform.h @@ -1,5 +1,7 @@ #include "llvm/ADT/SmallString.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualFileSystem.h" namespace clice::testing { @@ -45,22 +47,32 @@ constexpr inline bool CIEnvironment = true; constexpr inline bool CIEnvironment = false; #endif -/// Platform-appropriate absolute root for VFS-based tests. -/// Windows requires a drive letter for paths to be truly absolute. -inline const char* test_root() { +class TestVFS : public llvm::vfs::InMemoryFileSystem { +public: + TestVFS() { + setCurrentWorkingDirectory(root()); + } + + const static char* root() { #ifdef _WIN32 - return "C:\\clice-test"; + return "C:\\clice-test"; #else - return "/clice-test"; + return "/clice-test"; #endif -} + } + + /// root() + relative → absolute path. + static std::string path(llvm::StringRef relative) { + llvm::SmallString<128> result; + llvm::sys::path::append(result, root(), relative); + return std::string(result); + } -/// Build an absolute test path from a relative component. -/// e.g. test_path("main.cpp") → "/clice-test/main.cpp" or "C:\clice-test\main.cpp" -inline std::string test_path(llvm::StringRef relative) { - llvm::SmallString<128> result; - llvm::sys::path::append(result, test_root(), relative); - return std::string(result); -} + /// Add a file with an optional content (relative path, auto-prefixed with root()). + void add(llvm::StringRef relative, llvm::StringRef content = {}) { + auto p = path(relative); + addFile(p, 0, llvm::MemoryBuffer::getMemBufferCopy(content, p)); + } +}; } // namespace clice::testing