From ff05b15376ca6905b518d1217dc397687f32b541 Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 01:47:41 +0800 Subject: [PATCH 01/18] refactor(server): separate Workspace and Session state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce two new data structures that clearly separate project-wide persistent state from per-open-file volatile state: - **Workspace**: owns all disk-derived shared state (dep_graph, compile_graph, path_to_module, pch_cache, pcm_cache, project_index, merged_indices, cdb, config, path_pool). Updated only at startup and on didSave. Has on_file_saved()/on_file_closed() methods. - **Session**: owns all per-open-file state (text, version, generation, ast_dirty, pch_ref, ast_deps, header_context, active_context, file_index). Includes path_id for self-identification. Created on didOpen, destroyed on didClose. Compiler and Indexer are stateless service objects holding references to Workspace, sessions map, and WorkerPool. They own no data. Document lifecycle (open/change/close/save) is handled directly by MasterServer on the sessions map — Compiler no longer has these methods. Header context switching is done by directly setting Session fields. All 465 unit tests pass. Full binary builds successfully. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/compiler.cpp | 724 ++++++++++++++--------------------- src/server/compiler.h | 209 +++------- src/server/indexer.cpp | 266 +++++-------- src/server/indexer.h | 175 +++------ src/server/master_server.cpp | 375 +++++++++++++----- src/server/master_server.h | 35 +- src/server/session.h | 95 +++++ src/server/workspace.cpp | 91 +++++ src/server/workspace.h | 245 ++++++++++++ 9 files changed, 1198 insertions(+), 1017 deletions(-) create mode 100644 src/server/session.h create mode 100644 src/server/workspace.cpp create mode 100644 src/server/workspace.h diff --git a/src/server/compiler.cpp b/src/server/compiler.cpp index 705412660..ef0cf21b1 100644 --- a/src/server/compiler.cpp +++ b/src/server/compiler.cpp @@ -124,14 +124,10 @@ using serde_raw = et::serde::RawValue; Compiler::Compiler(et::event_loop& loop, et::ipc::JsonPeer& peer, - PathPool& path_pool, + Workspace& workspace, WorkerPool& pool, - Indexer& indexer, - const CliceConfig& config, - CompilationDatabase& cdb, - DependencyGraph& dep_graph) : - loop(loop), peer(peer), path_pool(path_pool), pool(pool), indexer(indexer), config(config), - cdb(cdb), dep_graph(dep_graph) {} + llvm::DenseMap& sessions) : + loop(loop), peer(peer), workspace(workspace), pool(pool), sessions(sessions) {} Compiler::~Compiler() { cancel_all(); @@ -139,27 +135,27 @@ Compiler::~Compiler() { void Compiler::fill_pcm_deps(std::unordered_map& pcms, std::uint32_t exclude_path_id) const { - for(auto& [pid, pcm_path]: pcm_paths) { + for(auto& [pid, pcm_path]: workspace.pcm_paths) { if(pid == exclude_path_id) continue; - auto mod_it = path_to_module.find(pid); - if(mod_it != path_to_module.end()) { + auto mod_it = workspace.path_to_module.find(pid); + if(mod_it != workspace.path_to_module.end()) { pcms[mod_it->second] = pcm_path; } } } void Compiler::cancel_all() { - if(compile_graph) { - compile_graph->cancel_all(); + if(workspace.compile_graph) { + workspace.compile_graph->cancel_all(); } } void Compiler::load_cache() { - if(config.cache_dir.empty()) + if(workspace.config.cache_dir.empty()) return; - auto cache_path = path::join(config.cache_dir, "cache", "cache.json"); + auto cache_path = path::join(workspace.config.cache_dir, "cache", "cache.json"); auto content = fs::read(cache_path); if(!content) { LOG_DEBUG("No cache.json found at {}", cache_path); @@ -184,20 +180,20 @@ void Compiler::load_cache() { auto dep_path = resolve(dep.path); if(dep_path.empty()) continue; - deps.path_ids.push_back(path_pool.intern(dep_path)); + deps.path_ids.push_back(workspace.path_pool.intern(dep_path)); deps.hashes.push_back(dep.hash); } return deps; }; for(auto& entry: data.pch) { - auto pch_path = path::join(config.cache_dir, "cache", "pch", entry.filename); + auto pch_path = path::join(workspace.config.cache_dir, "cache", "pch", entry.filename); auto source = resolve(entry.source_file); if(!llvm::sys::fs::exists(pch_path) || source.empty()) continue; - auto path_id = path_pool.intern(source); - auto& st = pch_states[path_id]; + auto path_id = workspace.path_pool.intern(source); + auto& st = workspace.pch_cache[path_id]; st.path = pch_path; st.hash = entry.hash; st.bound = entry.bound; @@ -207,32 +203,32 @@ void Compiler::load_cache() { } for(auto& entry: data.pcm) { - auto pcm_path = path::join(config.cache_dir, "cache", "pcm", entry.filename); + auto pcm_path = path::join(workspace.config.cache_dir, "cache", "pcm", entry.filename); auto source = resolve(entry.source_file); if(!llvm::sys::fs::exists(pcm_path) || source.empty()) continue; - auto path_id = path_pool.intern(source); - pcm_states[path_id] = {pcm_path, load_deps(entry.build_at, entry.deps)}; - pcm_paths[path_id] = pcm_path; + auto path_id = workspace.path_pool.intern(source); + workspace.pcm_cache[path_id] = {pcm_path, load_deps(entry.build_at, entry.deps)}; + workspace.pcm_paths[path_id] = pcm_path; LOG_DEBUG("Loaded cached PCM: {} (module {}) -> {}", source, entry.module_name, pcm_path); } LOG_INFO("Loaded cache.json: {} PCH entries, {} PCM entries", - pch_states.size(), - pcm_states.size()); + workspace.pch_cache.size(), + workspace.pcm_cache.size()); } void Compiler::save_cache() { - if(config.cache_dir.empty()) + if(workspace.config.cache_dir.empty()) return; CacheData data; std::unordered_map index_map; auto intern = [&](std::uint32_t runtime_path_id) -> std::uint32_t { - auto path = std::string(path_pool.resolve(runtime_path_id)); + auto path = std::string(workspace.path_pool.resolve(runtime_path_id)); auto [it, inserted] = index_map.try_emplace(path, static_cast(data.paths.size())); if(inserted) { @@ -241,7 +237,7 @@ void Compiler::save_cache() { return it->second; }; - for(auto& [path_id, st]: pch_states) { + for(auto& [path_id, st]: workspace.pch_cache) { if(st.path.empty()) continue; @@ -257,15 +253,15 @@ void Compiler::save_cache() { data.pch.push_back(std::move(entry)); } - for(auto& [path_id, st]: pcm_states) { + for(auto& [path_id, st]: workspace.pcm_cache) { if(st.path.empty()) continue; CachePCMEntry entry; entry.filename = std::string(path::filename(st.path)); entry.source_file = intern(path_id); - auto mod_it = path_to_module.find(path_id); - entry.module_name = mod_it != path_to_module.end() ? mod_it->second : ""; + auto mod_it = workspace.path_to_module.find(path_id); + entry.module_name = mod_it != workspace.path_to_module.end() ? mod_it->second : ""; entry.build_at = st.deps.build_at; for(std::size_t i = 0; i < st.deps.path_ids.size(); ++i) { entry.deps.push_back({intern(st.deps.path_ids[i]), st.deps.hashes[i]}); @@ -279,7 +275,7 @@ void Compiler::save_cache() { return; } - auto cache_path = path::join(config.cache_dir, "cache", "cache.json"); + auto cache_path = path::join(workspace.config.cache_dir, "cache", "cache.json"); auto tmp_path = cache_path + ".tmp"; auto write_result = fs::write(tmp_path, *json_str); if(!write_result) { @@ -294,14 +290,14 @@ void Compiler::save_cache() { } void Compiler::cleanup_cache(int max_age_days) { - if(config.cache_dir.empty()) + if(workspace.config.cache_dir.empty()) return; auto now = std::chrono::system_clock::now(); auto max_age = std::chrono::hours(max_age_days * 24); for(auto* subdir: {"cache/pch", "cache/pcm"}) { - auto dir = path::join(config.cache_dir, subdir); + auto dir = path::join(workspace.config.cache_dir, subdir); std::error_code ec; for(auto it = llvm::sys::fs::directory_iterator(dir, ec); !ec && it != llvm::sys::fs::directory_iterator(); @@ -321,23 +317,24 @@ void Compiler::cleanup_cache(int max_age_days) { } void Compiler::build_module_map() { - for(auto& [module_name, path_ids]: dep_graph.modules()) { + for(auto& [module_name, path_ids]: workspace.dep_graph.modules()) { for(auto path_id: path_ids) { - path_to_module[path_id] = module_name.str(); + workspace.path_to_module[path_id] = module_name.str(); } } } void Compiler::init_compile_graph() { - if(path_to_module.empty()) { + if(workspace.path_to_module.empty()) { LOG_INFO("No C++20 modules detected, skipping CompileGraph"); return; } // Lazy dependency resolver: scans a module file on demand to discover imports. auto resolve = [this](std::uint32_t path_id) -> llvm::SmallVector { - auto file_path = path_pool.resolve(path_id); - auto results = cdb.lookup(file_path, {.query_toolchain = true, .suppress_logging = true}); + auto file_path = workspace.path_pool.resolve(path_id); + auto results = + workspace.cdb.lookup(file_path, {.query_toolchain = true, .suppress_logging = true}); if(results.empty()) return {}; @@ -346,7 +343,7 @@ void Compiler::init_compile_graph() { llvm::SmallVector deps; for(auto& mod_name: scan_result.modules) { - auto mod_ids = dep_graph.lookup_module(mod_name); + auto mod_ids = workspace.dep_graph.lookup_module(mod_name); if(!mod_ids.empty()) { deps.push_back(mod_ids[0]); } @@ -354,7 +351,7 @@ void Compiler::init_compile_graph() { // Module implementation units implicitly depend on their interface unit. if(!scan_result.module_name.empty() && !scan_result.is_interface_unit) { - auto mod_ids = dep_graph.lookup_module(scan_result.module_name); + auto mod_ids = workspace.dep_graph.lookup_module(scan_result.module_name); if(!mod_ids.empty()) { deps.push_back(mod_ids[0]); } @@ -365,11 +362,11 @@ void Compiler::init_compile_graph() { // Dispatch: sends BuildPCM request to a stateless worker. auto dispatch = [this](std::uint32_t path_id) -> et::task { - auto mod_it = path_to_module.find(path_id); - if(mod_it == path_to_module.end()) + auto mod_it = workspace.path_to_module.find(path_id); + if(mod_it == workspace.path_to_module.end()) co_return false; - auto file_path = std::string(path_pool.resolve(path_id)); + auto file_path = std::string(workspace.path_pool.resolve(path_id)); worker::BuildParams bp; bp.kind = worker::BuildKind::BuildPCM; @@ -386,13 +383,13 @@ void Compiler::init_compile_graph() { } auto args_hash = llvm::xxh3_64bits(llvm::StringRef(hash_input)); auto pcm_filename = std::format("{}-{:016x}.pcm", safe_module_name, args_hash); - auto pcm_path = path::join(config.cache_dir, "cache", "pcm", pcm_filename); + auto pcm_path = path::join(workspace.config.cache_dir, "cache", "pcm", pcm_filename); // Check if cached PCM is still valid. - if(auto pcm_it = pcm_states.find(path_id); pcm_it != pcm_states.end()) { + if(auto pcm_it = workspace.pcm_cache.find(path_id); pcm_it != workspace.pcm_cache.end()) { if(!pcm_it->second.path.empty() && llvm::sys::fs::exists(pcm_it->second.path) && - !deps_changed(path_pool, pcm_it->second.deps)) { - pcm_paths[path_id] = pcm_it->second.path; + !deps_changed(workspace.path_pool, pcm_it->second.deps)) { + workspace.pcm_paths[path_id] = pcm_it->second.path; co_return true; } } @@ -411,39 +408,41 @@ void Compiler::init_compile_graph() { co_return false; } - pcm_paths[path_id] = result.value().output_path; - pcm_states[path_id] = {result.value().output_path, - capture_deps_snapshot(path_pool, result.value().deps)}; + workspace.pcm_paths[path_id] = result.value().output_path; + workspace.pcm_cache[path_id] = { + result.value().output_path, + capture_deps_snapshot(workspace.path_pool, result.value().deps)}; LOG_INFO("Built PCM for module {}: {}", mod_it->second, result.value().output_path); - // Merge module index into ProjectIndex/MergedIndex. - if(!result.value().tu_index_data.empty()) { - indexer.merge(result.value().tu_index_data.data(), result.value().tu_index_data.size()); - } - // Persist cache metadata after successful build. save_cache(); + + // Signal that new index data is available for background merge. + if(on_indexing_needed) + on_indexing_needed(); + co_return true; }; - compile_graph = std::make_unique(std::move(dispatch), std::move(resolve)); - LOG_INFO("CompileGraph initialized with {} module(s)", path_to_module.size()); + workspace.compile_graph = + std::make_unique(std::move(dispatch), std::move(resolve)); + LOG_INFO("CompileGraph initialized with {} module(s)", workspace.path_to_module.size()); } bool Compiler::fill_compile_args(llvm::StringRef path, std::string& directory, - std::vector& arguments) { - auto path_id = path_pool.intern(path); + std::vector& arguments, + Session* session) { + auto path_id = workspace.path_pool.intern(path); - // 1. If the user has set an active header context via switchContext, + // 1. If the session has an active header context via switchContext, // use the host source's CDB entry with file path replaced and preamble injected. - auto active_it = active_contexts.find(path_id); - if(active_it != active_contexts.end()) { - return fill_header_context_args(path, path_id, directory, arguments); + if(session && session->active_context.has_value()) { + return fill_header_context_args(path, path_id, directory, arguments, session); } // 2. Normal CDB lookup for the file itself. - auto results = cdb.lookup(path, {.query_toolchain = true}); + auto results = workspace.cdb.lookup(path, {.query_toolchain = true}); if(!results.empty()) { auto& ctx = results.front(); directory = ctx.directory.str(); @@ -455,38 +454,47 @@ bool Compiler::fill_compile_args(llvm::StringRef path, } // 3. No CDB entry — try automatic header context resolution. - return fill_header_context_args(path, path_id, directory, arguments); + return fill_header_context_args(path, path_id, directory, arguments, session); } bool Compiler::fill_header_context_args(llvm::StringRef path, std::uint32_t path_id, std::string& directory, - std::vector& arguments) { + std::vector& arguments, + Session* session) { // Use cached context if available; otherwise resolve. // If an active context override exists, invalidate cache if it points to // a different host so we re-resolve with the correct one. const HeaderFileContext* ctx_ptr = nullptr; - auto ctx_it = header_file_contexts.find(path_id); - auto active_it = active_contexts.find(path_id); - if(ctx_it != header_file_contexts.end()) { - if(active_it != active_contexts.end() && ctx_it->second.host_path_id != active_it->second) { - header_file_contexts.erase(ctx_it); + if(session && session->header_context.has_value()) { + if(session->active_context.has_value() && + session->header_context->host_path_id != *session->active_context) { + session->header_context.reset(); } else { - ctx_ptr = &ctx_it->second; + ctx_ptr = &*session->header_context; } } if(!ctx_ptr) { - auto resolved = resolve_header_context(path_id); + auto resolved = resolve_header_context(path_id, session); if(!resolved) { LOG_WARN("No CDB entry and no header context for {}", path); return false; } - header_file_contexts[path_id] = std::move(*resolved); - ctx_ptr = &header_file_contexts[path_id]; + if(session) { + session->header_context = std::move(*resolved); + ctx_ptr = &*session->header_context; + } else { + // Background indexing path — no session to store on. + // Use a temporary (caller will use it immediately). + // Store in a local and return. + static thread_local std::optional tl_ctx; + tl_ctx = std::move(*resolved); + ctx_ptr = &*tl_ctx; + } } - auto host_path = path_pool.resolve(ctx_ptr->host_path_id); - auto host_results = cdb.lookup(host_path, {.query_toolchain = true}); + auto host_path = workspace.path_pool.resolve(ctx_ptr->host_path_id); + auto host_results = workspace.cdb.lookup(host_path, {.query_toolchain = true}); if(host_results.empty()) { LOG_WARN("fill_header_context_args: host {} has no CDB entry", host_path); return false; @@ -527,9 +535,10 @@ bool Compiler::fill_header_context_args(llvm::StringRef path, return true; } -std::optional Compiler::resolve_header_context(std::uint32_t header_path_id) { +std::optional Compiler::resolve_header_context(std::uint32_t header_path_id, + Session* session) { // Find source files that transitively include this header. - auto hosts = dep_graph.find_host_sources(header_path_id); + auto hosts = workspace.dep_graph.find_host_sources(header_path_id); if(hosts.empty()) { LOG_DEBUG("resolve_header_context: no host sources for path_id={}", header_path_id); return std::nullopt; @@ -538,13 +547,12 @@ std::optional Compiler::resolve_header_context(std::uint32_t // If there's an active context override, prefer that host. std::uint32_t host_path_id = 0; std::vector chain; - auto active_it = active_contexts.find(header_path_id); - if(active_it != active_contexts.end()) { - auto preferred = active_it->second; - auto preferred_path = path_pool.resolve(preferred); - auto results = cdb.lookup(preferred_path, {.suppress_logging = true}); + if(session && session->active_context.has_value()) { + auto preferred = *session->active_context; + auto preferred_path = workspace.path_pool.resolve(preferred); + auto results = workspace.cdb.lookup(preferred_path, {.suppress_logging = true}); if(!results.empty()) { - auto c = dep_graph.find_include_chain(preferred, header_path_id); + auto c = workspace.dep_graph.find_include_chain(preferred, header_path_id); if(!c.empty()) { host_path_id = preferred; chain = std::move(c); @@ -555,11 +563,11 @@ std::optional Compiler::resolve_header_context(std::uint32_t // Fall back to the first available host that has a CDB entry. if(chain.empty()) { for(auto candidate: hosts) { - auto candidate_path = path_pool.resolve(candidate); - auto results = cdb.lookup(candidate_path, {.suppress_logging = true}); + auto candidate_path = workspace.path_pool.resolve(candidate); + auto results = workspace.cdb.lookup(candidate_path, {.suppress_logging = true}); if(results.empty()) continue; - auto c = dep_graph.find_include_chain(candidate, header_path_id); + auto c = workspace.dep_graph.find_include_chain(candidate, header_path_id); if(c.empty()) continue; host_path_id = candidate; @@ -582,22 +590,23 @@ std::optional Compiler::resolve_header_context(std::uint32_t auto cur_id = chain[i]; auto next_id = chain[i + 1]; - auto cur_path = path_pool.resolve(cur_id); - auto next_path = path_pool.resolve(next_id); + auto cur_path = workspace.path_pool.resolve(cur_id); + auto next_path = workspace.path_pool.resolve(next_id); auto next_filename = llvm::sys::path::filename(next_path); // Prefer in-memory document text over disk content. + // Use the session if this file matches the session's path, otherwise + // fall back to disk. std::string content; - if(auto doc_it = documents.find(cur_id); doc_it != documents.end()) { - content = doc_it->second.text; - } else { - auto buf = llvm::MemoryBuffer::getFile(cur_path); - if(!buf) { - LOG_WARN("resolve_header_context: cannot read {}", cur_path); - return std::nullopt; - } - content = (*buf)->getBuffer().str(); + // Note: we don't have the sessions map here, so we always read from disk + // for intermediate chain files. The session parameter only covers the + // header file itself (the target), not intermediate files in the chain. + auto buf = llvm::MemoryBuffer::getFile(cur_path); + if(!buf) { + LOG_WARN("resolve_header_context: cannot read {}", cur_path); + return std::nullopt; } + content = (*buf)->getBuffer().str(); // Scan line by line for the #include that brings in next_filename. llvm::StringRef content_ref(content); @@ -648,7 +657,7 @@ std::optional Compiler::resolve_header_context(std::uint32_t // Hash the preamble and write to cache directory. auto preamble_hash = llvm::xxh3_64bits(llvm::StringRef(preamble)); auto preamble_filename = std::format("{:016x}.h", preamble_hash); - auto preamble_dir = path::join(config.cache_dir, "header_context"); + auto preamble_dir = path::join(workspace.config.cache_dir, "header_context"); auto preamble_path = path::join(preamble_dir, preamble_filename); if(!llvm::sys::fs::exists(preamble_path)) { @@ -673,58 +682,70 @@ std::optional Compiler::resolve_header_context(std::uint32_t return HeaderFileContext{host_path_id, preamble_path, preamble_hash}; } -void Compiler::switch_context(std::uint32_t path_id, std::uint32_t context_path_id) { - active_contexts[path_id] = context_path_id; - header_file_contexts.erase(path_id); - pch_states.erase(path_id); - ast_deps.erase(path_id); - auto doc_it = documents.find(path_id); - if(doc_it != documents.end()) { - doc_it->second.ast_dirty = true; +std::string Compiler::uri_to_path(const std::string& uri) { + auto parsed = lsp::URI::parse(uri); + if(parsed.has_value()) { + auto path = parsed->file_path(); + if(path.has_value()) { + return std::move(*path); + } } + return uri; } -std::optional Compiler::get_active_context(std::uint32_t path_id) const { - auto it = active_contexts.find(path_id); - if(it != active_contexts.end()) - return it->second; - return std::nullopt; +void Compiler::publish_diagnostics(const std::string& uri, + int version, + const et::serde::RawValue& diagnostics_json) { + std::vector diagnostics; + if(!diagnostics_json.empty()) { + auto status = et::serde::json::from_json(diagnostics_json.data, diagnostics); + if(!status) { + LOG_WARN("Failed to deserialize diagnostics JSON for {}", uri); + } + } + protocol::PublishDiagnosticsParams params; + params.uri = uri; + params.version = version; + params.diagnostics = std::move(diagnostics); + peer.send_notification(params); } -void Compiler::invalidate_host_contexts(std::uint32_t host_path_id, - llvm::SmallVectorImpl& stale_headers) { - for(auto& [hdr_id, hdr_ctx]: header_file_contexts) { - if(hdr_ctx.host_path_id == host_path_id) - stale_headers.push_back(hdr_id); - } - for(auto hdr_id: stale_headers) { - header_file_contexts.erase(hdr_id); - } +void Compiler::clear_diagnostics(const std::string& uri) { + protocol::PublishDiagnosticsParams params; + params.uri = uri; + params.diagnostics = {}; + peer.send_notification(params); } -et::task Compiler::ensure_pch(std::uint32_t path_id, - llvm::StringRef path, - const std::string& text, +et::task Compiler::ensure_pch(Session& session, const std::string& directory, const std::vector& arguments) { + auto path_id = session.path_id; + auto path = workspace.path_pool.resolve(path_id); + auto& text = session.text; auto bound = compute_preamble_bound(text); if(bound == 0) { // No preamble directives — PCH would be empty. Clear any stale entry. - pch_states.erase(path_id); + workspace.pch_cache.erase(path_id); + session.pch_ref.reset(); co_return true; } auto preamble_hash = llvm::xxh3_64bits(llvm::StringRef(text).substr(0, bound)); // Deterministic content-addressed PCH path. - auto pch_path = - path::join(config.cache_dir, "cache", "pch", std::format("{:016x}.pch", preamble_hash)); + auto pch_path = path::join(workspace.config.cache_dir, + "cache", + "pch", + std::format("{:016x}.pch", preamble_hash)); // Reuse existing PCH if preamble content and deps haven't changed. - if(auto it = pch_states.find(path_id); it != pch_states.end()) { + if(auto it = workspace.pch_cache.find(path_id); it != workspace.pch_cache.end()) { auto& st = it->second; - if(st.hash == preamble_hash && !st.path.empty() && !deps_changed(path_pool, st.deps)) { + if(st.hash == preamble_hash && !st.path.empty() && + !deps_changed(workspace.path_pool, st.deps)) { st.bound = bound; + session.pch_ref = Session::PCHRef{path_id, preamble_hash, bound}; co_return true; } } @@ -732,18 +753,22 @@ et::task Compiler::ensure_pch(std::uint32_t path_id, // Preamble incomplete (user still typing) — defer rebuild, reuse old PCH if available. if(!is_preamble_complete(text, bound)) { LOG_DEBUG("Preamble incomplete for {}, deferring PCH rebuild", path); - co_return pch_states.count(path_id) && !pch_states[path_id].path.empty(); + co_return workspace.pch_cache.count(path_id) && !workspace.pch_cache[path_id].path.empty(); } // If another coroutine is already building PCH for this file, wait for it. - if(auto it = pch_states.find(path_id); it != pch_states.end() && it->second.building) { + if(auto it = workspace.pch_cache.find(path_id); + it != workspace.pch_cache.end() && it->second.building) { co_await it->second.building->wait(); - co_return !pch_states[path_id].path.empty(); + if(auto it2 = workspace.pch_cache.find(path_id); it2 != workspace.pch_cache.end()) { + session.pch_ref = Session::PCHRef{path_id, it2->second.hash, it2->second.bound}; + } + co_return workspace.pch_cache.count(path_id) && !workspace.pch_cache[path_id].path.empty(); } // Register in-flight build so concurrent requests wait on us. auto completion = std::make_shared(); - pch_states[path_id].building = completion; + workspace.pch_cache[path_id].building = completion; // Build a new PCH via stateless worker. worker::BuildParams bp; @@ -763,23 +788,21 @@ et::task Compiler::ensure_pch(std::uint32_t path_id, LOG_WARN("PCH build failed for {}: {}", path, result.has_value() ? result.value().error : result.error().message); - pch_states[path_id].building.reset(); + workspace.pch_cache[path_id].building.reset(); completion->set(); co_return false; } - auto& st = pch_states[path_id]; + auto& st = workspace.pch_cache[path_id]; st.path = result.value().output_path; st.bound = bound; st.hash = preamble_hash; - st.deps = capture_deps_snapshot(path_pool, result.value().deps); + st.deps = capture_deps_snapshot(workspace.path_pool, result.value().deps); st.building.reset(); - LOG_INFO("PCH built for {}: {}", path, result.value().output_path); + session.pch_ref = Session::PCHRef{path_id, preamble_hash, bound}; - if(!result.value().tu_index_data.empty()) { - indexer.merge(result.value().tu_index_data.data(), result.value().tu_index_data.size()); - } + LOG_INFO("PCH built for {}: {}", path, result.value().output_path); // Persist cache metadata after successful build. save_cache(); @@ -791,15 +814,15 @@ et::task Compiler::ensure_pch(std::uint32_t path_id, /// Compile module dependencies, build/reuse PCH, and fill PCM paths. /// Shared preparation step used by both ensure_compiled() (stateful path) /// and forward_stateless() (completion/signatureHelp path). -et::task Compiler::ensure_deps(std::uint32_t path_id, - llvm::StringRef path, - const std::string& text, +et::task Compiler::ensure_deps(Session& session, const std::string& directory, const std::vector& arguments, std::pair& pch, std::unordered_map& pcms) { + auto path_id = session.path_id; + // Compile C++20 module dependencies (PCMs). - if(compile_graph && !co_await compile_graph->compile_deps(path_id)) { + if(workspace.compile_graph && !co_await workspace.compile_graph->compile_deps(path_id)) { co_return false; } @@ -807,17 +830,17 @@ et::task Compiler::ensure_deps(std::uint32_t path_id, // When a user adds `import std;` without saving, the compile_graph (disk-based) // doesn't know about the new dependency. Scan the in-memory text to find them. { - auto scan_result = scan(text); + auto scan_result = scan(session.text); for(auto& mod_name: scan_result.modules) { if(mod_name.empty()) continue; bool found = false; - for(auto& [pid, name]: path_to_module) { + for(auto& [pid, name]: workspace.path_to_module) { if(name == mod_name) { // If PCM not already built, try to build it. - if(pcm_paths.find(pid) == pcm_paths.end()) { - if(compile_graph && compile_graph->has_unit(pid)) { - co_await compile_graph->compile_deps(pid); + if(workspace.pcm_paths.find(pid) == workspace.pcm_paths.end()) { + if(workspace.compile_graph && workspace.compile_graph->has_unit(pid)) { + co_await workspace.compile_graph->compile_deps(pid); } } found = true; @@ -831,9 +854,9 @@ et::task Compiler::ensure_deps(std::uint32_t path_id, } // Build or reuse PCH. - auto pch_ok = co_await ensure_pch(path_id, path, text, directory, arguments); + auto pch_ok = co_await ensure_pch(session, directory, arguments); if(pch_ok) { - if(auto pch_it = pch_states.find(path_id); pch_it != pch_states.end()) { + if(auto pch_it = workspace.pch_cache.find(path_id); pch_it != workspace.pch_cache.end()) { pch = {pch_it->second.path, pch_it->second.bound}; } } @@ -845,194 +868,35 @@ et::task Compiler::ensure_deps(std::uint32_t path_id, co_return true; } -bool Compiler::is_stale(std::uint32_t path_id) { - auto ast_deps_it = ast_deps.find(path_id); - if(ast_deps_it != ast_deps.end() && deps_changed(path_pool, ast_deps_it->second)) +bool Compiler::is_stale(const Session& session) { + if(session.ast_deps.has_value() && deps_changed(workspace.path_pool, *session.ast_deps)) return true; - auto pch_it = pch_states.find(path_id); - if(pch_it != pch_states.end() && deps_changed(path_pool, pch_it->second.deps)) - return true; - - return false; -} - -void Compiler::record_deps(std::uint32_t path_id, llvm::ArrayRef deps) { - ast_deps[path_id] = capture_deps_snapshot(path_pool, deps); -} - -void Compiler::on_file_closed(std::uint32_t path_id) { - if(compile_graph && compile_graph->has_unit(path_id)) { - compile_graph->update(path_id); - } - pch_states.erase(path_id); - ast_deps.erase(path_id); -} - -llvm::SmallVector Compiler::on_file_saved(std::uint32_t path_id) { - llvm::SmallVector dirtied; - if(compile_graph) { - auto result = compile_graph->update(path_id); - for(auto id: result) { - dirtied.push_back(id); - pcm_paths.erase(id); - pcm_states.erase(id); - } - } - return dirtied; -} - -std::string Compiler::uri_to_path(const std::string& uri) { - auto parsed = lsp::URI::parse(uri); - if(parsed.has_value()) { - auto path = parsed->file_path(); - if(path.has_value()) { - return std::move(*path); - } - } - return uri; -} - -void Compiler::open_document(const std::string& uri, std::string text, int version) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); - - auto& doc = documents[path_id]; - doc.version = version; - doc.text = std::move(text); - doc.generation++; - - LOG_DEBUG("didOpen: {} (v{})", path, version); -} - -void Compiler::apply_changes(const protocol::DidChangeTextDocumentParams& params) { - auto path = uri_to_path(params.text_document.uri); - auto path_id = path_pool.intern(path); - - auto it = documents.find(path_id); - if(it == documents.end()) - return; - - auto& doc = it->second; - doc.version = params.text_document.version; - - for(auto& change: params.content_changes) { - std::visit( - [&](auto& c) { - using T = std::remove_cvref_t; - if constexpr(std::is_same_v) { - doc.text = c.text; - } else { - auto& range = c.range; - lsp::PositionMapper mapper(doc.text, lsp::PositionEncoding::UTF16); - auto start = mapper.to_offset(range.start); - auto end = mapper.to_offset(range.end); - if(start && end && *start <= *end) { - doc.text.replace(*start, *end - *start, c.text); - } - } - }, - change); - } - - doc.generation++; - doc.ast_dirty = true; - - LOG_DEBUG("didChange: path={} version={} gen={}", path, doc.version, doc.generation); - - worker::DocumentUpdateParams update; - update.path = path; - update.version = doc.version; - pool.notify_stateful(path_id, update); -} - -std::uint32_t Compiler::close_document(const std::string& uri) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); - - on_file_closed(path_id); - indexer.remove_open_file(path_id, path); - pool.notify_stateful(path_id, worker::EvictParams{path}); - documents.erase(path_id); - clear_diagnostics(uri); - - LOG_DEBUG("didClose: {}", path); - return path_id; -} - -llvm::SmallVector Compiler::on_save(const std::string& uri) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); - - llvm::SmallVector to_index; - - auto dirtied = on_file_saved(path_id); - for(auto dirty_id: dirtied) { - auto doc_it = documents.find(dirty_id); - if(doc_it != documents.end()) { - doc_it->second.ast_dirty = true; - } else { - to_index.push_back(dirty_id); - } - } - - llvm::SmallVector stale_headers; - invalidate_host_contexts(path_id, stale_headers); - for(auto hdr_id: stale_headers) { - auto doc_it = documents.find(hdr_id); - if(doc_it != documents.end()) { - doc_it->second.ast_dirty = true; - LOG_DEBUG("didSave: invalidated header context for path_id={}", hdr_id); - } + // Check PCH staleness via the session's pch_ref. + if(session.pch_ref.has_value()) { + auto pch_it = workspace.pch_cache.find(session.pch_ref->path_id); + if(pch_it != workspace.pch_cache.end() && + deps_changed(workspace.path_pool, pch_it->second.deps)) + return true; } - LOG_DEBUG("didSave: {}", uri); - return to_index; -} - -bool Compiler::is_file_open(std::uint32_t path_id) const { - return documents.count(path_id); -} - -const DocumentState* Compiler::get_document(std::uint32_t path_id) const { - auto it = documents.find(path_id); - return it != documents.end() ? &it->second : nullptr; -} - -void Compiler::publish_diagnostics(const std::string& uri, - int version, - const et::serde::RawValue& diagnostics_json) { - std::vector diagnostics; - if(!diagnostics_json.empty()) { - auto status = et::serde::json::from_json(diagnostics_json.data, diagnostics); - if(!status) { - LOG_WARN("Failed to deserialize diagnostics JSON for {}", uri); - } - } - protocol::PublishDiagnosticsParams params; - params.uri = uri; - params.version = version; - params.diagnostics = std::move(diagnostics); - peer.send_notification(params); + return false; } -void Compiler::clear_diagnostics(const std::string& uri) { - protocol::PublishDiagnosticsParams params; - params.uri = uri; - params.diagnostics = {}; - peer.send_notification(params); +void Compiler::record_deps(Session& session, llvm::ArrayRef deps) { + session.ast_deps = capture_deps_snapshot(workspace.path_pool, deps); } /// Pull-based compilation entry point for user-opened files. /// -/// Called lazily by forward_stateful() / forward_stateless() before every +/// Called lazily by forward_query() / forward_build() before every /// feature request (hover, semantic tokens, etc.). Guarantees that when it /// returns true the stateful worker assigned to `path_id` holds an up-to-date /// AST and diagnostics have been published to the client. /// /// Lifecycle overview (pull-based model): /// -/// didOpen / didChange – only update DocumentState, mark ast_dirty +/// didOpen / didChange – only update Session, mark ast_dirty /// didSave – mark dependents dirty, queue indexing /// feature request arrives – calls ensure_compiled() first /// 1. Fast-path exit if AST is already clean (!ast_dirty). @@ -1051,40 +915,30 @@ void Compiler::clear_diagnostics(const std::string& uri) { /// task via loop.schedule(); subsequent ones wait on the shared event. /// The detached task cannot be cancelled by LSP $/cancelRequest, preventing /// the race where cancellation wakes all waiters and they all start compiles. -et::task Compiler::ensure_compiled(std::uint32_t path_id) { - auto it = documents.find(path_id); - if(it == documents.end()) { - LOG_WARN("ensure_compiled: doc not found for path_id={} path={}", - path_id, - path_pool.resolve(path_id)); - co_return false; - } +et::task Compiler::ensure_compiled(Session& session) { + auto path_id = session.path_id; - auto& doc = it->second; LOG_DEBUG("ensure_compiled: path_id={} version={} gen={} ast_dirty={}", path_id, - doc.version, - doc.generation, - doc.ast_dirty); + session.version, + session.generation, + session.ast_dirty); - if(!doc.ast_dirty) { - if(!is_stale(path_id)) { + if(!session.ast_dirty) { + if(!is_stale(session)) { co_return true; } - doc.ast_dirty = true; + session.ast_dirty = true; } // If another compile is already in flight, wait for it. // This co_await may be cancelled by LSP $/cancelRequest — that's fine, // it just means this particular feature request is abandoned. The // detached compile task keeps running independently. - while(it->second.compiling) { - auto pending = it->second.compiling; + while(session.compiling) { + auto pending = session.compiling; co_await pending->done.wait(); - it = documents.find(path_id); - if(it == documents.end()) - co_return false; - if(!it->second.ast_dirty) + if(!session.ast_dirty) co_return true; } @@ -1093,80 +947,82 @@ et::task Compiler::ensure_compiled(std::uint32_t path_id) { // to LSP $/cancelRequest cancellation. This eliminates the race where // cancellation fires the RAII guard, waking all waiters simultaneously // and causing them all to start new compiles. - auto pending_compile = std::make_shared(); - it->second.compiling = pending_compile; + auto pending_compile = std::make_shared(); + session.compiling = pending_compile; LOG_INFO("ensure_compiled: launching detached compile path_id={} gen={}", path_id, - doc.generation); + session.generation); + // Capture path_id by value so the detached lambda can re-lookup the session + // from the sessions map after co_await (DenseMap may invalidate pointers). loop.schedule([](Compiler* self, std::uint32_t pid, - std::shared_ptr pc) -> et::task<> { + std::shared_ptr pc) -> et::task<> { + // Re-lookup session from the sessions map (pointer may have been + // invalidated by DenseMap growth during co_await). + auto find_session = [&]() -> Session* { + auto it = self->sessions.find(pid); + return it != self->sessions.end() ? &it->second : nullptr; + }; + + auto* sess = find_session(); + if(!sess) { + pc->done.set(); + co_return; + } + auto finish_compile = [&]() { - if(auto it = self->documents.find(pid); it != self->documents.end()) { - if(it->second.compiling == pc) { - it->second.compiling.reset(); - } + auto* s = find_session(); + if(s && s->compiling == pc) { + s->compiling.reset(); } LOG_INFO("ensure_compiled: finish_compile (detached) path_id={}", pid); pc->done.set(); }; - auto it = self->documents.find(pid); - if(it == self->documents.end()) { - finish_compile(); - co_return; - } - - auto gen = it->second.generation; + auto gen = sess->generation; LOG_INFO("ensure_compiled: starting compile (detached) path_id={} gen={}", pid, gen); - auto file_path = std::string(self->path_pool.resolve(pid)); + auto file_path = std::string(self->workspace.path_pool.resolve(pid)); auto uri = lsp::URI::from_file_path(file_path); std::string uri_str = uri.has_value() ? uri->str() : file_path; worker::CompileParams params; params.path = file_path; - params.version = it->second.version; - params.text = it->second.text; - if(!self->fill_compile_args(file_path, params.directory, params.arguments)) { + params.version = sess->version; + params.text = sess->text; + if(!self->fill_compile_args(file_path, params.directory, params.arguments, sess)) { finish_compile(); co_return; } - if(!co_await self->ensure_deps(pid, - params.path, - params.text, - params.directory, - params.arguments, - params.pch, - params.pcms)) { + if(!co_await self + ->ensure_deps(*sess, params.directory, params.arguments, params.pch, params.pcms)) { LOG_WARN("Dependency preparation failed for {}, skipping compile", uri_str); finish_compile(); co_return; } - it = self->documents.find(pid); - if(it == self->documents.end()) { - finish_compile(); + // Re-lookup after co_await (DenseMap may have grown). + sess = find_session(); + if(!sess) { + pc->done.set(); co_return; } auto result = co_await self->pool.send_stateful(pid, params); - // Re-lookup: the document may have been closed while we were compiling. - it = self->documents.find(pid); - if(it == self->documents.end()) { - finish_compile(); + // Re-lookup after co_await. + sess = find_session(); + if(!sess) { + pc->done.set(); co_return; } - auto& doc2 = it->second; - - if(doc2.generation != gen) { + if(sess->generation != gen) { LOG_INFO("ensure_compiled: generation mismatch ({} vs {}) for {}", - doc2.generation, + sess->generation, gen, uri_str); finish_compile(); @@ -1180,9 +1036,9 @@ et::task Compiler::ensure_compiled(std::uint32_t path_id) { co_return; } - doc2.ast_dirty = false; + sess->ast_dirty = false; pc->succeeded = true; - self->record_deps(pid, result.value().deps); + self->record_deps(*sess, result.value().deps); // Store open file index from the stateful worker's TUIndex. if(!result.value().tu_index_data.empty()) { @@ -1190,15 +1046,16 @@ et::task Compiler::ensure_compiled(std::uint32_t path_id) { OpenFileIndex ofi; ofi.file_index = std::move(tu_index.main_file_index); ofi.symbols = std::move(tu_index.symbols); - ofi.content = doc2.text; - self->indexer.set_open_file(pid, file_path, std::move(ofi)); + ofi.content = sess->text; + ofi.mapper.emplace(ofi.content, lsp::PositionEncoding::UTF16); + sess->file_index = std::move(ofi); } finish_compile(); // Publish diagnostics AFTER marking compile as done, so that concurrent - // forward_stateful() calls can proceed immediately. - self->publish_diagnostics(uri_str, doc2.version, result.value().diagnostics); + // forward_query() calls can proceed immediately. + self->publish_diagnostics(uri_str, sess->version, result.value().diagnostics); if(self->on_indexing_needed) self->on_indexing_needed(); }(this, path_id, pending_compile)); @@ -1207,23 +1064,18 @@ et::task Compiler::ensure_compiled(std::uint32_t path_id) { // by LSP $/cancelRequest, the detached task continues unaffected. co_await pending_compile->done.wait(); - it = documents.find(path_id); - if(it == documents.end()) - co_return false; - - co_return !it->second.ast_dirty; + co_return !session.ast_dirty; } -Compiler::RawResult Compiler::forward_query(worker::QueryKind kind, const std::string& uri) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); +Compiler::RawResult Compiler::forward_query(worker::QueryKind kind, Session& session) { + auto path_id = session.path_id; + auto path = std::string(workspace.path_pool.resolve(path_id)); - if(!co_await ensure_compiled(path_id)) { + if(!co_await ensure_compiled(session)) { co_return serde_raw{"null"}; } - auto dit = documents.find(path_id); - if(dit != documents.end() && dit->second.ast_dirty) { + if(session.ast_dirty) { co_return serde_raw{"null"}; } @@ -1236,21 +1088,20 @@ Compiler::RawResult Compiler::forward_query(worker::QueryKind kind, const std::s } Compiler::RawResult Compiler::forward_query(worker::QueryKind kind, - const std::string& uri, - const protocol::Position& position) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); + const protocol::Position& position, + Session& session) { + auto path_id = session.path_id; + auto path = std::string(workspace.path_pool.resolve(path_id)); - if(!co_await ensure_compiled(path_id)) { + if(!co_await ensure_compiled(session)) { co_return serde_raw{"null"}; } - auto doc_it = documents.find(path_id); - if(doc_it == documents.end() || doc_it->second.ast_dirty) { + if(session.ast_dirty) { co_return serde_raw{"null"}; } - lsp::PositionMapper mapper(doc_it->second.text, lsp::PositionEncoding::UTF16); + lsp::PositionMapper mapper(session.text, lsp::PositionEncoding::UTF16); auto offset = mapper.to_offset(position); if(!offset) co_return serde_raw{"null"}; @@ -1264,28 +1115,21 @@ Compiler::RawResult Compiler::forward_query(worker::QueryKind kind, } Compiler::RawResult Compiler::forward_build(worker::BuildKind kind, - const std::string& uri, - const protocol::Position& position) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); - - auto doc_it = documents.find(path_id); - if(doc_it == documents.end()) { - co_return serde_raw{}; - } - - auto& doc = doc_it->second; + const protocol::Position& position, + Session& session) { + auto path_id = session.path_id; + auto path = std::string(workspace.path_pool.resolve(path_id)); worker::BuildParams wp; wp.kind = kind; wp.file = path; - wp.version = doc.version; - wp.text = doc.text; - if(!fill_compile_args(path, wp.directory, wp.arguments)) { + wp.version = session.version; + wp.text = session.text; + if(!fill_compile_args(path, wp.directory, wp.arguments, &session)) { co_return serde_raw{}; } - if(!co_await ensure_deps(path_id, path, wp.text, wp.directory, wp.arguments, wp.pch, wp.pcms)) { + if(!co_await ensure_deps(session, wp.directory, wp.arguments, wp.pch, wp.pcms)) { co_return serde_raw{}; } @@ -1302,27 +1146,25 @@ Compiler::RawResult Compiler::forward_build(worker::BuildKind kind, co_return std::move(result.value().result_json); } -Compiler::RawResult Compiler::handle_completion(const std::string& uri, - const protocol::Position& position) { - auto path = uri_to_path(uri); - auto path_id = path_pool.intern(path); - auto doc_it = documents.find(path_id); - if(doc_it != documents.end()) { - lsp::PositionMapper mapper(doc_it->second.text, lsp::PositionEncoding::UTF16); - auto offset = mapper.to_offset(position); - if(offset) { - auto pctx = detect_completion_context(doc_it->second.text, *offset); - if(pctx.kind == CompletionContext::IncludeQuoted || - pctx.kind == CompletionContext::IncludeAngled) { - co_return complete_include(pctx, path); - } - if(pctx.kind == CompletionContext::Import) { - co_return complete_import(pctx); - } +Compiler::RawResult Compiler::handle_completion(const protocol::Position& position, + Session& session) { + auto path_id = session.path_id; + auto path = std::string(workspace.path_pool.resolve(path_id)); + + lsp::PositionMapper mapper(session.text, lsp::PositionEncoding::UTF16); + auto offset = mapper.to_offset(position); + if(offset) { + auto pctx = detect_completion_context(session.text, *offset); + if(pctx.kind == CompletionContext::IncludeQuoted || + pctx.kind == CompletionContext::IncludeAngled) { + co_return complete_include(pctx, path); + } + if(pctx.kind == CompletionContext::Import) { + co_return complete_import(pctx); } } - co_return co_await forward_build(worker::BuildKind::Completion, uri, position); + co_return co_await forward_build(worker::BuildKind::Completion, position, session); } PreambleCompletionContext Compiler::detect_completion_context(const std::string& text, @@ -1452,7 +1294,7 @@ et::serde::RawValue Compiler::complete_import(const PreambleCompletionContext& c std::vector items; llvm::StringRef prefix_ref(ctx.prefix); - for(auto& [path_id, module_name]: path_to_module) { + for(auto& [path_id, module_name]: workspace.path_to_module) { llvm::StringRef name_ref(module_name); if(!name_ref.starts_with(prefix_ref)) continue; diff --git a/src/server/compiler.h b/src/server/compiler.h index f3012e4b7..afada9815 100644 --- a/src/server/compiler.h +++ b/src/server/compiler.h @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -13,12 +12,9 @@ #include "eventide/ipc/lsp/protocol.h" #include "eventide/ipc/peer.h" #include "eventide/serde/serde/raw_value.h" -#include "server/compile_graph.h" -#include "server/config.h" -#include "server/indexer.h" +#include "server/session.h" #include "server/worker_pool.h" -#include "support/path_pool.h" -#include "syntax/dependency_graph.h" +#include "server/workspace.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" @@ -30,60 +26,6 @@ namespace clice { namespace et = eventide; namespace protocol = et::ipc::protocol; -/// State of a document opened by the client. -struct DocumentState { - int version = 0; - std::string text; - std::uint64_t generation = 0; - bool ast_dirty = true; - - /// Non-null while a compile is in flight. Callers wait on the event; - /// the compile task runs independently and cannot be cancelled by LSP - /// $/cancelRequest. - struct PendingCompile { - et::event done; - bool succeeded = false; - }; - - std::shared_ptr compiling; -}; - -/// Context for compiling a header file that lacks its own CDB entry. -struct HeaderFileContext { - std::uint32_t host_path_id; /// Source file acting as host - std::string preamble_path; /// Path to generated preamble file on disk - std::uint64_t preamble_hash; /// Hash of preamble content for staleness -}; - -/// Two-layer staleness snapshot for compilation artifacts (PCH, AST, etc.). -/// -/// Layer 1 (fast): compare each file's current mtime against build_at. -/// If all mtimes <= build_at, the artifact is fresh (zero I/O beyond stat). -/// -/// Layer 2 (precise): for files whose mtime changed, re-hash their content -/// and compare against the stored hash. If the hash matches, the file was -/// "touched" but not actually modified — skip the rebuild. -struct DepsSnapshot { - llvm::SmallVector path_ids; - llvm::SmallVector hashes; - std::int64_t build_at = 0; -}; - -/// Cached PCH state for a single source file. -struct PCHState { - std::string path; - std::uint32_t bound = 0; - std::uint64_t hash = 0; - DepsSnapshot deps; - std::shared_ptr building; -}; - -/// Cached PCM state for a single module file. -struct PCMState { - std::string path; - DepsSnapshot deps; -}; - enum class CompletionContext { None, IncludeQuoted, IncludeAngled, Import }; struct PreambleCompletionContext { @@ -91,138 +33,109 @@ struct PreambleCompletionContext { std::string prefix; }; -/// Manages the full compilation lifecycle: document state, compilation -/// artifacts (PCH/PCM cache), compile argument resolution, header context, -/// and feature request forwarding to workers. +/// Stateless compilation service. +/// +/// Compiler does not own any persistent state. It holds references to +/// Workspace (project-wide shared state) and WorkerPool (child processes), +/// both owned by MasterServer. Per-file state lives in Session objects +/// managed by the caller. /// -/// MasterServer delegates all document and compilation operations here, -/// keeping itself as a pure LSP handler registration layer. +/// Document lifecycle (open/change/close) is handled directly by +/// MasterServer on the sessions map. Compiler is only responsible for: +/// - Compilation (ensure_compiled, ensure_pch, ensure_deps) +/// - Feature request forwarding to workers +/// - Compile argument resolution +/// - Cache persistence +/// - Module/compile graph initialization class Compiler { public: Compiler(et::event_loop& loop, et::ipc::JsonPeer& peer, - PathPool& path_pool, + Workspace& workspace, WorkerPool& pool, - Indexer& indexer, - const CliceConfig& config, - CompilationDatabase& cdb, - DependencyGraph& dep_graph); - + llvm::DenseMap& sessions); ~Compiler(); /// Convert a file:// URI to a local file path. static std::string uri_to_path(const std::string& uri); - /// Document lifecycle — called from MasterServer handlers. - void open_document(const std::string& uri, std::string text, int version); - void apply_changes(const protocol::DidChangeTextDocumentParams& params); - std::uint32_t close_document(const std::string& uri); - llvm::SmallVector on_save(const std::string& uri); + // ----- Cache persistence (operates on Workspace) ----- - /// Document accessors. - bool is_file_open(std::uint32_t path_id) const; - const DocumentState* get_document(std::uint32_t path_id) const; - - /// Cache persistence. void load_cache(); void save_cache(); void cleanup_cache(int max_age_days = 7); - /// Build path_to_module reverse mapping from dependency graph. - void build_module_map(); + // ----- Module / compile graph (operates on Workspace) ----- - /// Initialize the CompileGraph for C++20 module compilation ordering. + void build_module_map(); void init_compile_graph(); + // ----- Compile argument resolution ----- + /// Fill compile arguments for a file (CDB lookup + header context fallback). + /// @param session If non-null, used for header context resolution on open files. bool fill_compile_args(llvm::StringRef path, std::string& directory, - std::vector& arguments); + std::vector& arguments, + Session* session = nullptr); /// Fill PCM paths for all built modules (for background indexing). void fill_pcm_deps(std::unordered_map& pcms, std::uint32_t exclude_path_id = UINT32_MAX) const; - /// Pull-based compilation entry point for user-opened files. - et::task ensure_compiled(std::uint32_t path_id); + // ----- Pull-based compilation ----- + + /// Compile an open file's AST if dirty. On success, updates session's + /// file_index, pch_ref, ast_deps, and publishes diagnostics. + et::task ensure_compiled(Session& session); + + // ----- Feature request forwarding to workers ----- - /// Feature request forwarding to workers. using RawResult = et::task; - /// Forward a stateful AST query to the worker owning this file. - RawResult forward_query(worker::QueryKind kind, const std::string& uri); + RawResult forward_query(worker::QueryKind kind, Session& session); RawResult forward_query(worker::QueryKind kind, - const std::string& uri, - const protocol::Position& position); + const protocol::Position& position, + Session& session); - /// Forward a stateless build request (completion/signatureHelp). RawResult forward_build(worker::BuildKind kind, - const std::string& uri, - const protocol::Position& position); - - /// Completion with preamble-aware include/import handling. - RawResult handle_completion(const std::string& uri, const protocol::Position& position); + const protocol::Position& position, + Session& session); - /// Header context management. - void switch_context(std::uint32_t path_id, std::uint32_t context_path_id); - std::optional get_active_context(std::uint32_t path_id) const; - void invalidate_host_contexts(std::uint32_t host_path_id, - llvm::SmallVectorImpl& stale_headers); - - CompileGraph* compile_graph_ptr() { - return compile_graph.get(); - } - - const llvm::DenseMap& module_map() const { - return path_to_module; - } + RawResult handle_completion(const protocol::Position& position, Session& session); void cancel_all(); - /// Callback invoked when indexing should be scheduled (e.g. after compile success). + void clear_diagnostics(const std::string& uri); + + /// Callback invoked when indexing should be scheduled. std::function on_indexing_needed; private: - /// Compile module dependencies, build/reuse PCH, and fill PCM paths. - et::task ensure_deps(std::uint32_t path_id, - llvm::StringRef path, - const std::string& text, + et::task ensure_deps(Session& session, const std::string& directory, const std::vector& arguments, std::pair& pch, std::unordered_map& pcms); - /// Build or reuse PCH for a source file. - et::task ensure_pch(std::uint32_t path_id, - llvm::StringRef path, - const std::string& text, + et::task ensure_pch(Session& session, const std::string& directory, const std::vector& arguments); - /// Check if a file's AST or PCH deps have changed since last compile. - bool is_stale(std::uint32_t path_id); - - /// Record dependency snapshot after a successful compile. - void record_deps(std::uint32_t path_id, llvm::ArrayRef deps); + bool is_stale(const Session& session); + void record_deps(Session& session, llvm::ArrayRef deps); void publish_diagnostics(const std::string& uri, int version, const et::serde::RawValue& diags); - void clear_diagnostics(const std::string& uri); - - /// Clean up compilation state for a closed file. - void on_file_closed(std::uint32_t path_id); - /// Invalidate artifacts after a file save. - /// Returns path_ids of all files dirtied (via compile_graph cascade). - llvm::SmallVector on_file_saved(std::uint32_t path_id); + std::optional resolve_header_context(std::uint32_t header_path_id, + Session* session); - /// Header context resolution. - std::optional resolve_header_context(std::uint32_t header_path_id); bool fill_header_context_args(llvm::StringRef path, std::uint32_t path_id, std::string& directory, - std::vector& arguments); + std::vector& arguments, + Session* session); - /// Include/import completion helpers. PreambleCompletionContext detect_completion_context(const std::string& text, uint32_t offset); et::serde::RawValue complete_include(const PreambleCompletionContext& ctx, llvm::StringRef path); @@ -231,29 +144,9 @@ class Compiler { private: et::event_loop& loop; et::ipc::JsonPeer& peer; - PathPool& path_pool; + Workspace& workspace; WorkerPool& pool; - Indexer& indexer; - const CliceConfig& config; - CompilationDatabase& cdb; - DependencyGraph& dep_graph; - - /// Open document state, keyed by server-level path_id. - llvm::DenseMap documents; - - /// PCH/PCM cache state. - llvm::DenseMap pch_states; - llvm::DenseMap pcm_states; - llvm::DenseMap pcm_paths; - - /// Module compilation ordering. - llvm::DenseMap path_to_module; - std::unique_ptr compile_graph; - - /// Per-file compilation state. - llvm::DenseMap ast_deps; - llvm::DenseMap header_file_contexts; - llvm::DenseMap active_contexts; + llvm::DenseMap& sessions; }; } // namespace clice diff --git a/src/server/indexer.cpp b/src/server/indexer.cpp index bae5fb207..5ebf71960 100644 --- a/src/server/indexer.cpp +++ b/src/server/indexer.cpp @@ -1,6 +1,5 @@ #include "server/indexer.h" -#include #include #include #include @@ -9,6 +8,7 @@ #include "eventide/ipc/lsp/protocol.h" #include "eventide/ipc/lsp/uri.h" #include "index/tu_index.h" +#include "server/session.h" #include "support/filesystem.h" #include "support/logging.h" @@ -21,64 +21,7 @@ namespace clice { namespace lsp = eventide::ipc::lsp; -/// Find the tightest (innermost) occurrence containing `offset` via binary search. -const static index::Occurrence* lookup_occurrence(const std::vector& occs, - std::uint32_t offset) { - auto it = std::ranges::lower_bound(occs, offset, {}, [](const index::Occurrence& o) { - return o.range.end; - }); - const index::Occurrence* best = nullptr; - while(it != occs.end() && it->range.contains(offset)) { - if(!best || (it->range.end - it->range.begin) < (best->range.end - best->range.begin)) { - best = &*it; - } - ++it; - } - return best; -} - -// ── OpenFileIndex ──────────────────────────────────────────────────────── - -std::optional> - OpenFileIndex::find_occurrence(std::uint32_t offset) const { - if(!mapper) - return std::nullopt; - auto* occ = lookup_occurrence(file_index.occurrences, offset); - if(!occ) - return std::nullopt; - auto start = mapper->to_position(occ->range.begin); - auto end = mapper->to_position(occ->range.end); - if(!start || !end) - return std::nullopt; - return std::pair{ - occ->target, - protocol::Range{*start, *end} - }; -} - -// ── MergedIndexShard ───────────────────────────────────────────────────── - -std::optional> - MergedIndexShard::find_occurrence(std::uint32_t offset) const { - auto* m = mapper(); - if(!m) - return std::nullopt; - std::optional> result; - index.lookup(offset, [&](const index::Occurrence& o) { - auto start = m->to_position(o.range.begin); - auto end = m->to_position(o.range.end); - if(start && end) { - result = { - o.target, - protocol::Range{*start, *end} - }; - } - return false; - }); - return result; -} - -// ── Indexer: data management ───────────────────────────────────────────── +// -- Indexer: data management ------------------------------------------------- void Indexer::merge(const void* tu_index_data, std::size_t size) { auto tu_index = index::TUIndex::from(tu_index_data); @@ -86,12 +29,12 @@ void Indexer::merge(const void* tu_index_data, std::size_t size) { LOG_WARN("Ignoring TUIndex with empty path graph"); return; } - auto file_ids_map = project_index.merge(tu_index); + auto file_ids_map = workspace.project_index.merge(tu_index); auto main_tu_path_id = static_cast(tu_index.graph.paths.size() - 1); auto merge_file_index = [&](std::uint32_t tu_path_id, index::FileIndex& file_idx) { auto global_path_id = file_ids_map[tu_path_id]; - auto& shard = merged_indices[global_path_id]; + auto& shard = workspace.merged_indices[global_path_id]; if(tu_path_id == main_tu_path_id) { std::vector include_locs; @@ -100,7 +43,7 @@ void Indexer::merge(const void* tu_index_data, std::size_t size) { remapped.path_id = file_ids_map[loc.path_id]; include_locs.push_back(remapped); } - auto file_path = project_index.path_pool.path(global_path_id); + auto file_path = workspace.project_index.path_pool.path(global_path_id); llvm::StringRef file_content; std::string file_content_storage; auto buf = llvm::MemoryBuffer::getFile(file_path); @@ -125,7 +68,7 @@ void Indexer::merge(const void* tu_index_data, std::size_t size) { LOG_WARN("Skip merge for path {}: include location not found", global_path_id); return; } - auto header_path = project_index.path_pool.path(global_path_id); + auto header_path = workspace.project_index.path_pool.path(global_path_id); llvm::StringRef header_content; std::string header_content_storage; auto header_buf = llvm::MemoryBuffer::getFile(header_path); @@ -146,7 +89,7 @@ void Indexer::merge(const void* tu_index_data, std::size_t size) { LOG_INFO("Merged TUIndex: {} paths, {} symbols, {} merged_shards", tu_index.graph.paths.size(), tu_index.symbols.size(), - merged_indices.size()); + workspace.merged_indices.size()); } void Indexer::save(llvm::StringRef index_dir) { @@ -164,7 +107,7 @@ void Indexer::save(llvm::StringRef index_dir) { std::error_code write_ec; llvm::raw_fd_ostream os(project_path, write_ec); if(!write_ec) { - project_index.serialize(os); + workspace.project_index.serialize(os); LOG_INFO("Saved ProjectIndex to {}", project_path); } else { LOG_WARN("Failed to save ProjectIndex: {}", write_ec.message()); @@ -179,7 +122,7 @@ void Indexer::save(llvm::StringRef index_dir) { } std::size_t saved = 0; - for(auto& [path_id, shard]: merged_indices) { + for(auto& [path_id, shard]: workspace.merged_indices) { if(!shard.index.need_rewrite()) continue; auto shard_path = path::join(shards_dir, std::to_string(path_id) + ".idx"); @@ -190,7 +133,7 @@ void Indexer::save(llvm::StringRef index_dir) { ++saved; } } - LOG_INFO("Saved {} MergedIndex shards (of {} total)", saved, merged_indices.size()); + LOG_INFO("Saved {} MergedIndex shards (of {} total)", saved, workspace.merged_indices.size()); } void Indexer::load(llvm::StringRef index_dir) { @@ -200,8 +143,8 @@ void Indexer::load(llvm::StringRef index_dir) { auto project_path = path::join(index_dir, "project.idx"); auto buf = llvm::MemoryBuffer::getFile(project_path); if(buf) { - project_index = index::ProjectIndex::from((*buf)->getBufferStart()); - LOG_INFO("Loaded ProjectIndex: {} symbols", project_index.symbols.size()); + workspace.project_index = index::ProjectIndex::from((*buf)->getBufferStart()); + LOG_INFO("Loaded ProjectIndex: {} symbols", workspace.project_index.symbols.size()); } auto shards_dir = path::join(index_dir, "shards"); @@ -216,62 +159,45 @@ void Indexer::load(llvm::StringRef index_dir) { std::uint32_t path_id = 0; if(stem.getAsInteger(10, path_id)) continue; - merged_indices[path_id] = MergedIndexShard{index::MergedIndex::load(it->path())}; + workspace.merged_indices[path_id] = MergedIndexShard{index::MergedIndex::load(it->path())}; } - if(!merged_indices.empty()) { - LOG_INFO("Loaded {} MergedIndex shards", merged_indices.size()); + if(!workspace.merged_indices.empty()) { + LOG_INFO("Loaded {} MergedIndex shards", workspace.merged_indices.size()); } } bool Indexer::need_update(llvm::StringRef file_path) { - auto cache_it = project_index.path_pool.find(file_path); - if(cache_it == project_index.path_pool.cache.end()) + auto cache_it = workspace.project_index.path_pool.find(file_path); + if(cache_it == workspace.project_index.path_pool.cache.end()) return true; - auto merged_it = merged_indices.find(cache_it->second); - if(merged_it == merged_indices.end()) + auto merged_it = workspace.merged_indices.find(cache_it->second); + if(merged_it == workspace.merged_indices.end()) return true; llvm::SmallVector path_mapping; - for(auto& p: project_index.path_pool.paths) { + for(auto& p: workspace.project_index.path_pool.paths) { path_mapping.push_back(p); } return merged_it->second.index.need_update(path_mapping); } -void Indexer::set_open_file(std::uint32_t server_path_id, - llvm::StringRef file_path, - OpenFileIndex index) { - auto& stored = (open_file_indices[server_path_id] = std::move(index)); - stored.mapper.emplace(stored.content, lsp::PositionEncoding::UTF16); - auto proj_cache_it = project_index.path_pool.find(file_path); - if(proj_cache_it != project_index.path_pool.cache.end()) { - open_proj_path_ids.insert(proj_cache_it->second); - } -} - -void Indexer::remove_open_file(std::uint32_t server_path_id, llvm::StringRef file_path) { - open_file_indices.erase(server_path_id); - auto proj_cache_it = project_index.path_pool.find(file_path); - if(proj_cache_it != project_index.path_pool.cache.end()) { - open_proj_path_ids.erase(proj_cache_it->second); - } -} - -// ── Indexer: symbol queries ────────────────────────────────────────────── +// -- Indexer: symbol queries -------------------------------------------------- bool Indexer::find_symbol_info(index::SymbolHash hash, std::string& name, SymbolKind& kind) const { - for(auto& [_, index]: open_file_indices) { - auto it = index.symbols.find(hash); - if(it != index.symbols.end()) { + for(auto& [_, session]: sessions) { + if(!session.file_index) + continue; + auto it = session.file_index->symbols.find(hash); + if(it != session.file_index->symbols.end()) { name = it->second.name; kind = it->second.kind; return true; } } - auto it = project_index.symbols.find(hash); - if(it != project_index.symbols.end()) { + auto it = workspace.project_index.symbols.find(hash); + if(it != workspace.project_index.symbols.end()) { name = it->second.name; kind = it->second.kind; return true; @@ -280,13 +206,11 @@ bool Indexer::find_symbol_info(index::SymbolHash hash, std::string& name, Symbol } Indexer::CursorHit Indexer::resolve_cursor(llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Position& position, - const std::string* doc_text) { - // Try open file index first. - auto it = open_file_indices.find(server_path_id); - if(it != open_file_indices.end()) { - auto& index = it->second; + Session* session) { + // Try the session's open file index first. + if(session && session->file_index) { + auto& index = *session->file_index; if(!index.mapper) return {}; auto offset = index.mapper->to_offset(position); @@ -297,7 +221,8 @@ Indexer::CursorHit Indexer::resolve_cursor(llvm::StringRef path, return {}; } - // Fallback to MergedIndex, using doc_text for position → offset. + // Fallback to MergedIndex, using session text (or reading from disk) for position -> offset. + const std::string* doc_text = session ? &session->text : nullptr; if(!doc_text) return {}; lsp::PositionMapper doc_mapper(*doc_text, lsp::PositionEncoding::UTF16); @@ -305,11 +230,11 @@ Indexer::CursorHit Indexer::resolve_cursor(llvm::StringRef path, if(!offset) return {}; - auto proj_it = project_index.path_pool.find(path); - if(proj_it == project_index.path_pool.cache.end()) + auto proj_it = workspace.project_index.path_pool.find(path); + if(proj_it == workspace.project_index.path_pool.cache.end()) return {}; - auto shard_it = merged_indices.find(proj_it->second); - if(shard_it == merged_indices.end()) + auto shard_it = workspace.merged_indices.find(proj_it->second); + if(shard_it == workspace.merged_indices.end()) return {}; if(auto found = shard_it->second.find_occurrence(*offset)) @@ -318,25 +243,24 @@ Indexer::CursorHit Indexer::resolve_cursor(llvm::StringRef path, } std::vector Indexer::query_relations(llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Position& position, RelationKind kind, - const std::string* doc_text) { - auto hit = resolve_cursor(path, server_path_id, position, doc_text); + Session* session) { + auto hit = resolve_cursor(path, position, session); if(hit.hash == 0) return {}; std::vector locations; - auto sym_it = project_index.symbols.find(hit.hash); - if(sym_it != project_index.symbols.end()) { + auto sym_it = workspace.project_index.symbols.find(hit.hash); + if(sym_it != workspace.project_index.symbols.end()) { for(auto file_id: sym_it->second.reference_files) { - if(open_proj_path_ids.contains(file_id)) + if(is_proj_path_open(file_id)) continue; - auto shard_it = merged_indices.find(file_id); - if(shard_it == merged_indices.end()) + auto shard_it = workspace.merged_indices.find(file_id); + if(shard_it == workspace.merged_indices.end()) continue; - auto uri = lsp::URI::from_file_path(project_index.path_pool.path(file_id)); + auto uri = lsp::URI::from_file_path(workspace.project_index.path_pool.path(file_id)); if(!uri) continue; shard_it->second.find_relations(hit.hash, @@ -348,11 +272,13 @@ std::vector Indexer::query_relations(llvm::StringRef path, } } - for(auto& [id, index]: open_file_indices) { - auto uri = lsp::URI::from_file_path(std::string(path_pool.resolve(id))); + for(auto& [id, sess]: sessions) { + if(!sess.file_index) + continue; + auto uri = lsp::URI::from_file_path(std::string(workspace.path_pool.resolve(id))); if(!uri) continue; - index.find_relations(hit.hash, kind, [&](const auto&, protocol::Range range) { + sess.file_index->find_relations(hit.hash, kind, [&](const auto&, protocol::Range range) { locations.push_back({uri->str(), range}); return true; }); @@ -363,10 +289,9 @@ std::vector Indexer::query_relations(llvm::StringRef path, std::optional Indexer::lookup_symbol(const std::string& uri, llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Position& position, - const std::string* doc_text) { - auto hit = resolve_cursor(path, server_path_id, position, doc_text); + Session* session) { + auto hit = resolve_cursor(path, position, session); if(hit.hash == 0) return std::nullopt; @@ -380,33 +305,35 @@ std::optional Indexer::lookup_symbol(const std::string& uri, std::optional Indexer::find_definition_location(index::SymbolHash hash) { // Open file indices first (fresher data for actively-edited files). - for(auto& [id, index]: open_file_indices) { - auto uri = lsp::URI::from_file_path(std::string(path_pool.resolve(id))); + for(auto& [id, sess]: sessions) { + if(!sess.file_index) + continue; + auto uri = lsp::URI::from_file_path(std::string(workspace.path_pool.resolve(id))); if(!uri) continue; std::optional result; - index.find_relations(hash, - RelationKind::Definition, - [&](const auto&, protocol::Range range) { - result = protocol::Location{uri->str(), range}; - return false; - }); + sess.file_index->find_relations(hash, + RelationKind::Definition, + [&](const auto&, protocol::Range range) { + result = protocol::Location{uri->str(), range}; + return false; + }); if(result) return result; } // Fall back to ProjectIndex reference files. - auto sym_it = project_index.symbols.find(hash); - if(sym_it == project_index.symbols.end()) + auto sym_it = workspace.project_index.symbols.find(hash); + if(sym_it == workspace.project_index.symbols.end()) return std::nullopt; for(auto file_id: sym_it->second.reference_files) { - if(open_proj_path_ids.contains(file_id)) + if(is_proj_path_open(file_id)) continue; - auto shard_it = merged_indices.find(file_id); - if(shard_it == merged_indices.end()) + auto shard_it = workspace.merged_indices.find(file_id); + if(shard_it == workspace.merged_indices.end()) continue; - auto uri = lsp::URI::from_file_path(project_index.path_pool.path(file_id)); + auto uri = lsp::URI::from_file_path(workspace.project_index.path_pool.path(file_id)); if(!uri) continue; std::optional result; @@ -426,10 +353,9 @@ std::optional Indexer::find_definition_location(index::Symbo std::optional Indexer::resolve_hierarchy_item(const std::string& uri, llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Range& range, const std::optional& data, - const std::string* doc_text) { + Session* session) { if(data) { if(auto* int_val = std::get_if(&*data)) { auto hash = static_cast(*int_val); @@ -440,22 +366,22 @@ std::optional } } } - return lookup_symbol(uri, path, server_path_id, range.start, doc_text); + return lookup_symbol(uri, path, range.start, session); } -// ── Indexer: relation collection helpers ───────────────────────────────── +// -- Indexer: relation collection helpers -------------------------------------- void Indexer::collect_grouped_relations( index::SymbolHash hash, RelationKind kind, llvm::DenseMap>& target_ranges) { - auto sym_it = project_index.symbols.find(hash); - if(sym_it != project_index.symbols.end()) { + auto sym_it = workspace.project_index.symbols.find(hash); + if(sym_it != workspace.project_index.symbols.end()) { for(auto file_id: sym_it->second.reference_files) { - if(open_proj_path_ids.contains(file_id)) + if(is_proj_path_open(file_id)) continue; - auto shard_it = merged_indices.find(file_id); - if(shard_it == merged_indices.end()) + auto shard_it = workspace.merged_indices.find(file_id); + if(shard_it == workspace.merged_indices.end()) continue; shard_it->second.find_relations(hash, kind, [&](const auto& r, protocol::Range range) { target_ranges[r.target_symbol].push_back(range); @@ -463,8 +389,10 @@ void Indexer::collect_grouped_relations( }); } } - for(auto& [_, index]: open_file_indices) { - index.find_relations(hash, kind, [&](const auto& r, protocol::Range range) { + for(auto& [_, sess]: sessions) { + if(!sess.file_index) + continue; + sess.file_index->find_relations(hash, kind, [&](const auto& r, protocol::Range range) { target_ranges[r.target_symbol].push_back(range); return true; }); @@ -475,15 +403,15 @@ void Indexer::collect_unique_targets(index::SymbolHash hash, RelationKind kind, llvm::SmallVectorImpl& targets) { llvm::DenseSet seen; - auto sym_it = project_index.symbols.find(hash); - if(sym_it != project_index.symbols.end()) { + auto sym_it = workspace.project_index.symbols.find(hash); + if(sym_it != workspace.project_index.symbols.end()) { for(auto file_id: sym_it->second.reference_files) { - if(open_proj_path_ids.contains(file_id)) + if(is_proj_path_open(file_id)) continue; - auto shard_it = merged_indices.find(file_id); - if(shard_it == merged_indices.end()) + auto shard_it = workspace.merged_indices.find(file_id); + if(shard_it == workspace.merged_indices.end()) continue; - /// No position conversion needed — just collect target symbol hashes. + /// No position conversion needed -- just collect target symbol hashes. shard_it->second.index.lookup(hash, kind, [&](const index::Relation& r) { if(seen.insert(r.target_symbol).second) { targets.push_back(r.target_symbol); @@ -492,9 +420,11 @@ void Indexer::collect_unique_targets(index::SymbolHash hash, }); } } - for(auto& [_, index]: open_file_indices) { - auto rel_it = index.file_index.relations.find(hash); - if(rel_it == index.file_index.relations.end()) + for(auto& [_, sess]: sessions) { + if(!sess.file_index) + continue; + auto rel_it = sess.file_index->file_index.relations.find(hash); + if(rel_it == sess.file_index->file_index.relations.end()) continue; for(auto& r: rel_it->second) { if(r.kind & kind) { @@ -506,7 +436,7 @@ void Indexer::collect_unique_targets(index::SymbolHash hash, } } -// ── Indexer: hierarchy queries ─────────────────────────────────────────── +// -- Indexer: hierarchy queries ------------------------------------------------ /// Resolve a symbol hash into a SymbolInfo with definition location. /// Returns nullopt if the symbol or its definition cannot be found. @@ -603,7 +533,7 @@ std::vector Indexer::search_symbols(llvm::StringRef std::vector results; llvm::DenseSet seen; - for(auto& [hash, symbol]: project_index.symbols) { + for(auto& [hash, symbol]: workspace.project_index.symbols) { if(results.size() >= max_results) break; if(!is_indexable_kind(symbol.kind) || symbol.name.empty()) @@ -622,10 +552,12 @@ std::vector Indexer::search_symbols(llvm::StringRef seen.insert(hash); } - for(auto& [_, index]: open_file_indices) { + for(auto& [_, sess]: sessions) { if(results.size() >= max_results) break; - for(auto& [hash, symbol]: index.symbols) { + if(!sess.file_index) + continue; + for(auto& [hash, symbol]: sess.file_index->symbols) { if(results.size() >= max_results) break; if(seen.contains(hash)) @@ -649,7 +581,7 @@ std::vector Indexer::search_symbols(llvm::StringRef return results; } -// ── Indexer: static utilities ──────────────────────────────────────────── +// -- Indexer: static utilities ------------------------------------------------ protocol::SymbolKind Indexer::to_lsp_symbol_kind(SymbolKind kind) { switch(kind) { diff --git a/src/server/indexer.h b/src/server/indexer.h index 214e10d8b..947e2081d 100644 --- a/src/server/indexer.h +++ b/src/server/indexer.h @@ -1,21 +1,18 @@ #pragma once #include +#include #include #include -#include #include #include "eventide/ipc/lsp/position.h" #include "eventide/ipc/lsp/protocol.h" -#include "index/merged_index.h" -#include "index/project_index.h" #include "semantic/relation_kind.h" #include "semantic/symbol_kind.h" -#include "support/path_pool.h" +#include "server/workspace.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" @@ -25,88 +22,7 @@ namespace et = eventide; namespace protocol = et::ipc::protocol; namespace lsp = et::ipc::lsp; -/// In-memory index for an open file. Kept separate from MergedIndex because -/// open files change frequently, are based on unsaved buffer content, and only -/// need to track the main file (headers are covered by PCH/PCM indexing). -struct OpenFileIndex { - index::FileIndex file_index; - index::SymbolTable symbols; - std::string content; ///< Buffer text at index time (for position mapping). - - /// Cached PositionMapper built from `content`. Avoids re-scanning line - /// offsets on every query. Initialized by Indexer::set_open_file(). - std::optional mapper; - - /// Find the tightest occurrence containing `offset`. - /// Returns (symbol_hash, LSP range) with positions already converted. - std::optional> - find_occurrence(std::uint32_t offset) const; - - /// Iterate relations matching `kind`, calling back with pre-converted ranges. - /// Callback: (const index::Relation&, protocol::Range) -> bool (true = continue). - template - void find_relations(index::SymbolHash hash, RelationKind kind, Fn&& fn) const { - if(!mapper) - return; - auto it = file_index.relations.find(hash); - if(it == file_index.relations.end()) - return; - for(auto& r: it->second) { - if(r.kind & kind) { - auto start = mapper->to_position(r.range.begin); - auto end = mapper->to_position(r.range.end); - if(start && end) { - if(!fn(r, protocol::Range{*start, *end})) - return; - } - } - } - } -}; - -/// Wraps index::MergedIndex with a lazily-cached PositionMapper. -struct MergedIndexShard { - index::MergedIndex index; - mutable std::optional cached_mapper; - - /// Get or lazily build a PositionMapper from the index's stored content. - const lsp::PositionMapper* mapper() const { - if(!cached_mapper) { - auto c = index.content(); - if(!c.empty()) { - cached_mapper.emplace(c, lsp::PositionEncoding::UTF16); - } - } - return cached_mapper ? &*cached_mapper : nullptr; - } - - /// Invalidate the cached mapper (call after merge changes content). - void invalidate_mapper() { - cached_mapper.reset(); - } - - /// Find occurrence at byte offset. - /// Returns (symbol_hash, LSP range) with positions already converted. - std::optional> - find_occurrence(std::uint32_t offset) const; - - /// Iterate relations matching `kind`, calling back with pre-converted ranges. - /// Callback: (const index::Relation&, protocol::Range) -> bool (true = continue). - template - void find_relations(index::SymbolHash hash, RelationKind kind, Fn&& fn) const { - auto* m = mapper(); - if(!m) - return; - index.lookup(hash, kind, [&](const index::Relation& r) { - auto start = m->to_position(r.range.begin); - auto end = m->to_position(r.range.end); - if(start && end) { - return fn(r, protocol::Range{*start, *end}); - } - return true; - }); - } -}; +struct Session; /// Information about a symbol at a given position. struct SymbolInfo { @@ -117,49 +33,52 @@ struct SymbolInfo { protocol::Range range; }; -/// Owns all index state (ProjectIndex, MergedIndex shards, open file indices) -/// and provides query methods for cross-file navigation. +/// Provides query methods for cross-file navigation over Workspace index data +/// and open Session file indices. +/// +/// Indexer does NOT own any index state. All persistent data lives in +/// Workspace (disk-derived ProjectIndex + MergedIndex shards) and per-file +/// data lives in Session (OpenFileIndex from unsaved buffers). /// /// Background indexing scheduling is driven by MasterServer; Indexer is the -/// pure data + query layer. +/// pure query layer. class Indexer { public: - explicit Indexer(PathPool& path_pool) : path_pool(path_pool) {} - - /// Merge a TUIndex result into ProjectIndex and MergedIndex shards. + /// @param workspace Project-wide persistent state (ProjectIndex, MergedIndex shards). + /// @param sessions Map of open editing sessions, keyed by server-level path_id. + /// @param is_file_open Callback that returns true if a *project-level* path_id + /// has an active Session. Used to skip stale MergedIndex shards when + /// fresher OpenFileIndex data is available. + Indexer(Workspace& workspace, + llvm::DenseMap& sessions, + std::function is_file_open = {}) : + workspace(workspace), sessions(sessions), is_file_open(std::move(is_file_open)) {} + + /// Merge a TUIndex result into Workspace's ProjectIndex and MergedIndex shards. void merge(const void* tu_index_data, std::size_t size); - /// Save ProjectIndex and MergedIndex shards to disk. + /// Save Workspace's ProjectIndex and MergedIndex shards to disk. void save(llvm::StringRef index_dir); - /// Load ProjectIndex and MergedIndex shards from disk. + /// Load Workspace's ProjectIndex and MergedIndex shards from disk. void load(llvm::StringRef index_dir); /// Check whether a file needs re-indexing (stale or missing shard). bool need_update(llvm::StringRef file_path); - /// Store or replace the open file index for a server-level path_id. - void set_open_file(std::uint32_t server_path_id, - llvm::StringRef file_path, - OpenFileIndex index); - - /// Remove the open file index and untrack project-level path_id. - void remove_open_file(std::uint32_t server_path_id, llvm::StringRef file_path); - /// Query relations (Definition, Reference, etc.) for a symbol at cursor. - /// @param doc_text Fallback text when the file has no open file index yet. + /// @param session Active Session for this file, or nullptr to use MergedIndex only. std::vector query_relations(llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Position& position, RelationKind kind, - const std::string* doc_text); + Session* session); /// Look up symbol info (hash, name, kind, range) at a cursor position. + /// @param session Active Session for this file, or nullptr. std::optional lookup_symbol(const std::string& uri, llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Position& position, - const std::string* doc_text); + Session* session); /// Find the definition location of a symbol by hash. std::optional find_definition_location(index::SymbolHash hash); @@ -168,12 +87,12 @@ class Indexer { bool find_symbol_info(index::SymbolHash hash, std::string& name, SymbolKind& kind) const; /// Resolve a hierarchy item (from stored data or by position lookup). + /// @param session Active Session for this file, or nullptr. std::optional resolve_hierarchy_item(const std::string& uri, llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Range& range, const std::optional& data, - const std::string* doc_text); + Session* session); /// Find incoming calls to a function. std::vector find_incoming_calls(index::SymbolHash hash); @@ -198,11 +117,6 @@ class Indexer { static protocol::CallHierarchyItem build_call_hierarchy_item(const SymbolInfo& info); static protocol::TypeHierarchyItem build_type_hierarchy_item(const SymbolInfo& info); - /// Direct access to ProjectIndex for background indexing. - index::ProjectIndex& project_index_ref() { - return project_index; - } - private: /// Result of resolving a symbol at a cursor position. struct CursorHit { @@ -210,12 +124,11 @@ class Indexer { protocol::Range range{}; }; - /// Resolve the symbol at (position), checking open file index first then - /// falling back to MergedIndex. + /// Resolve the symbol at (position), checking Session's file_index first + /// then falling back to Workspace's MergedIndex. CursorHit resolve_cursor(llvm::StringRef path, - std::uint32_t server_path_id, const protocol::Position& position, - const std::string* doc_text); + Session* session); /// Collect relations grouped by target symbol, across all index sources. void collect_grouped_relations( @@ -231,21 +144,19 @@ class Indexer { /// Resolve a symbol hash into a SymbolInfo with definition location. std::optional resolve_symbol(index::SymbolHash hash); -private: - PathPool& path_pool; - - /// Global symbol table and path pool shared across all TUs. - index::ProjectIndex project_index; - - /// Per-file MergedIndex shards (keyed by project-level path_id). - llvm::DenseMap merged_indices; + /// Check whether a project-level path_id has an active Session. + bool is_proj_path_open(std::uint32_t proj_path_id) const { + return is_file_open && is_file_open(proj_path_id); + } - /// In-memory indices for currently open files (keyed by server-level path_id). - llvm::DenseMap open_file_indices; +private: + Workspace& workspace; + llvm::DenseMap& sessions; - /// Project-level path_ids of open files, used to skip stale MergedIndex - /// shards when fresher OpenFileIndex data is available. - llvm::DenseSet open_proj_path_ids; + /// Callback that checks if a *project-level* path_id has an active + /// Session. Set by the owner (e.g. MasterServer) to bridge the + /// server-path-id-keyed sessions map to project-level path_ids. + std::function is_file_open; }; } // namespace clice diff --git a/src/server/master_server.cpp b/src/server/master_server.cpp index efe85988a..db5938d5f 100644 --- a/src/server/master_server.cpp +++ b/src/server/master_server.cpp @@ -3,7 +3,10 @@ #include #include #include +#include +#include +#include "eventide/ipc/lsp/position.h" #include "eventide/ipc/lsp/protocol.h" #include "eventide/ipc/lsp/uri.h" #include "eventide/reflection/enum.h" @@ -34,9 +37,9 @@ static serde_raw to_raw(const T& value) { } MasterServer::MasterServer(et::event_loop& loop, et::ipc::JsonPeer& peer, std::string self_path) : - loop(loop), peer(peer), pool(loop), indexer(path_pool), - compiler(loop, peer, path_pool, pool, indexer, config, cdb, dependency_graph), - self_path(std::move(self_path)) {} + loop(loop), peer(peer), pool(loop), + indexer(workspace, sessions, [this](uint32_t id) { return sessions.contains(id); }), + compiler(loop, peer, workspace, pool, sessions), self_path(std::move(self_path)) {} MasterServer::~MasterServer() = default; @@ -44,16 +47,18 @@ et::task<> MasterServer::load_workspace() { if(workspace_root.empty()) co_return; - if(!config.cache_dir.empty()) { - auto ec = llvm::sys::fs::create_directories(config.cache_dir); + if(!workspace.config.cache_dir.empty()) { + auto ec = llvm::sys::fs::create_directories(workspace.config.cache_dir); if(ec) { - LOG_WARN("Failed to create cache directory {}: {}", config.cache_dir, ec.message()); + LOG_WARN("Failed to create cache directory {}: {}", + workspace.config.cache_dir, + ec.message()); } else { - LOG_INFO("Cache directory: {}", config.cache_dir); + LOG_INFO("Cache directory: {}", workspace.config.cache_dir); } for(auto* subdir: {"cache/pch", "cache/pcm"}) { - auto dir = path::join(config.cache_dir, subdir); + auto dir = path::join(workspace.config.cache_dir, subdir); auto ec2 = llvm::sys::fs::create_directories(dir); if(ec2) { LOG_WARN("Failed to create {}: {}", dir, ec2.message()); @@ -67,12 +72,12 @@ et::task<> MasterServer::load_workspace() { } std::string cdb_path; - if(!config.compile_commands_path.empty()) { - if(llvm::sys::fs::exists(config.compile_commands_path)) { - cdb_path = config.compile_commands_path; + if(!workspace.config.compile_commands_path.empty()) { + if(llvm::sys::fs::exists(workspace.config.compile_commands_path)) { + cdb_path = workspace.config.compile_commands_path; } else { LOG_WARN("Configured compile_commands_path not found: {}", - config.compile_commands_path); + workspace.config.compile_commands_path); } } @@ -91,11 +96,11 @@ et::task<> MasterServer::load_workspace() { co_return; } - auto count = cdb.load(cdb_path); + auto count = workspace.cdb.load(cdb_path); LOG_INFO("Loaded CDB from {} with {} entries", cdb_path, count); - auto report = scan_dependency_graph(cdb, path_pool, dependency_graph); - dependency_graph.build_reverse_map(); + auto report = scan_dependency_graph(workspace.cdb, workspace.path_pool, workspace.dep_graph); + workspace.dep_graph.build_reverse_map(); auto unresolved = report.includes_found - report.includes_resolved; double accuracy = @@ -118,12 +123,12 @@ et::task<> MasterServer::load_workspace() { } compiler.build_module_map(); - indexer.load(config.index_dir); + indexer.load(workspace.config.index_dir); - if(config.enable_indexing) { - for(auto& entry: cdb.get_entries()) { - auto file = cdb.resolve_path(entry.file); - auto server_id = path_pool.intern(file); + if(workspace.config.enable_indexing) { + for(auto& entry: workspace.cdb.get_entries()) { + auto file = workspace.cdb.resolve_path(entry.file); + auto server_id = workspace.path_pool.intern(file); index_queue.push_back(server_id); } if(!index_queue.empty()) { @@ -136,14 +141,14 @@ et::task<> MasterServer::load_workspace() { } void MasterServer::schedule_indexing() { - if(!config.enable_indexing || indexing_active || indexing_scheduled) + if(!workspace.config.enable_indexing || indexing_active || indexing_scheduled) return; indexing_scheduled = true; if(!index_idle_timer) { index_idle_timer = std::make_shared(et::timer::create(loop)); } - index_idle_timer->start(std::chrono::milliseconds(config.idle_timeout_ms)); + index_idle_timer->start(std::chrono::milliseconds(workspace.config.idle_timeout_ms)); loop.schedule(run_background_indexing()); } @@ -165,10 +170,10 @@ et::task<> MasterServer::run_background_indexing() { auto server_path_id = index_queue[index_queue_pos]; index_queue_pos++; - auto file_path = std::string(path_pool.resolve(server_path_id)); + auto file_path = std::string(workspace.path_pool.resolve(server_path_id)); /// Skip open files — their index comes from the stateful worker. - if(compiler.is_file_open(server_path_id)) { + if(sessions.contains(server_path_id)) { continue; } @@ -178,7 +183,7 @@ et::task<> MasterServer::run_background_indexing() { worker::BuildParams params; params.kind = worker::BuildKind::Index; params.file = file_path; - if(!compiler.fill_compile_args(file_path, params.directory, params.arguments)) + if(!compiler.fill_compile_args(file_path, params.directory, params.arguments, nullptr)) continue; compiler.fill_pcm_deps(params.pcms); @@ -203,7 +208,7 @@ et::task<> MasterServer::run_background_indexing() { indexing_active = false; LOG_INFO("Background indexing complete: {} files processed", processed); - indexer.save(config.index_dir); + indexer.save(workspace.config.index_dir); } void MasterServer::register_handlers() { @@ -300,27 +305,27 @@ void MasterServer::register_handlers() { }); peer.on_notification([this](const protocol::InitializedParams& params) { - config = CliceConfig::load_from_workspace(workspace_root); + workspace.config = CliceConfig::load_from_workspace(workspace_root); - if(!config.logging_dir.empty()) { + if(!workspace.config.logging_dir.empty()) { auto now = std::chrono::system_clock::now(); auto pid = llvm::sys::Process::getProcessId(); - auto session_dir = - path::join(config.logging_dir, std::format("{:%Y-%m-%d_%H-%M-%S}_{}", now, pid)); + auto session_dir = path::join(workspace.config.logging_dir, + std::format("{:%Y-%m-%d_%H-%M-%S}_{}", now, pid)); logging::file_logger("master", session_dir, logging::options); session_log_dir = session_dir; } LOG_INFO("Server ready (stateful={}, stateless={}, idle={}ms)", - config.stateful_worker_count, - config.stateless_worker_count, - config.idle_timeout_ms); + workspace.config.stateful_worker_count, + workspace.config.stateless_worker_count, + workspace.config.idle_timeout_ms); WorkerPoolOptions pool_opts; pool_opts.self_path = self_path; - pool_opts.stateful_count = config.stateful_worker_count; - pool_opts.stateless_count = config.stateless_worker_count; - pool_opts.worker_memory_limit = config.worker_memory_limit; + pool_opts.stateful_count = workspace.config.stateful_worker_count; + pool_opts.stateless_count = workspace.config.stateless_worker_count; + pool_opts.worker_memory_limit = workspace.config.worker_memory_limit; pool_opts.log_dir = session_log_dir; if(!pool.start(pool_opts)) { LOG_ERROR("Failed to start worker pool"); @@ -348,7 +353,7 @@ void MasterServer::register_handlers() { lifecycle = ServerLifecycle::Exited; LOG_INFO("Exit notification received"); - indexer.save(config.index_dir); + indexer.save(workspace.config.index_dir); compiler.save_cache(); loop.schedule([this]() -> et::task<> { @@ -357,105 +362,225 @@ void MasterServer::register_handlers() { }()); }); - /// Document lifecycle — delegate to Compiler. + /// --------------------------------------------------------------- + /// Document lifecycle — handled directly by MasterServer. + /// --------------------------------------------------------------- + peer.on_notification([this](const protocol::DidOpenTextDocumentParams& params) { if(lifecycle != ServerLifecycle::Ready) return; - compiler.open_document(params.text_document.uri, - params.text_document.text, - params.text_document.version); + + auto path = Compiler::uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + + auto& session = sessions[path_id]; + session.path_id = path_id; + session.version = params.text_document.version; + session.text = params.text_document.text; + session.generation++; + + LOG_DEBUG("didOpen: {} (v{})", path, params.text_document.version); }); peer.on_notification([this](const protocol::DidChangeTextDocumentParams& params) { if(lifecycle != ServerLifecycle::Ready) return; - compiler.apply_changes(params); + + auto path = Compiler::uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + + auto it = sessions.find(path_id); + if(it == sessions.end()) + return; + + auto& session = it->second; + session.version = params.text_document.version; + + for(auto& change: params.content_changes) { + std::visit( + [&](auto& c) { + using T = std::remove_cvref_t; + if constexpr(std::is_same_v) { + session.text = c.text; + } else { + auto& range = c.range; + lsp::PositionMapper mapper(session.text, lsp::PositionEncoding::UTF16); + auto start = mapper.to_offset(range.start); + auto end = mapper.to_offset(range.end); + if(start && end && *start <= *end) { + session.text.replace(*start, *end - *start, c.text); + } + } + }, + change); + } + + session.generation++; + session.ast_dirty = true; + + LOG_DEBUG("didChange: path={} version={} gen={}", + path, + session.version, + session.generation); + + worker::DocumentUpdateParams update; + update.path = path; + update.version = session.version; + pool.notify_stateful(path_id, update); }); peer.on_notification([this](const protocol::DidCloseTextDocumentParams& params) { if(lifecycle != ServerLifecycle::Ready) return; - auto path_id = compiler.close_document(params.text_document.uri); + + auto path = Compiler::uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + + workspace.on_file_closed(path_id); + pool.notify_stateful(path_id, worker::EvictParams{path}); + + // Clear diagnostics for the closed file. + protocol::PublishDiagnosticsParams diag_params; + diag_params.uri = params.text_document.uri; + peer.send_notification(diag_params); + + sessions.erase(path_id); + index_queue.push_back(path_id); schedule_indexing(); + + LOG_DEBUG("didClose: {}", path); }); peer.on_notification([this](const protocol::DidSaveTextDocumentParams& params) { if(lifecycle != ServerLifecycle::Ready) return; - auto to_index = compiler.on_save(params.text_document.uri); - for(auto id: to_index) - index_queue.push_back(id); + + auto path = Compiler::uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + + auto dirtied = workspace.on_file_saved(path_id); + for(auto dirty_id: dirtied) { + if(auto sit = sessions.find(dirty_id); sit != sessions.end()) { + sit->second.ast_dirty = true; + } else { + index_queue.push_back(dirty_id); + } + } + + // Invalidate header contexts for sessions whose host is this file. + for(auto& [hdr_id, session]: sessions) { + if(session.header_context && session.header_context->host_path_id == path_id) { + session.header_context.reset(); + session.ast_dirty = true; + } + } + schedule_indexing(); + + LOG_DEBUG("didSave: {}", path); }); + /// --------------------------------------------------------------- /// Feature requests — stateful forwarding. + /// --------------------------------------------------------------- + peer.on_request([this](RequestContext& ctx, const protocol::HoverParams& params) -> RawResult { - co_return co_await compiler.forward_query( - worker::QueryKind::Hover, - params.text_document_position_params.text_document.uri, - params.text_document_position_params.position); + auto path = Compiler::uri_to_path(params.text_document_position_params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::Hover, + params.text_document_position_params.position, + sit->second); }); - peer.on_request( - [this](RequestContext& ctx, const protocol::SemanticTokensParams& params) -> RawResult { - co_return co_await compiler.forward_query(worker::QueryKind::SemanticTokens, - params.text_document.uri); - }); + peer.on_request([this](RequestContext& ctx, + const protocol::SemanticTokensParams& params) -> RawResult { + auto path = Compiler::uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::SemanticTokens, sit->second); + }); peer.on_request( [this](RequestContext& ctx, const protocol::InlayHintParams& params) -> RawResult { - co_return co_await compiler.forward_query(worker::QueryKind::InlayHints, - params.text_document.uri); + auto path = Compiler::uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::InlayHints, sit->second); }); peer.on_request( [this](RequestContext& ctx, const protocol::FoldingRangeParams& params) -> RawResult { - co_return co_await compiler.forward_query(worker::QueryKind::FoldingRange, - params.text_document.uri); + auto path = Compiler::uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::FoldingRange, sit->second); }); - peer.on_request( - [this](RequestContext& ctx, const protocol::DocumentSymbolParams& params) -> RawResult { - co_return co_await compiler.forward_query(worker::QueryKind::DocumentSymbol, - params.text_document.uri); - }); + peer.on_request([this](RequestContext& ctx, + const protocol::DocumentSymbolParams& params) -> RawResult { + auto path = Compiler::uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::DocumentSymbol, sit->second); + }); peer.on_request( [this](RequestContext& ctx, const protocol::DocumentLinkParams& params) -> RawResult { - co_return co_await compiler.forward_query(worker::QueryKind::DocumentLink, - params.text_document.uri); + auto path = Compiler::uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::DocumentLink, sit->second); }); peer.on_request( [this](RequestContext& ctx, const protocol::CodeActionParams& params) -> RawResult { - co_return co_await compiler.forward_query(worker::QueryKind::CodeAction, - params.text_document.uri); + auto path = Compiler::uri_to_path(params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::CodeAction, sit->second); }); - /// Resolve URI to the context needed for index queries. + /// Helper: resolve URI to path, path_id, and Session pointer. auto resolve_uri = [this](const std::string& uri) { struct Result { std::string path; std::uint32_t path_id; - const std::string* doc_text; + Session* session; }; auto path = Compiler::uri_to_path(uri); - auto path_id = path_pool.intern(path); - auto* doc = compiler.get_document(path_id); - return Result{std::move(path), path_id, doc ? &doc->text : nullptr}; + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + Session* session = (sit != sessions.end()) ? &sit->second : nullptr; + return Result{std::move(path), path_id, session}; }; auto lookup_at = [this, resolve_uri](const std::string& uri, const protocol::Position& pos) { - auto [path, path_id, doc_text] = resolve_uri(uri); - return indexer.lookup_symbol(uri, path, path_id, pos, doc_text); + auto [path, path_id, session] = resolve_uri(uri); + return indexer.lookup_symbol(uri, path, pos, session); }; auto query_at = [this, resolve_uri](const std::string& uri, const protocol::Position& pos, RelationKind kind) -> std::vector { - auto [path, path_id, doc_text] = resolve_uri(uri); - return indexer.query_relations(path, path_id, pos, kind, doc_text); + auto [path, path_id, session] = resolve_uri(uri); + return indexer.query_relations(path, pos, kind, session); }; auto resolve_item = @@ -463,11 +588,14 @@ void MasterServer::register_handlers() { resolve_uri](const std::string& uri, const protocol::Range& range, const std::optional& data) -> std::optional { - auto [path, path_id, doc_text] = resolve_uri(uri); - return indexer.resolve_hierarchy_item(uri, path, path_id, range, data, doc_text); + auto [path, path_id, session] = resolve_uri(uri); + return indexer.resolve_hierarchy_item(uri, path, range, data, session); }; + /// --------------------------------------------------------------- /// Feature requests — index-based with AST fallback. + /// --------------------------------------------------------------- + peer.on_request([this, query_at](RequestContext& ctx, const protocol::DefinitionParams& params) -> RawResult { auto& uri = params.text_document_position_params.text_document.uri; @@ -478,7 +606,14 @@ void MasterServer::register_handlers() { co_return to_raw(result); } - co_return co_await compiler.forward_query(worker::QueryKind::GoToDefinition, uri, pos); + auto path = Compiler::uri_to_path(uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_query(worker::QueryKind::GoToDefinition, + pos, + sit->second); }); peer.on_request([this, query_at](RequestContext& ctx, @@ -515,23 +650,37 @@ void MasterServer::register_handlers() { co_return serde_raw{"null"}; }); + /// --------------------------------------------------------------- /// Feature requests — stateless forwarding. - peer.on_request( - [this](RequestContext& ctx, const protocol::CompletionParams& params) -> RawResult { - co_return co_await compiler.handle_completion( - params.text_document_position_params.text_document.uri, - params.text_document_position_params.position); - }); + /// --------------------------------------------------------------- + + peer.on_request([this](RequestContext& ctx, + const protocol::CompletionParams& params) -> RawResult { + auto path = Compiler::uri_to_path(params.text_document_position_params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.handle_completion(params.text_document_position_params.position, + sit->second); + }); - peer.on_request( - [this](RequestContext& ctx, const protocol::SignatureHelpParams& params) -> RawResult { - co_return co_await compiler.forward_build( - worker::BuildKind::SignatureHelp, - params.text_document_position_params.text_document.uri, - params.text_document_position_params.position); - }); + peer.on_request([this](RequestContext& ctx, + const protocol::SignatureHelpParams& params) -> RawResult { + auto path = Compiler::uri_to_path(params.text_document_position_params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_build(worker::BuildKind::SignatureHelp, + params.text_document_position_params.position, + sit->second); + }); + /// --------------------------------------------------------------- /// Hierarchy queries — index-based. + /// --------------------------------------------------------------- + peer.on_request( [this, lookup_at](RequestContext& ctx, const protocol::CallHierarchyPrepareParams& params) -> RawResult { @@ -623,22 +772,25 @@ void MasterServer::register_handlers() { co_return to_raw(results); }); + /// --------------------------------------------------------------- /// clice/ extension commands. + /// --------------------------------------------------------------- + peer.on_request( "clice/queryContext", [this](RequestContext& ctx, const ext::QueryContextParams& params) -> RawResult { auto path = Compiler::uri_to_path(params.uri); - auto path_id = path_pool.intern(path); + auto path_id = workspace.path_pool.intern(path); int offset_val = std::max(0, params.offset.value_or(0)); constexpr int page_size = 10; ext::QueryContextResult result; std::vector all_items; - auto hosts = dependency_graph.find_host_sources(path_id); + auto hosts = workspace.dep_graph.find_host_sources(path_id); for(auto host_id: hosts) { - auto host_path = path_pool.resolve(host_id); - auto host_cdb = cdb.lookup(host_path, {.suppress_logging = true}); + auto host_path = workspace.path_pool.resolve(host_id); + auto host_cdb = workspace.cdb.lookup(host_path, {.suppress_logging = true}); if(host_cdb.empty()) continue; auto host_uri_opt = lsp::URI::from_file_path(std::string(host_path)); @@ -652,7 +804,7 @@ void MasterServer::register_handlers() { } if(hosts.empty()) { - auto entries = cdb.lookup(path, {.suppress_logging = true}); + auto entries = workspace.cdb.lookup(path, {.suppress_logging = true}); for(std::size_t i = 0; i < entries.size(); ++i) { auto& entry = entries[i]; std::string desc; @@ -694,12 +846,12 @@ void MasterServer::register_handlers() { "clice/currentContext", [this](RequestContext& ctx, const ext::CurrentContextParams& params) -> RawResult { auto path = Compiler::uri_to_path(params.uri); - auto path_id = path_pool.intern(path); + auto path_id = workspace.path_pool.intern(path); ext::CurrentContextResult result; - auto active_ctx = compiler.get_active_context(path_id); - if(active_ctx) { - auto ctx_path = path_pool.resolve(*active_ctx); + auto sit = sessions.find(path_id); + if(sit != sessions.end() && sit->second.active_context) { + auto ctx_path = workspace.path_pool.resolve(*sit->second.active_context); auto ctx_uri_opt = lsp::URI::from_file_path(std::string(ctx_path)); if(ctx_uri_opt) { ext::ContextItem item; @@ -716,19 +868,30 @@ void MasterServer::register_handlers() { "clice/switchContext", [this](RequestContext& ctx, const ext::SwitchContextParams& params) -> RawResult { auto path = Compiler::uri_to_path(params.uri); - auto path_id = path_pool.intern(path); + auto path_id = workspace.path_pool.intern(path); auto context_path = Compiler::uri_to_path(params.context_uri); - auto context_path_id = path_pool.intern(context_path); + auto context_path_id = workspace.path_pool.intern(context_path); ext::SwitchContextResult result; - auto context_cdb = cdb.lookup(context_path, {.suppress_logging = true}); + auto context_cdb = workspace.cdb.lookup(context_path, {.suppress_logging = true}); if(context_cdb.empty()) { result.success = false; co_return to_raw(result); } - compiler.switch_context(path_id, context_path_id); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) { + result.success = false; + co_return to_raw(result); + } + + sit->second.active_context = context_path_id; + sit->second.header_context.reset(); + sit->second.pch_ref.reset(); + sit->second.ast_deps.reset(); + sit->second.ast_dirty = true; + result.success = true; co_return to_raw(result); }); diff --git a/src/server/master_server.h b/src/server/master_server.h index 99a2a717b..0ee4d2f09 100644 --- a/src/server/master_server.h +++ b/src/server/master_server.h @@ -9,11 +9,12 @@ #include "eventide/ipc/peer.h" #include "eventide/serde/serde/raw_value.h" #include "server/compiler.h" -#include "server/config.h" #include "server/indexer.h" +#include "server/session.h" #include "server/worker_pool.h" -#include "support/path_pool.h" -#include "syntax/dependency_graph.h" +#include "server/workspace.h" + +#include "llvm/ADT/DenseMap.h" namespace clice { @@ -29,9 +30,10 @@ enum class ServerLifecycle : std::uint8_t { /// Top-level LSP server. /// -/// Registers LSP handlers and delegates all compilation, document management, -/// and index queries to Compiler and Indexer respectively. MasterServer -/// itself only owns workspace initialization and background indexing scheduling. +/// Owns all persistent state and registers LSP handlers. Document lifecycle +/// (open/change/close/save) is handled directly here on the sessions map. +/// Compilation and feature-request forwarding are delegated to Compiler; +/// index queries are delegated to Indexer. class MasterServer { public: MasterServer(et::event_loop& loop, et::ipc::JsonPeer& peer, std::string self_path); @@ -42,21 +44,28 @@ class MasterServer { private: et::event_loop& loop; et::ipc::JsonPeer& peer; + + /// Persistent project-wide state (config, CDB, path pool, dependency + /// graphs, compilation caches, symbol index). + Workspace workspace; + + /// Per-file editing sessions, keyed by server-level path_id. + llvm::DenseMap sessions; + + /// Worker process pool for offloading compilation and queries. WorkerPool pool; - PathPool path_pool; + + /// Index query layer (reads from workspace and sessions). Indexer indexer; + /// Compilation lifecycle manager (reads/writes workspace and sessions). + Compiler compiler; + ServerLifecycle lifecycle = ServerLifecycle::Uninitialized; std::string self_path; std::string workspace_root; - CliceConfig config; std::string session_log_dir; - CompilationDatabase cdb; - DependencyGraph dependency_graph; - - Compiler compiler; - /// Background indexing state. std::vector index_queue; std::size_t index_queue_pos = 0; diff --git a/src/server/session.h b/src/server/session.h new file mode 100644 index 000000000..9a4fc229c --- /dev/null +++ b/src/server/session.h @@ -0,0 +1,95 @@ +#pragma once + +#include +#include +#include +#include + +#include "eventide/async/async.h" +#include "server/workspace.h" + +#include "llvm/ADT/SmallVector.h" + +namespace clice { + +namespace et = eventide; + +/// An editing session for a single file opened in the editor. +/// +/// Created on didOpen, destroyed on didClose. All fields are local to this +/// file's translation unit and NEVER leak to Workspace or other Sessions. +/// +/// The only path from Session to Workspace is didSave, which triggers +/// Workspace to rescan the disk file — Session itself does not write to +/// Workspace. +/// +/// Sessions may READ from Workspace (e.g. to obtain PCH/PCM paths, module +/// mappings, include graph) but all compilation results stay here. +struct Session { + // ----- Identity ----- + + /// Path ID of this file in PathPool. Set on creation, never changes. + std::uint32_t path_id = 0; + + // ----- Document state (from LSP) ----- + + /// LSP document version, incremented by the client on each edit. + int version = 0; + + /// Current buffer content (may differ from disk until saved). + std::string text; + + /// Monotonic generation counter, incremented on every didChange. + /// Used to detect stale compilation results (ABA prevention). + std::uint64_t generation = 0; + + /// Whether the AST needs to be rebuilt before serving queries. + bool ast_dirty = true; + + /// Non-null while a compilation is in flight for this file. + /// Other queries wait on the event; the compilation task itself + /// runs independently and cannot be cancelled by LSP $/cancelRequest. + struct PendingCompile { + et::event done; + bool succeeded = false; + }; + + std::shared_ptr compiling; + + // ----- Compilation artifacts (per-file, references Workspace caches) ----- + + /// Reference to the PCH entry in Workspace.pch_cache, if any. + /// The PCH itself is owned by Workspace (shared, content-addressed); + /// Session only stores enough to locate and validate it. + struct PCHRef { + std::uint32_t path_id = 0; ///< Key into Workspace.pch_cache. + std::uint64_t hash = 0; ///< Preamble hash at build time. + std::uint32_t bound = 0; ///< Preamble byte boundary. + }; + + std::optional pch_ref; + + /// Dependency snapshot from the last successful AST compilation. + /// Used for two-layer staleness detection (mtime + content hash). + std::optional ast_deps; + + // ----- Header context (only meaningful for header files) ----- + + /// Compilation context for header files that lack their own CDB entry. + /// Stores the host source file and synthesized preamble for this header. + std::optional header_context; + + /// User-selected compilation context override (via clice/switchContext). + /// When set, overrides automatic header context resolution. + std::optional active_context; + + // ----- Index (per-file, not merged into Workspace) ----- + + /// Symbol index built from the latest compilation of this file's buffer. + /// Used for queries (hover, goto, references) on this file. + /// NOT merged into Workspace.project_index — that only gets disk-derived + /// data from background indexing. + std::optional file_index; +}; + +} // namespace clice diff --git a/src/server/workspace.cpp b/src/server/workspace.cpp new file mode 100644 index 000000000..1982b9a99 --- /dev/null +++ b/src/server/workspace.cpp @@ -0,0 +1,91 @@ +#include "server/workspace.h" + +#include + +#include "eventide/ipc/lsp/position.h" +#include "eventide/ipc/lsp/protocol.h" + +namespace clice { + +namespace lsp = eventide::ipc::lsp; + +/// Find the tightest (innermost) occurrence containing `offset` via binary search. +const static index::Occurrence* lookup_occurrence(const std::vector& occs, + std::uint32_t offset) { + auto it = std::ranges::lower_bound(occs, offset, {}, [](const index::Occurrence& o) { + return o.range.end; + }); + const index::Occurrence* best = nullptr; + while(it != occs.end() && it->range.contains(offset)) { + if(!best || (it->range.end - it->range.begin) < (best->range.end - best->range.begin)) { + best = &*it; + } + ++it; + } + return best; +} + +// -- OpenFileIndex ------------------------------------------------------------ + +std::optional> + OpenFileIndex::find_occurrence(std::uint32_t offset) const { + if(!mapper) + return std::nullopt; + auto* occ = lookup_occurrence(file_index.occurrences, offset); + if(!occ) + return std::nullopt; + auto start = mapper->to_position(occ->range.begin); + auto end = mapper->to_position(occ->range.end); + if(!start || !end) + return std::nullopt; + return std::pair{ + occ->target, + protocol::Range{*start, *end} + }; +} + +// -- MergedIndexShard --------------------------------------------------------- + +std::optional> + MergedIndexShard::find_occurrence(std::uint32_t offset) const { + auto* m = mapper(); + if(!m) + return std::nullopt; + std::optional> result; + index.lookup(offset, [&](const index::Occurrence& o) { + auto start = m->to_position(o.range.begin); + auto end = m->to_position(o.range.end); + if(start && end) { + result = { + o.target, + protocol::Range{*start, *end} + }; + } + return false; + }); + return result; +} + +// -- Workspace ---------------------------------------------------------------- + +llvm::SmallVector Workspace::on_file_saved(std::uint32_t path_id) { + llvm::SmallVector dirtied; + if(compile_graph) { + auto result = compile_graph->update(path_id); + for(auto id: result) { + dirtied.push_back(id); + pcm_paths.erase(id); + pcm_cache.erase(id); + } + } + return dirtied; +} + +void Workspace::on_file_closed(std::uint32_t path_id) { + if(compile_graph && compile_graph->has_unit(path_id)) { + compile_graph->update(path_id); + } + pch_cache.erase(path_id); +} + +} // namespace clice diff --git a/src/server/workspace.h b/src/server/workspace.h new file mode 100644 index 000000000..7ea130787 --- /dev/null +++ b/src/server/workspace.h @@ -0,0 +1,245 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "command/command.h" +#include "eventide/ipc/lsp/position.h" +#include "eventide/ipc/lsp/protocol.h" +#include "index/merged_index.h" +#include "index/project_index.h" +#include "semantic/relation_kind.h" +#include "server/compile_graph.h" +#include "server/config.h" +#include "support/path_pool.h" +#include "syntax/dependency_graph.h" + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" + +namespace clice { + +namespace et = eventide; +namespace protocol = et::ipc::protocol; +namespace lsp = et::ipc::lsp; + +// --------------------------------------------------------------------------- +// Shared types used by both Workspace and Session +// --------------------------------------------------------------------------- + +/// Two-layer staleness snapshot for compilation artifacts (PCH, AST, etc.). +/// +/// Layer 1 (fast): compare each file's current mtime against build_at. +/// If all mtimes <= build_at, the artifact is fresh (zero I/O beyond stat). +/// +/// Layer 2 (precise): for files whose mtime changed, re-hash their content +/// and compare against the stored hash. If the hash matches, the file was +/// "touched" but not actually modified — skip the rebuild. +struct DepsSnapshot { + llvm::SmallVector path_ids; + llvm::SmallVector hashes; + std::int64_t build_at = 0; +}; + +/// Context for compiling a header file that lacks its own CDB entry. +struct HeaderFileContext { + std::uint32_t host_path_id; ///< Source file acting as host. + std::string preamble_path; ///< Path to generated preamble file on disk. + std::uint64_t preamble_hash; ///< Hash of preamble content for staleness. +}; + +// --------------------------------------------------------------------------- +// Index wrapper types (used by Workspace and Session) +// --------------------------------------------------------------------------- + +/// In-memory index for an open file. Kept separate from MergedIndex because +/// open files change frequently, are based on unsaved buffer content, and only +/// need to track the main file (headers are covered by PCH/PCM indexing). +struct OpenFileIndex { + index::FileIndex file_index; + index::SymbolTable symbols; + std::string content; ///< Buffer text at index time (for position mapping). + + /// Cached PositionMapper built from `content`. Avoids re-scanning line + /// offsets on every query. Initialized by Indexer::set_open_file(). + std::optional mapper; + + /// Find the tightest occurrence containing `offset`. + /// Returns (symbol_hash, LSP range) with positions already converted. + std::optional> + find_occurrence(std::uint32_t offset) const; + + /// Iterate relations matching `kind`, calling back with pre-converted ranges. + /// Callback: (const index::Relation&, protocol::Range) -> bool (true = continue). + template + void find_relations(index::SymbolHash hash, RelationKind kind, Fn&& fn) const { + if(!mapper) + return; + auto it = file_index.relations.find(hash); + if(it == file_index.relations.end()) + return; + for(auto& r: it->second) { + if(r.kind & kind) { + auto start = mapper->to_position(r.range.begin); + auto end = mapper->to_position(r.range.end); + if(start && end) { + if(!fn(r, protocol::Range{*start, *end})) + return; + } + } + } + } +}; + +/// Wraps index::MergedIndex with a lazily-cached PositionMapper. +struct MergedIndexShard { + index::MergedIndex index; + mutable std::optional cached_mapper; + + /// Get or lazily build a PositionMapper from the index's stored content. + const lsp::PositionMapper* mapper() const { + if(!cached_mapper) { + auto c = index.content(); + if(!c.empty()) { + cached_mapper.emplace(c, lsp::PositionEncoding::UTF16); + } + } + return cached_mapper ? &*cached_mapper : nullptr; + } + + /// Invalidate the cached mapper (call after merge changes content). + void invalidate_mapper() { + cached_mapper.reset(); + } + + /// Find occurrence at byte offset. + /// Returns (symbol_hash, LSP range) with positions already converted. + std::optional> + find_occurrence(std::uint32_t offset) const; + + /// Iterate relations matching `kind`, calling back with pre-converted ranges. + /// Callback: (const index::Relation&, protocol::Range) -> bool (true = continue). + template + void find_relations(index::SymbolHash hash, RelationKind kind, Fn&& fn) const { + auto* m = mapper(); + if(!m) + return; + index.lookup(hash, kind, [&](const index::Relation& r) { + auto start = m->to_position(r.range.begin); + auto end = m->to_position(r.range.end); + if(start && end) { + return fn(r, protocol::Range{*start, *end}); + } + return true; + }); + } +}; + +// --------------------------------------------------------------------------- +// PCH / PCM caches +// --------------------------------------------------------------------------- + +/// Cached PCH state. Content-addressed by preamble hash — shared across all +/// files (open or on-disk) that have the same preamble content. +struct PCHState { + std::string path; + std::uint32_t bound = 0; + std::uint64_t hash = 0; + DepsSnapshot deps; + std::shared_ptr building; +}; + +/// Cached PCM state for a single C++20 module. Shared across all files that +/// import the same module. +struct PCMState { + std::string path; + DepsSnapshot deps; +}; + +// --------------------------------------------------------------------------- +// Workspace +// --------------------------------------------------------------------------- + +/// All persistent, project-wide state derived from files on disk. +/// +/// Workspace is the single source of truth for: +/// - dependency relationships (include graph, module DAG) +/// - compilation artifacts shared across files (PCH/PCM caches) +/// - symbol index (ProjectIndex + per-file MergedIndex shards) +/// - compilation database and configuration +/// +/// Workspace is NEVER modified by unsaved buffer content. The only mutation +/// paths are: +/// - Initialization (load_workspace at startup) +/// - didSave (on_file_saved: rescan disk, cascade invalidation) +/// - Background index (merge TUIndex results from stateless workers) +/// +/// Sessions (open files) may READ from Workspace but must not WRITE to it +/// except through the didSave path. +struct Workspace { + // ----- Configuration ----- + + CliceConfig config; + CompilationDatabase cdb; + + // ----- Path management ----- + + PathPool path_pool; + + // ----- Dependency graphs ----- + + /// Include relationships between files on disk (#include edges). + /// Built once at startup from CDB scan; updated incrementally on didSave. + DependencyGraph dep_graph; + + /// C++20 module compilation ordering DAG. + /// Lazily resolves module dependencies; updated on didSave via cascade. + std::unique_ptr compile_graph; + + /// Reverse mapping: file path_id → module name (e.g. "std", "foo.bar"). + /// Built from dep_graph at startup; updated on didSave when module + /// declarations change. + llvm::DenseMap path_to_module; + + // ----- Compilation caches (shared, content-addressed) ----- + + /// PCH cache, keyed by file path_id. Content-addressed by preamble hash, + /// so different files with identical preambles share the same PCH. + /// Both open files and background indexing may reference entries here. + llvm::DenseMap pch_cache; + + /// PCM cache, keyed by module source path_id. + llvm::DenseMap pcm_cache; + + /// PCM output paths, keyed by module source path_id. + /// Maps to the .pcm file on disk used as -fmodule-file argument. + llvm::DenseMap pcm_paths; + + // ----- Symbol index (disk-derived) ----- + + /// Global symbol table across all indexed translation units. + index::ProjectIndex project_index; + + /// Per-file index shards from background indexing, keyed by project-level + /// path_id. Contains symbol occurrences, relations, and stored content + /// for position mapping. + llvm::DenseMap merged_indices; + + // ----- Lifecycle methods ----- + + /// Called when a file is saved to disk. Cascades invalidation through + /// compile_graph and clears affected PCM caches. + /// Returns path_ids of all files dirtied by the cascade. + llvm::SmallVector on_file_saved(std::uint32_t path_id); + + /// Called when a file is closed. Notifies compile_graph if this file + /// is a module unit so dependents can be re-evaluated on next compile. + void on_file_closed(std::uint32_t path_id); +}; + +} // namespace clice From 6006270b25739a8eab4cd7d8033d6215e5125229 Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 10:58:18 +0800 Subject: [PATCH 02/18] refactor(server): move cache/module/utility methods to Workspace Move methods from Compiler to their natural owners: - Workspace: load_cache, save_cache, cleanup_cache, build_module_map, fill_pcm_deps, cancel_all - Free functions: uri_to_path, hash_file, capture_deps_snapshot, deps_changed - File-scope static: detect_completion_context Compiler now only contains genuine compilation logic (ensure_compiled, ensure_pch, ensure_deps, forward_query/build, header context resolution, include/import completion). Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/compiler.cpp | 372 +++++------------------------------ src/server/compiler.h | 23 +-- src/server/master_server.cpp | 70 +++---- src/server/workspace.cpp | 284 ++++++++++++++++++++++++++ src/server/workspace.h | 29 +++ 5 files changed, 397 insertions(+), 381 deletions(-) diff --git a/src/server/compiler.cpp b/src/server/compiler.cpp index ef0cf21b1..8c183cda7 100644 --- a/src/server/compiler.cpp +++ b/src/server/compiler.cpp @@ -27,301 +27,62 @@ namespace clice { -/// Hash a file's content using xxh3_64bits. Returns 0 on read failure. -static std::uint64_t hash_file(llvm::StringRef path) { - auto buf = llvm::MemoryBuffer::getFile(path); - if(!buf) - return 0; - return llvm::xxh3_64bits((*buf)->getBuffer()); -} - -/// Capture a two-layer staleness snapshot after a successful compilation. -/// Interns dependency paths into the PathPool and hashes each file's content. -static DepsSnapshot capture_deps_snapshot(PathPool& pool, llvm::ArrayRef deps) { - DepsSnapshot snap; - // Capture timestamp BEFORE hashing to avoid TOCTOU: if a file is modified - // during hashing, its mtime will be > build_at, triggering Layer 2 re-hash. - snap.build_at = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); - snap.path_ids.reserve(deps.size()); - snap.hashes.reserve(deps.size()); - for(const auto& file: deps) { - snap.path_ids.push_back(pool.intern(file)); - snap.hashes.push_back(hash_file(file)); - } - return snap; -} - -/// Two-layer staleness check. -/// -/// Layer 1 (fast): stat each dep file, compare mtime against build_at. -/// If all mtimes <= build_at → nothing changed, return false immediately. -/// -/// Layer 2 (precise): for files with mtime > build_at, re-hash their content. -/// If the hash matches the stored hash → file was touched but not modified. -/// If any hash differs → truly changed, return true. -static bool deps_changed(const PathPool& pool, const DepsSnapshot& snap) { - for(std::size_t i = 0; i < snap.path_ids.size(); ++i) { - auto path = pool.resolve(snap.path_ids[i]); - llvm::sys::fs::file_status status; - if(auto ec = llvm::sys::fs::status(path, status)) { - // File disappeared — definitely changed. - if(snap.hashes[i] != 0) - return true; - continue; - } - - // Layer 1: mtime check (cheap, stat only). - auto current_mtime = llvm::sys::toTimeT(status.getLastModificationTime()); - if(current_mtime <= snap.build_at) - continue; - - // Layer 2: mtime is newer — re-hash content to confirm actual change. - auto current_hash = hash_file(path); - if(current_hash != snap.hashes[i]) - return true; - } - return false; -} - -/// Serializable cache structures for cache.json persistence. -/// Paths are stored in a shared table and referenced by index to avoid -/// redundant storage (a single file can depend on thousands of headers, -/// many of which are shared across entries). -namespace { - -struct CacheDepEntry { - std::uint32_t path; // index into CacheData::paths - std::uint64_t hash; -}; - -struct CachePCHEntry { - std::string filename; - std::uint32_t source_file; // index into CacheData::paths - std::uint64_t hash; - std::uint32_t bound; - std::int64_t build_at; - std::vector deps; -}; - -struct CachePCMEntry { - std::string filename; - std::uint32_t source_file; - std::string module_name; - std::int64_t build_at; - std::vector deps; -}; - -struct CacheData { - std::vector paths; - std::vector pch; - std::vector pcm; -}; - -} // namespace - namespace lsp = eventide::ipc::lsp; using serde_raw = et::serde::RawValue; -Compiler::Compiler(et::event_loop& loop, - et::ipc::JsonPeer& peer, - Workspace& workspace, - WorkerPool& pool, - llvm::DenseMap& sessions) : - loop(loop), peer(peer), workspace(workspace), pool(pool), sessions(sessions) {} - -Compiler::~Compiler() { - cancel_all(); -} - -void Compiler::fill_pcm_deps(std::unordered_map& pcms, - std::uint32_t exclude_path_id) const { - for(auto& [pid, pcm_path]: workspace.pcm_paths) { - if(pid == exclude_path_id) - continue; - auto mod_it = workspace.path_to_module.find(pid); - if(mod_it != workspace.path_to_module.end()) { - pcms[mod_it->second] = pcm_path; - } - } -} - -void Compiler::cancel_all() { - if(workspace.compile_graph) { - workspace.compile_graph->cancel_all(); - } -} - -void Compiler::load_cache() { - if(workspace.config.cache_dir.empty()) - return; - - auto cache_path = path::join(workspace.config.cache_dir, "cache", "cache.json"); - auto content = fs::read(cache_path); - if(!content) { - LOG_DEBUG("No cache.json found at {}", cache_path); - return; - } +/// Detect whether the cursor is inside a preamble directive (include/import). +static PreambleCompletionContext detect_completion_context(const std::string& text, + uint32_t offset) { + auto line_start = text.rfind('\n', offset > 0 ? offset - 1 : 0); + line_start = (line_start == std::string::npos) ? 0 : line_start + 1; - CacheData data; - auto status = et::serde::json::from_json(*content, data); - if(!status) { - LOG_WARN("Failed to parse cache.json"); - return; - } + auto line_end = text.find('\n', offset); + if(line_end == std::string::npos) + line_end = text.size(); - auto resolve = [&](std::uint32_t idx) -> llvm::StringRef { - return idx < data.paths.size() ? llvm::StringRef(data.paths[idx]) : ""; - }; + auto line = llvm::StringRef(text).slice(line_start, offset); + auto trimmed = line.ltrim(); - auto load_deps = [&](std::int64_t build_at, const auto& dep_entries) -> DepsSnapshot { - DepsSnapshot deps; - deps.build_at = build_at; - for(auto& dep: dep_entries) { - auto dep_path = resolve(dep.path); - if(dep_path.empty()) - continue; - deps.path_ids.push_back(workspace.path_pool.intern(dep_path)); - deps.hashes.push_back(dep.hash); + if(trimmed.starts_with("#")) { + auto directive = trimmed.drop_front(1).ltrim(); + if(directive.consume_front("include")) { + directive = directive.ltrim(); + if(directive.consume_front("\"")) { + return {CompletionContext::IncludeQuoted, directive.str()}; + } + if(directive.consume_front("<")) { + return {CompletionContext::IncludeAngled, directive.str()}; + } } - return deps; - }; - - for(auto& entry: data.pch) { - auto pch_path = path::join(workspace.config.cache_dir, "cache", "pch", entry.filename); - auto source = resolve(entry.source_file); - if(!llvm::sys::fs::exists(pch_path) || source.empty()) - continue; - - auto path_id = workspace.path_pool.intern(source); - auto& st = workspace.pch_cache[path_id]; - st.path = pch_path; - st.hash = entry.hash; - st.bound = entry.bound; - st.deps = load_deps(entry.build_at, entry.deps); - - LOG_DEBUG("Loaded cached PCH: {} -> {}", source, pch_path); - } - - for(auto& entry: data.pcm) { - auto pcm_path = path::join(workspace.config.cache_dir, "cache", "pcm", entry.filename); - auto source = resolve(entry.source_file); - if(!llvm::sys::fs::exists(pcm_path) || source.empty()) - continue; - - auto path_id = workspace.path_pool.intern(source); - workspace.pcm_cache[path_id] = {pcm_path, load_deps(entry.build_at, entry.deps)}; - workspace.pcm_paths[path_id] = pcm_path; - - LOG_DEBUG("Loaded cached PCM: {} (module {}) -> {}", source, entry.module_name, pcm_path); + return {}; } - LOG_INFO("Loaded cache.json: {} PCH entries, {} PCM entries", - workspace.pch_cache.size(), - workspace.pcm_cache.size()); -} - -void Compiler::save_cache() { - if(workspace.config.cache_dir.empty()) - return; - - CacheData data; - std::unordered_map index_map; - - auto intern = [&](std::uint32_t runtime_path_id) -> std::uint32_t { - auto path = std::string(workspace.path_pool.resolve(runtime_path_id)); - auto [it, inserted] = - index_map.try_emplace(path, static_cast(data.paths.size())); - if(inserted) { - data.paths.push_back(path); - } - return it->second; - }; - - for(auto& [path_id, st]: workspace.pch_cache) { - if(st.path.empty()) - continue; - - CachePCHEntry entry; - entry.filename = std::string(path::filename(st.path)); - entry.source_file = intern(path_id); - entry.hash = st.hash; - entry.bound = st.bound; - entry.build_at = st.deps.build_at; - for(std::size_t i = 0; i < st.deps.path_ids.size(); ++i) { - entry.deps.push_back({intern(st.deps.path_ids[i]), st.deps.hashes[i]}); - } - data.pch.push_back(std::move(entry)); + auto import_check = trimmed; + if(import_check.consume_front("export") && !import_check.empty() && + !std::isalnum(import_check[0])) { + import_check = import_check.ltrim(); } - - for(auto& [path_id, st]: workspace.pcm_cache) { - if(st.path.empty()) - continue; - - CachePCMEntry entry; - entry.filename = std::string(path::filename(st.path)); - entry.source_file = intern(path_id); - auto mod_it = workspace.path_to_module.find(path_id); - entry.module_name = mod_it != workspace.path_to_module.end() ? mod_it->second : ""; - entry.build_at = st.deps.build_at; - for(std::size_t i = 0; i < st.deps.path_ids.size(); ++i) { - entry.deps.push_back({intern(st.deps.path_ids[i]), st.deps.hashes[i]}); + if(import_check.consume_front("import") && + (import_check.empty() || !std::isalnum(import_check[0]))) { + import_check = import_check.ltrim(); + auto rest_of_line = llvm::StringRef(text).slice(line_start, line_end); + if(!rest_of_line.contains(';')) { + return {CompletionContext::Import, import_check.str()}; } - data.pcm.push_back(std::move(entry)); } - auto json_str = et::serde::json::to_json(data); - if(!json_str) { - LOG_WARN("Failed to serialize cache.json"); - return; - } - - auto cache_path = path::join(workspace.config.cache_dir, "cache", "cache.json"); - auto tmp_path = cache_path + ".tmp"; - auto write_result = fs::write(tmp_path, *json_str); - if(!write_result) { - LOG_WARN("Failed to write cache.json.tmp: {}", write_result.error().message()); - return; - } - auto rename_result = fs::rename(tmp_path, cache_path); - if(!rename_result) { - LOG_WARN("Failed to rename cache.json.tmp to cache.json: {}", - rename_result.error().message()); - } + return {}; } -void Compiler::cleanup_cache(int max_age_days) { - if(workspace.config.cache_dir.empty()) - return; - - auto now = std::chrono::system_clock::now(); - auto max_age = std::chrono::hours(max_age_days * 24); - - for(auto* subdir: {"cache/pch", "cache/pcm"}) { - auto dir = path::join(workspace.config.cache_dir, subdir); - std::error_code ec; - for(auto it = llvm::sys::fs::directory_iterator(dir, ec); - !ec && it != llvm::sys::fs::directory_iterator(); - it.increment(ec)) { - llvm::sys::fs::file_status status; - if(auto stat_ec = llvm::sys::fs::status(it->path(), status)) - continue; - - auto mtime = status.getLastModificationTime(); - auto age = now - mtime; - if(age > max_age) { - llvm::sys::fs::remove(it->path()); - LOG_DEBUG("Cleaned up stale cache file: {}", it->path()); - } - } - } -} +Compiler::Compiler(et::event_loop& loop, + et::ipc::JsonPeer& peer, + Workspace& workspace, + WorkerPool& pool, + llvm::DenseMap& sessions) : + loop(loop), peer(peer), workspace(workspace), pool(pool), sessions(sessions) {} -void Compiler::build_module_map() { - for(auto& [module_name, path_ids]: workspace.dep_graph.modules()) { - for(auto path_id: path_ids) { - workspace.path_to_module[path_id] = module_name.str(); - } - } +Compiler::~Compiler() { + workspace.cancel_all(); } void Compiler::init_compile_graph() { @@ -398,7 +159,7 @@ void Compiler::init_compile_graph() { bp.output_path = pcm_path; // Clang needs ALL transitive PCM deps, not just direct imports. - fill_pcm_deps(bp.pcms); + workspace.fill_pcm_deps(bp.pcms); auto result = co_await pool.send_stateless(bp); if(!result.has_value() || !result.value().success) { @@ -415,7 +176,7 @@ void Compiler::init_compile_graph() { LOG_INFO("Built PCM for module {}: {}", mod_it->second, result.value().output_path); // Persist cache metadata after successful build. - save_cache(); + workspace.save_cache(); // Signal that new index data is available for background merge. if(on_indexing_needed) @@ -682,7 +443,7 @@ std::optional Compiler::resolve_header_context(std::uint32_t return HeaderFileContext{host_path_id, preamble_path, preamble_hash}; } -std::string Compiler::uri_to_path(const std::string& uri) { +std::string uri_to_path(const std::string& uri) { auto parsed = lsp::URI::parse(uri); if(parsed.has_value()) { auto path = parsed->file_path(); @@ -805,7 +566,7 @@ et::task Compiler::ensure_pch(Session& session, LOG_INFO("PCH built for {}: {}", path, result.value().output_path); // Persist cache metadata after successful build. - save_cache(); + workspace.save_cache(); completion->set(); co_return true; @@ -863,7 +624,7 @@ et::task Compiler::ensure_deps(Session& session, // Fill all available PCM paths, excluding the file's own PCM // to avoid "multiple module declarations". - fill_pcm_deps(pcms, path_id); + workspace.fill_pcm_deps(pcms, path_id); co_return true; } @@ -1167,49 +928,6 @@ Compiler::RawResult Compiler::handle_completion(const protocol::Position& positi co_return co_await forward_build(worker::BuildKind::Completion, position, session); } -PreambleCompletionContext Compiler::detect_completion_context(const std::string& text, - uint32_t offset) { - auto line_start = text.rfind('\n', offset > 0 ? offset - 1 : 0); - line_start = (line_start == std::string::npos) ? 0 : line_start + 1; - - auto line_end = text.find('\n', offset); - if(line_end == std::string::npos) - line_end = text.size(); - - auto line = llvm::StringRef(text).slice(line_start, offset); - auto trimmed = line.ltrim(); - - if(trimmed.starts_with("#")) { - auto directive = trimmed.drop_front(1).ltrim(); - if(directive.consume_front("include")) { - directive = directive.ltrim(); - if(directive.consume_front("\"")) { - return {CompletionContext::IncludeQuoted, directive.str()}; - } - if(directive.consume_front("<")) { - return {CompletionContext::IncludeAngled, directive.str()}; - } - } - return {}; - } - - auto import_check = trimmed; - if(import_check.consume_front("export") && !import_check.empty() && - !std::isalnum(import_check[0])) { - import_check = import_check.ltrim(); - } - if(import_check.consume_front("import") && - (import_check.empty() || !std::isalnum(import_check[0]))) { - import_check = import_check.ltrim(); - auto rest_of_line = llvm::StringRef(text).slice(line_start, line_end); - if(!rest_of_line.contains(';')) { - return {CompletionContext::Import, import_check.str()}; - } - } - - return {}; -} - et::serde::RawValue Compiler::complete_include(const PreambleCompletionContext& ctx, llvm::StringRef path) { std::string directory; diff --git a/src/server/compiler.h b/src/server/compiler.h index afada9815..0a10afa6d 100644 --- a/src/server/compiler.h +++ b/src/server/compiler.h @@ -33,6 +33,9 @@ struct PreambleCompletionContext { std::string prefix; }; +/// Convert a file:// URI to a local file path. +std::string uri_to_path(const std::string& uri); + /// Stateless compilation service. /// /// Compiler does not own any persistent state. It holds references to @@ -45,8 +48,7 @@ struct PreambleCompletionContext { /// - Compilation (ensure_compiled, ensure_pch, ensure_deps) /// - Feature request forwarding to workers /// - Compile argument resolution -/// - Cache persistence -/// - Module/compile graph initialization +/// - Compile graph initialization class Compiler { public: Compiler(et::event_loop& loop, @@ -56,18 +58,8 @@ class Compiler { llvm::DenseMap& sessions); ~Compiler(); - /// Convert a file:// URI to a local file path. - static std::string uri_to_path(const std::string& uri); - - // ----- Cache persistence (operates on Workspace) ----- - - void load_cache(); - void save_cache(); - void cleanup_cache(int max_age_days = 7); - // ----- Module / compile graph (operates on Workspace) ----- - void build_module_map(); void init_compile_graph(); // ----- Compile argument resolution ----- @@ -79,10 +71,6 @@ class Compiler { std::vector& arguments, Session* session = nullptr); - /// Fill PCM paths for all built modules (for background indexing). - void fill_pcm_deps(std::unordered_map& pcms, - std::uint32_t exclude_path_id = UINT32_MAX) const; - // ----- Pull-based compilation ----- /// Compile an open file's AST if dirty. On success, updates session's @@ -104,8 +92,6 @@ class Compiler { RawResult handle_completion(const protocol::Position& position, Session& session); - void cancel_all(); - void clear_diagnostics(const std::string& uri); /// Callback invoked when indexing should be scheduled. @@ -136,7 +122,6 @@ class Compiler { std::vector& arguments, Session* session); - PreambleCompletionContext detect_completion_context(const std::string& text, uint32_t offset); et::serde::RawValue complete_include(const PreambleCompletionContext& ctx, llvm::StringRef path); et::serde::RawValue complete_import(const PreambleCompletionContext& ctx); diff --git a/src/server/master_server.cpp b/src/server/master_server.cpp index db5938d5f..7411dccd7 100644 --- a/src/server/master_server.cpp +++ b/src/server/master_server.cpp @@ -67,8 +67,8 @@ et::task<> MasterServer::load_workspace() { // Clean up stale files first, then load — load_cache() only restores // entries still listed in cache.json, so cleanup won't delete live files. - compiler.cleanup_cache(); - compiler.load_cache(); + workspace.cleanup_cache(); + workspace.load_cache(); } std::string cdb_path; @@ -122,7 +122,7 @@ et::task<> MasterServer::load_workspace() { LOG_WARN("{} unresolved includes", unresolved); } - compiler.build_module_map(); + workspace.build_module_map(); indexer.load(workspace.config.index_dir); if(workspace.config.enable_indexing) { @@ -186,7 +186,7 @@ et::task<> MasterServer::run_background_indexing() { if(!compiler.fill_compile_args(file_path, params.directory, params.arguments, nullptr)) continue; - compiler.fill_pcm_deps(params.pcms); + workspace.fill_pcm_deps(params.pcms); LOG_INFO("Background indexing: {}", file_path); @@ -222,7 +222,7 @@ void MasterServer::register_handlers() { auto& init = params.lsp__initialize_params; if(init.root_uri.has_value()) { - workspace_root = Compiler::uri_to_path(*init.root_uri); + workspace_root = uri_to_path(*init.root_uri); } lifecycle = ServerLifecycle::Initialized; @@ -354,7 +354,7 @@ void MasterServer::register_handlers() { LOG_INFO("Exit notification received"); indexer.save(workspace.config.index_dir); - compiler.save_cache(); + workspace.save_cache(); loop.schedule([this]() -> et::task<> { co_await pool.stop(); @@ -370,7 +370,7 @@ void MasterServer::register_handlers() { if(lifecycle != ServerLifecycle::Ready) return; - auto path = Compiler::uri_to_path(params.text_document.uri); + auto path = uri_to_path(params.text_document.uri); auto path_id = workspace.path_pool.intern(path); auto& session = sessions[path_id]; @@ -386,7 +386,7 @@ void MasterServer::register_handlers() { if(lifecycle != ServerLifecycle::Ready) return; - auto path = Compiler::uri_to_path(params.text_document.uri); + auto path = uri_to_path(params.text_document.uri); auto path_id = workspace.path_pool.intern(path); auto it = sessions.find(path_id); @@ -434,7 +434,7 @@ void MasterServer::register_handlers() { if(lifecycle != ServerLifecycle::Ready) return; - auto path = Compiler::uri_to_path(params.text_document.uri); + auto path = uri_to_path(params.text_document.uri); auto path_id = workspace.path_pool.intern(path); workspace.on_file_closed(path_id); @@ -457,7 +457,7 @@ void MasterServer::register_handlers() { if(lifecycle != ServerLifecycle::Ready) return; - auto path = Compiler::uri_to_path(params.text_document.uri); + auto path = uri_to_path(params.text_document.uri); auto path_id = workspace.path_pool.intern(path); auto dirtied = workspace.on_file_saved(path_id); @@ -487,7 +487,7 @@ void MasterServer::register_handlers() { /// --------------------------------------------------------------- peer.on_request([this](RequestContext& ctx, const protocol::HoverParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.text_document_position_params.text_document.uri); + auto path = uri_to_path(params.text_document_position_params.text_document.uri); auto path_id = workspace.path_pool.intern(path); auto sit = sessions.find(path_id); if(sit == sessions.end()) @@ -499,7 +499,7 @@ void MasterServer::register_handlers() { peer.on_request([this](RequestContext& ctx, const protocol::SemanticTokensParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.text_document.uri); + auto path = uri_to_path(params.text_document.uri); auto path_id = workspace.path_pool.intern(path); auto sit = sessions.find(path_id); if(sit == sessions.end()) @@ -509,7 +509,7 @@ void MasterServer::register_handlers() { peer.on_request( [this](RequestContext& ctx, const protocol::InlayHintParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.text_document.uri); + auto path = uri_to_path(params.text_document.uri); auto path_id = workspace.path_pool.intern(path); auto sit = sessions.find(path_id); if(sit == sessions.end()) @@ -519,7 +519,7 @@ void MasterServer::register_handlers() { peer.on_request( [this](RequestContext& ctx, const protocol::FoldingRangeParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.text_document.uri); + auto path = uri_to_path(params.text_document.uri); auto path_id = workspace.path_pool.intern(path); auto sit = sessions.find(path_id); if(sit == sessions.end()) @@ -529,7 +529,7 @@ void MasterServer::register_handlers() { peer.on_request([this](RequestContext& ctx, const protocol::DocumentSymbolParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.text_document.uri); + auto path = uri_to_path(params.text_document.uri); auto path_id = workspace.path_pool.intern(path); auto sit = sessions.find(path_id); if(sit == sessions.end()) @@ -539,7 +539,7 @@ void MasterServer::register_handlers() { peer.on_request( [this](RequestContext& ctx, const protocol::DocumentLinkParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.text_document.uri); + auto path = uri_to_path(params.text_document.uri); auto path_id = workspace.path_pool.intern(path); auto sit = sessions.find(path_id); if(sit == sessions.end()) @@ -549,7 +549,7 @@ void MasterServer::register_handlers() { peer.on_request( [this](RequestContext& ctx, const protocol::CodeActionParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.text_document.uri); + auto path = uri_to_path(params.text_document.uri); auto path_id = workspace.path_pool.intern(path); auto sit = sessions.find(path_id); if(sit == sessions.end()) @@ -564,7 +564,7 @@ void MasterServer::register_handlers() { std::uint32_t path_id; Session* session; }; - auto path = Compiler::uri_to_path(uri); + auto path = uri_to_path(uri); auto path_id = workspace.path_pool.intern(path); auto sit = sessions.find(path_id); Session* session = (sit != sessions.end()) ? &sit->second : nullptr; @@ -606,7 +606,7 @@ void MasterServer::register_handlers() { co_return to_raw(result); } - auto path = Compiler::uri_to_path(uri); + auto path = uri_to_path(uri); auto path_id = workspace.path_pool.intern(path); auto sit = sessions.find(path_id); if(sit == sessions.end()) @@ -656,7 +656,7 @@ void MasterServer::register_handlers() { peer.on_request([this](RequestContext& ctx, const protocol::CompletionParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.text_document_position_params.text_document.uri); + auto path = uri_to_path(params.text_document_position_params.text_document.uri); auto path_id = workspace.path_pool.intern(path); auto sit = sessions.find(path_id); if(sit == sessions.end()) @@ -665,17 +665,17 @@ void MasterServer::register_handlers() { sit->second); }); - peer.on_request([this](RequestContext& ctx, - const protocol::SignatureHelpParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.text_document_position_params.text_document.uri); - auto path_id = workspace.path_pool.intern(path); - auto sit = sessions.find(path_id); - if(sit == sessions.end()) - co_return serde_raw{"null"}; - co_return co_await compiler.forward_build(worker::BuildKind::SignatureHelp, - params.text_document_position_params.position, - sit->second); - }); + peer.on_request( + [this](RequestContext& ctx, const protocol::SignatureHelpParams& params) -> RawResult { + auto path = uri_to_path(params.text_document_position_params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + co_return co_await compiler.forward_build(worker::BuildKind::SignatureHelp, + params.text_document_position_params.position, + sit->second); + }); /// --------------------------------------------------------------- /// Hierarchy queries — index-based. @@ -779,7 +779,7 @@ void MasterServer::register_handlers() { peer.on_request( "clice/queryContext", [this](RequestContext& ctx, const ext::QueryContextParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.uri); + auto path = uri_to_path(params.uri); auto path_id = workspace.path_pool.intern(path); int offset_val = std::max(0, params.offset.value_or(0)); constexpr int page_size = 10; @@ -845,7 +845,7 @@ void MasterServer::register_handlers() { peer.on_request( "clice/currentContext", [this](RequestContext& ctx, const ext::CurrentContextParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.uri); + auto path = uri_to_path(params.uri); auto path_id = workspace.path_pool.intern(path); ext::CurrentContextResult result; @@ -867,9 +867,9 @@ void MasterServer::register_handlers() { peer.on_request( "clice/switchContext", [this](RequestContext& ctx, const ext::SwitchContextParams& params) -> RawResult { - auto path = Compiler::uri_to_path(params.uri); + auto path = uri_to_path(params.uri); auto path_id = workspace.path_pool.intern(path); - auto context_path = Compiler::uri_to_path(params.context_uri); + auto context_path = uri_to_path(params.context_uri); auto context_path_id = workspace.path_pool.intern(context_path); ext::SwitchContextResult result; diff --git a/src/server/workspace.cpp b/src/server/workspace.cpp index 1982b9a99..5268b1c85 100644 --- a/src/server/workspace.cpp +++ b/src/server/workspace.cpp @@ -1,9 +1,19 @@ #include "server/workspace.h" #include +#include #include "eventide/ipc/lsp/position.h" #include "eventide/ipc/lsp/protocol.h" +#include "eventide/serde/json/json.h" +#include "support/filesystem.h" +#include "support/logging.h" + +#include "llvm/Support/Chrono.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/xxhash.h" namespace clice { @@ -88,4 +98,278 @@ void Workspace::on_file_closed(std::uint32_t path_id) { pch_cache.erase(path_id); } +// -- Free helper functions (used by Workspace and Compiler) ------------------- + +std::uint64_t hash_file(llvm::StringRef path) { + auto buf = llvm::MemoryBuffer::getFile(path); + if(!buf) + return 0; + return llvm::xxh3_64bits((*buf)->getBuffer()); +} + +DepsSnapshot capture_deps_snapshot(PathPool& pool, llvm::ArrayRef deps) { + DepsSnapshot snap; + // Capture timestamp BEFORE hashing to avoid TOCTOU: if a file is modified + // during hashing, its mtime will be > build_at, triggering Layer 2 re-hash. + snap.build_at = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + snap.path_ids.reserve(deps.size()); + snap.hashes.reserve(deps.size()); + for(const auto& file: deps) { + snap.path_ids.push_back(pool.intern(file)); + snap.hashes.push_back(hash_file(file)); + } + return snap; +} + +bool deps_changed(const PathPool& pool, const DepsSnapshot& snap) { + for(std::size_t i = 0; i < snap.path_ids.size(); ++i) { + auto path = pool.resolve(snap.path_ids[i]); + llvm::sys::fs::file_status status; + if(auto ec = llvm::sys::fs::status(path, status)) { + // File disappeared — definitely changed. + if(snap.hashes[i] != 0) + return true; + continue; + } + + // Layer 1: mtime check (cheap, stat only). + auto current_mtime = llvm::sys::toTimeT(status.getLastModificationTime()); + if(current_mtime <= snap.build_at) + continue; + + // Layer 2: mtime is newer — re-hash content to confirm actual change. + auto current_hash = hash_file(path); + if(current_hash != snap.hashes[i]) + return true; + } + return false; +} + +// -- Cache persistence structs ------------------------------------------------ + +namespace { + +struct CacheDepEntry { + std::uint32_t path; // index into CacheData::paths + std::uint64_t hash; +}; + +struct CachePCHEntry { + std::string filename; + std::uint32_t source_file; // index into CacheData::paths + std::uint64_t hash; + std::uint32_t bound; + std::int64_t build_at; + std::vector deps; +}; + +struct CachePCMEntry { + std::string filename; + std::uint32_t source_file; + std::string module_name; + std::int64_t build_at; + std::vector deps; +}; + +struct CacheData { + std::vector paths; + std::vector pch; + std::vector pcm; +}; + +} // namespace + +// -- Workspace cache / module methods ----------------------------------------- + +void Workspace::load_cache() { + if(config.cache_dir.empty()) + return; + + auto cache_path = path::join(config.cache_dir, "cache", "cache.json"); + auto content = fs::read(cache_path); + if(!content) { + LOG_DEBUG("No cache.json found at {}", cache_path); + return; + } + + CacheData data; + auto status = eventide::serde::json::from_json(*content, data); + if(!status) { + LOG_WARN("Failed to parse cache.json"); + return; + } + + auto resolve = [&](std::uint32_t idx) -> llvm::StringRef { + return idx < data.paths.size() ? llvm::StringRef(data.paths[idx]) : ""; + }; + + auto load_deps = [&](std::int64_t build_at, const auto& dep_entries) -> DepsSnapshot { + DepsSnapshot deps; + deps.build_at = build_at; + for(auto& dep: dep_entries) { + auto dep_path = resolve(dep.path); + if(dep_path.empty()) + continue; + deps.path_ids.push_back(path_pool.intern(dep_path)); + deps.hashes.push_back(dep.hash); + } + return deps; + }; + + for(auto& entry: data.pch) { + auto pch_path = path::join(config.cache_dir, "cache", "pch", entry.filename); + auto source = resolve(entry.source_file); + if(!llvm::sys::fs::exists(pch_path) || source.empty()) + continue; + + auto path_id = path_pool.intern(source); + auto& st = pch_cache[path_id]; + st.path = pch_path; + st.hash = entry.hash; + st.bound = entry.bound; + st.deps = load_deps(entry.build_at, entry.deps); + + LOG_DEBUG("Loaded cached PCH: {} -> {}", source, pch_path); + } + + for(auto& entry: data.pcm) { + auto pcm_path = path::join(config.cache_dir, "cache", "pcm", entry.filename); + auto source = resolve(entry.source_file); + if(!llvm::sys::fs::exists(pcm_path) || source.empty()) + continue; + + auto path_id = path_pool.intern(source); + pcm_cache[path_id] = {pcm_path, load_deps(entry.build_at, entry.deps)}; + pcm_paths[path_id] = pcm_path; + + LOG_DEBUG("Loaded cached PCM: {} (module {}) -> {}", source, entry.module_name, pcm_path); + } + + LOG_INFO("Loaded cache.json: {} PCH entries, {} PCM entries", + pch_cache.size(), + pcm_cache.size()); +} + +void Workspace::save_cache() { + if(config.cache_dir.empty()) + return; + + CacheData data; + std::unordered_map index_map; + + auto intern = [&](std::uint32_t runtime_path_id) -> std::uint32_t { + auto path = std::string(path_pool.resolve(runtime_path_id)); + auto [it, inserted] = + index_map.try_emplace(path, static_cast(data.paths.size())); + if(inserted) { + data.paths.push_back(path); + } + return it->second; + }; + + for(auto& [path_id, st]: pch_cache) { + if(st.path.empty()) + continue; + + CachePCHEntry entry; + entry.filename = std::string(path::filename(st.path)); + entry.source_file = intern(path_id); + entry.hash = st.hash; + entry.bound = st.bound; + entry.build_at = st.deps.build_at; + for(std::size_t i = 0; i < st.deps.path_ids.size(); ++i) { + entry.deps.push_back({intern(st.deps.path_ids[i]), st.deps.hashes[i]}); + } + data.pch.push_back(std::move(entry)); + } + + for(auto& [path_id, st]: pcm_cache) { + if(st.path.empty()) + continue; + + CachePCMEntry entry; + entry.filename = std::string(path::filename(st.path)); + entry.source_file = intern(path_id); + auto mod_it = path_to_module.find(path_id); + entry.module_name = mod_it != path_to_module.end() ? mod_it->second : ""; + entry.build_at = st.deps.build_at; + for(std::size_t i = 0; i < st.deps.path_ids.size(); ++i) { + entry.deps.push_back({intern(st.deps.path_ids[i]), st.deps.hashes[i]}); + } + data.pcm.push_back(std::move(entry)); + } + + auto json_str = eventide::serde::json::to_json(data); + if(!json_str) { + LOG_WARN("Failed to serialize cache.json"); + return; + } + + auto cache_path = path::join(config.cache_dir, "cache", "cache.json"); + auto tmp_path = cache_path + ".tmp"; + auto write_result = fs::write(tmp_path, *json_str); + if(!write_result) { + LOG_WARN("Failed to write cache.json.tmp: {}", write_result.error().message()); + return; + } + auto rename_result = fs::rename(tmp_path, cache_path); + if(!rename_result) { + LOG_WARN("Failed to rename cache.json.tmp to cache.json: {}", + rename_result.error().message()); + } +} + +void Workspace::cleanup_cache(int max_age_days) { + if(config.cache_dir.empty()) + return; + + auto now = std::chrono::system_clock::now(); + auto max_age = std::chrono::hours(max_age_days * 24); + + for(auto* subdir: {"cache/pch", "cache/pcm"}) { + auto dir = path::join(config.cache_dir, subdir); + std::error_code ec; + for(auto it = llvm::sys::fs::directory_iterator(dir, ec); + !ec && it != llvm::sys::fs::directory_iterator(); + it.increment(ec)) { + llvm::sys::fs::file_status status; + if(auto stat_ec = llvm::sys::fs::status(it->path(), status)) + continue; + + auto mtime = status.getLastModificationTime(); + auto age = now - mtime; + if(age > max_age) { + llvm::sys::fs::remove(it->path()); + LOG_DEBUG("Cleaned up stale cache file: {}", it->path()); + } + } + } +} + +void Workspace::build_module_map() { + for(auto& [module_name, path_ids]: dep_graph.modules()) { + for(auto path_id: path_ids) { + path_to_module[path_id] = module_name.str(); + } + } +} + +void Workspace::fill_pcm_deps(std::unordered_map& pcms, + std::uint32_t exclude_path_id) const { + for(auto& [pid, pcm_path]: pcm_paths) { + if(pid == exclude_path_id) + continue; + auto mod_it = path_to_module.find(pid); + if(mod_it != path_to_module.end()) { + pcms[mod_it->second] = pcm_path; + } + } +} + +void Workspace::cancel_all() { + if(compile_graph) { + compile_graph->cancel_all(); + } +} + } // namespace clice diff --git a/src/server/workspace.h b/src/server/workspace.h index 7ea130787..8d681f54d 100644 --- a/src/server/workspace.h +++ b/src/server/workspace.h @@ -18,6 +18,7 @@ #include "support/path_pool.h" #include "syntax/dependency_graph.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" @@ -240,6 +241,34 @@ struct Workspace { /// Called when a file is closed. Notifies compile_graph if this file /// is a module unit so dependents can be re-evaluated on next compile. void on_file_closed(std::uint32_t path_id); + + // ----- Cache / module methods (moved from Compiler) ----- + + /// Load PCH/PCM cache from cache.json on disk. + void load_cache(); + /// Save PCH/PCM cache to cache.json on disk. + void save_cache(); + /// Remove stale PCH/PCM files older than max_age_days. + void cleanup_cache(int max_age_days = 7); + /// Build path_to_module reverse mapping from dep_graph. + void build_module_map(); + /// Fill PCM paths for all built modules, excluding exclude_path_id. + void fill_pcm_deps(std::unordered_map& pcms, + std::uint32_t exclude_path_id = UINT32_MAX) const; + /// Cancel all in-flight compilations. + void cancel_all(); }; +/// Hash a file's content using xxh3_64bits. Returns 0 on read failure. +std::uint64_t hash_file(llvm::StringRef path); + +/// Capture a two-layer staleness snapshot after a successful compilation. +/// Interns dependency paths into the PathPool and hashes each file's content. +DepsSnapshot capture_deps_snapshot(PathPool& pool, llvm::ArrayRef deps); + +/// Two-layer staleness check. +/// Layer 1 (fast): stat each dep file, compare mtime against build_at. +/// Layer 2 (precise): for files with mtime > build_at, re-hash content. +bool deps_changed(const PathPool& pool, const DepsSnapshot& snap); + } // namespace clice From 7a8dcd060657d7511e622f22b9eceed5e4368c38 Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 11:04:12 +0800 Subject: [PATCH 03/18] style: remove section divider comments from workspace.h/cpp Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/workspace.h | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/server/workspace.h b/src/server/workspace.h index 8d681f54d..ab87b0eeb 100644 --- a/src/server/workspace.h +++ b/src/server/workspace.h @@ -29,9 +29,6 @@ namespace et = eventide; namespace protocol = et::ipc::protocol; namespace lsp = et::ipc::lsp; -// --------------------------------------------------------------------------- -// Shared types used by both Workspace and Session -// --------------------------------------------------------------------------- /// Two-layer staleness snapshot for compilation artifacts (PCH, AST, etc.). /// @@ -54,9 +51,6 @@ struct HeaderFileContext { std::uint64_t preamble_hash; ///< Hash of preamble content for staleness. }; -// --------------------------------------------------------------------------- -// Index wrapper types (used by Workspace and Session) -// --------------------------------------------------------------------------- /// In-memory index for an open file. Kept separate from MergedIndex because /// open files change frequently, are based on unsaved buffer content, and only @@ -141,9 +135,6 @@ struct MergedIndexShard { } }; -// --------------------------------------------------------------------------- -// PCH / PCM caches -// --------------------------------------------------------------------------- /// Cached PCH state. Content-addressed by preamble hash — shared across all /// files (open or on-disk) that have the same preamble content. @@ -162,9 +153,6 @@ struct PCMState { DepsSnapshot deps; }; -// --------------------------------------------------------------------------- -// Workspace -// --------------------------------------------------------------------------- /// All persistent, project-wide state derived from files on disk. /// From 408c1a6bfc3da50fd9013e5caf30c43dec1a5020 Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 11:16:29 +0800 Subject: [PATCH 04/18] style: remove section divider comments across src/ Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/protocol.h | 3 --- src/server/worker_pool.h | 1 - src/syntax/dependency_graph.cpp | 6 ------ 3 files changed, 10 deletions(-) diff --git a/src/server/protocol.h b/src/server/protocol.h index 5ea5c5650..7f79ee25c 100644 --- a/src/server/protocol.h +++ b/src/server/protocol.h @@ -103,7 +103,6 @@ struct BuildResult { eventide::serde::RawValue result_json; ///< Completion/SignatureHelp result }; -// === Notifications === struct DocumentUpdateParams { std::string path; @@ -120,7 +119,6 @@ struct EvictedParams { } // namespace clice::worker -// === clice/ LSP Extension Types === namespace clice::ext { @@ -179,7 +177,6 @@ struct RequestTraits { constexpr inline static std::string_view method = "clice/worker/build"; }; -// === Notifications === template <> struct NotificationTraits { diff --git a/src/server/worker_pool.h b/src/server/worker_pool.h index 162bfcd6c..c4b80257e 100644 --- a/src/server/worker_pool.h +++ b/src/server/worker_pool.h @@ -83,7 +83,6 @@ class WorkerPool { bool spawn_worker(const std::string& self_path, bool stateful, std::uint64_t memory_limit); }; -// --- Template implementations --------------------------------------------------- template RequestResult WorkerPool::send_stateful(std::uint32_t path_id, diff --git a/src/syntax/dependency_graph.cpp b/src/syntax/dependency_graph.cpp index 9443dc13f..47fbee66c 100644 --- a/src/syntax/dependency_graph.cpp +++ b/src/syntax/dependency_graph.cpp @@ -20,9 +20,7 @@ namespace clice { namespace et = eventide; -// ============================================================================ // DependencyGraph implementation -// ============================================================================ void DependencyGraph::add_module(llvm::StringRef module_name, std::uint32_t path_id) { auto& ids = module_to_path[module_name]; @@ -205,9 +203,7 @@ std::vector DependencyGraph::find_include_chain(std::uint32_t hos return chain; } -// ============================================================================ // Wavefront BFS scanner — async implementation -// ============================================================================ namespace { @@ -819,9 +815,7 @@ et::task<> scan_impl(CompilationDatabase& cdb, } // namespace -// ============================================================================ // Public sync entry point -// ============================================================================ ScanReport scan_dependency_graph(CompilationDatabase& cdb, PathPool& path_pool, From 86f3f896eec9bb36cf048ff850f180c70bd479e1 Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 11:18:59 +0800 Subject: [PATCH 05/18] style: remove all section divider comments across src/server/ Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/compiler.h | 4 ---- src/server/indexer.cpp | 5 ----- src/server/master_server.cpp | 12 ------------ src/server/session.h | 5 ----- src/server/workspace.cpp | 6 ------ src/server/workspace.h | 7 ------- 6 files changed, 39 deletions(-) diff --git a/src/server/compiler.h b/src/server/compiler.h index 0a10afa6d..6ece99853 100644 --- a/src/server/compiler.h +++ b/src/server/compiler.h @@ -58,11 +58,9 @@ class Compiler { llvm::DenseMap& sessions); ~Compiler(); - // ----- Module / compile graph (operates on Workspace) ----- void init_compile_graph(); - // ----- Compile argument resolution ----- /// Fill compile arguments for a file (CDB lookup + header context fallback). /// @param session If non-null, used for header context resolution on open files. @@ -71,13 +69,11 @@ class Compiler { std::vector& arguments, Session* session = nullptr); - // ----- Pull-based compilation ----- /// Compile an open file's AST if dirty. On success, updates session's /// file_index, pch_ref, ast_deps, and publishes diagnostics. et::task ensure_compiled(Session& session); - // ----- Feature request forwarding to workers ----- using RawResult = et::task; diff --git a/src/server/indexer.cpp b/src/server/indexer.cpp index 5ebf71960..c31b5a230 100644 --- a/src/server/indexer.cpp +++ b/src/server/indexer.cpp @@ -21,7 +21,6 @@ namespace clice { namespace lsp = eventide::ipc::lsp; -// -- Indexer: data management ------------------------------------------------- void Indexer::merge(const void* tu_index_data, std::size_t size) { auto tu_index = index::TUIndex::from(tu_index_data); @@ -183,7 +182,6 @@ bool Indexer::need_update(llvm::StringRef file_path) { return merged_it->second.index.need_update(path_mapping); } -// -- Indexer: symbol queries -------------------------------------------------- bool Indexer::find_symbol_info(index::SymbolHash hash, std::string& name, SymbolKind& kind) const { for(auto& [_, session]: sessions) { @@ -369,7 +367,6 @@ std::optional return lookup_symbol(uri, path, range.start, session); } -// -- Indexer: relation collection helpers -------------------------------------- void Indexer::collect_grouped_relations( index::SymbolHash hash, @@ -436,7 +433,6 @@ void Indexer::collect_unique_targets(index::SymbolHash hash, } } -// -- Indexer: hierarchy queries ------------------------------------------------ /// Resolve a symbol hash into a SymbolInfo with definition location. /// Returns nullopt if the symbol or its definition cannot be found. @@ -581,7 +577,6 @@ std::vector Indexer::search_symbols(llvm::StringRef return results; } -// -- Indexer: static utilities ------------------------------------------------ protocol::SymbolKind Indexer::to_lsp_symbol_kind(SymbolKind kind) { switch(kind) { diff --git a/src/server/master_server.cpp b/src/server/master_server.cpp index 7411dccd7..04be093e3 100644 --- a/src/server/master_server.cpp +++ b/src/server/master_server.cpp @@ -362,9 +362,7 @@ void MasterServer::register_handlers() { }()); }); - /// --------------------------------------------------------------- /// Document lifecycle — handled directly by MasterServer. - /// --------------------------------------------------------------- peer.on_notification([this](const protocol::DidOpenTextDocumentParams& params) { if(lifecycle != ServerLifecycle::Ready) @@ -482,9 +480,7 @@ void MasterServer::register_handlers() { LOG_DEBUG("didSave: {}", path); }); - /// --------------------------------------------------------------- /// Feature requests — stateful forwarding. - /// --------------------------------------------------------------- peer.on_request([this](RequestContext& ctx, const protocol::HoverParams& params) -> RawResult { auto path = uri_to_path(params.text_document_position_params.text_document.uri); @@ -592,9 +588,7 @@ void MasterServer::register_handlers() { return indexer.resolve_hierarchy_item(uri, path, range, data, session); }; - /// --------------------------------------------------------------- /// Feature requests — index-based with AST fallback. - /// --------------------------------------------------------------- peer.on_request([this, query_at](RequestContext& ctx, const protocol::DefinitionParams& params) -> RawResult { @@ -650,9 +644,7 @@ void MasterServer::register_handlers() { co_return serde_raw{"null"}; }); - /// --------------------------------------------------------------- /// Feature requests — stateless forwarding. - /// --------------------------------------------------------------- peer.on_request([this](RequestContext& ctx, const protocol::CompletionParams& params) -> RawResult { @@ -677,9 +669,7 @@ void MasterServer::register_handlers() { sit->second); }); - /// --------------------------------------------------------------- /// Hierarchy queries — index-based. - /// --------------------------------------------------------------- peer.on_request( [this, lookup_at](RequestContext& ctx, @@ -772,9 +762,7 @@ void MasterServer::register_handlers() { co_return to_raw(results); }); - /// --------------------------------------------------------------- /// clice/ extension commands. - /// --------------------------------------------------------------- peer.on_request( "clice/queryContext", diff --git a/src/server/session.h b/src/server/session.h index 9a4fc229c..1ba04cae2 100644 --- a/src/server/session.h +++ b/src/server/session.h @@ -26,12 +26,10 @@ namespace et = eventide; /// Sessions may READ from Workspace (e.g. to obtain PCH/PCM paths, module /// mappings, include graph) but all compilation results stay here. struct Session { - // ----- Identity ----- /// Path ID of this file in PathPool. Set on creation, never changes. std::uint32_t path_id = 0; - // ----- Document state (from LSP) ----- /// LSP document version, incremented by the client on each edit. int version = 0; @@ -56,7 +54,6 @@ struct Session { std::shared_ptr compiling; - // ----- Compilation artifacts (per-file, references Workspace caches) ----- /// Reference to the PCH entry in Workspace.pch_cache, if any. /// The PCH itself is owned by Workspace (shared, content-addressed); @@ -73,7 +70,6 @@ struct Session { /// Used for two-layer staleness detection (mtime + content hash). std::optional ast_deps; - // ----- Header context (only meaningful for header files) ----- /// Compilation context for header files that lack their own CDB entry. /// Stores the host source file and synthesized preamble for this header. @@ -83,7 +79,6 @@ struct Session { /// When set, overrides automatic header context resolution. std::optional active_context; - // ----- Index (per-file, not merged into Workspace) ----- /// Symbol index built from the latest compilation of this file's buffer. /// Used for queries (hover, goto, references) on this file. diff --git a/src/server/workspace.cpp b/src/server/workspace.cpp index 5268b1c85..b233aa03c 100644 --- a/src/server/workspace.cpp +++ b/src/server/workspace.cpp @@ -35,7 +35,6 @@ const static index::Occurrence* lookup_occurrence(const std::vector> OpenFileIndex::find_occurrence(std::uint32_t offset) const { @@ -54,7 +53,6 @@ std::optional> }; } -// -- MergedIndexShard --------------------------------------------------------- std::optional> MergedIndexShard::find_occurrence(std::uint32_t offset) const { @@ -76,7 +74,6 @@ std::optional> return result; } -// -- Workspace ---------------------------------------------------------------- llvm::SmallVector Workspace::on_file_saved(std::uint32_t path_id) { llvm::SmallVector dirtied; @@ -98,7 +95,6 @@ void Workspace::on_file_closed(std::uint32_t path_id) { pch_cache.erase(path_id); } -// -- Free helper functions (used by Workspace and Compiler) ------------------- std::uint64_t hash_file(llvm::StringRef path) { auto buf = llvm::MemoryBuffer::getFile(path); @@ -145,7 +141,6 @@ bool deps_changed(const PathPool& pool, const DepsSnapshot& snap) { return false; } -// -- Cache persistence structs ------------------------------------------------ namespace { @@ -179,7 +174,6 @@ struct CacheData { } // namespace -// -- Workspace cache / module methods ----------------------------------------- void Workspace::load_cache() { if(config.cache_dir.empty()) diff --git a/src/server/workspace.h b/src/server/workspace.h index ab87b0eeb..1af414039 100644 --- a/src/server/workspace.h +++ b/src/server/workspace.h @@ -171,16 +171,13 @@ struct PCMState { /// Sessions (open files) may READ from Workspace but must not WRITE to it /// except through the didSave path. struct Workspace { - // ----- Configuration ----- CliceConfig config; CompilationDatabase cdb; - // ----- Path management ----- PathPool path_pool; - // ----- Dependency graphs ----- /// Include relationships between files on disk (#include edges). /// Built once at startup from CDB scan; updated incrementally on didSave. @@ -195,7 +192,6 @@ struct Workspace { /// declarations change. llvm::DenseMap path_to_module; - // ----- Compilation caches (shared, content-addressed) ----- /// PCH cache, keyed by file path_id. Content-addressed by preamble hash, /// so different files with identical preambles share the same PCH. @@ -209,7 +205,6 @@ struct Workspace { /// Maps to the .pcm file on disk used as -fmodule-file argument. llvm::DenseMap pcm_paths; - // ----- Symbol index (disk-derived) ----- /// Global symbol table across all indexed translation units. index::ProjectIndex project_index; @@ -219,7 +214,6 @@ struct Workspace { /// for position mapping. llvm::DenseMap merged_indices; - // ----- Lifecycle methods ----- /// Called when a file is saved to disk. Cascades invalidation through /// compile_graph and clears affected PCM caches. @@ -230,7 +224,6 @@ struct Workspace { /// is a module unit so dependents can be re-evaluated on next compile. void on_file_closed(std::uint32_t path_id); - // ----- Cache / module methods (moved from Compiler) ----- /// Load PCH/PCM cache from cache.json on disk. void load_cache(); From 65587a42960f8049b0124315966edbefac6a792b Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 11:26:13 +0800 Subject: [PATCH 06/18] style: remove section divider comments and apply formatting Co-Authored-By: Claude Opus 4.6 (1M context) --- benchmarks/pch_chain/README.md | 56 ++++++ benchmarks/pch_chain/bench_all_headers.sh | 201 ++++++++++++++++++++ benchmarks/pch_chain/bench_pch_chain.sh | 218 ++++++++++++++++++++++ benchmarks/pch_chain/find_breakpoint.sh | 40 ++++ benchmarks/pch_chain/find_exact.sh | 85 +++++++++ benchmarks/pch_chain/isolate_cmath.sh | 81 ++++++++ benchmarks/pch_chain/minimal_repro.sh | 64 +++++++ benchmarks/pch_chain/test_versions.sh | 29 +++ src/server/compiler.h | 4 - src/server/indexer.cpp | 5 - src/server/protocol.h | 3 - src/server/session.h | 5 - src/server/worker_pool.h | 1 - src/server/workspace.cpp | 6 - src/server/workspace.h | 11 -- 15 files changed, 774 insertions(+), 35 deletions(-) create mode 100644 benchmarks/pch_chain/README.md create mode 100755 benchmarks/pch_chain/bench_all_headers.sh create mode 100755 benchmarks/pch_chain/bench_pch_chain.sh create mode 100644 benchmarks/pch_chain/find_breakpoint.sh create mode 100644 benchmarks/pch_chain/find_exact.sh create mode 100644 benchmarks/pch_chain/isolate_cmath.sh create mode 100644 benchmarks/pch_chain/minimal_repro.sh create mode 100644 benchmarks/pch_chain/test_versions.sh diff --git a/benchmarks/pch_chain/README.md b/benchmarks/pch_chain/README.md new file mode 100644 index 000000000..6e2bf1b29 --- /dev/null +++ b/benchmarks/pch_chain/README.md @@ -0,0 +1,56 @@ +# Chained PCH Benchmark Results + +clang 20.1.8, libstdc++ 14.2.0, Linux x86_64 + +## Summary + +| Metric | Monolithic PCH | Chained PCH | +| ------------------------------- | -------------------- | ----------------------- | +| Initial build (20 headers) | 869ms | 1488ms (1.7x slower) | +| Incremental rebuild (+1 header) | 917ms (full rebuild) | 226ms (append one link) | +| Incremental speedup | — | **4x** | + +## Bug: `` breaks macro visibility in chained PCH + +**Reproduction**: Any PCH chain that includes `` followed by `` fails: + +``` +algorithm.pch -> cmath.pch -> deque.pch ← FAIL: undeclared '__N' +``` + +But both of these work: + +``` +cmath.pch -> deque.pch ← PASS (2 links, no prior chain) +algorithm.pch -> deque.pch ← PASS (no cmath) +``` + +**Affected headers** (all fail when chained after ``): + +- ``, ``, ``, ``, ``, + ``, ``, ``, ``, `` + +All use `__N` macro from libstdc++'s `bits/c++config.h`. + +**Root cause**: The `__N` macro IS present in the chained PCH (verified via `-E`), +but becomes invisible to certain headers during compilation with the chained PCH. +This is likely a clang serialization bug where macro visibility scope is lost +across PCH chain boundaries when the defining header (`c++config.h`) was included +in a link prior to ``. + +**Workaround for clice**: When building chained PCH, merge `` and its +predecessor into a single link, or use a monolithic PCH when `` is present. + +## Chain depth limit + +Tested up to **50 trivial links** and **90 real-header links** — no depth limit +observed. The bug is specific to `` + libstdc++ `__N` macro interaction, +not chain depth. + +## Files + +- `bench_pch_chain.sh` — Basic benchmark (20 headers, monolithic vs chain, incremental) +- `bench_all_headers.sh` — Exhaustive test with all 100 C++ standard headers +- `find_breakpoint.sh` — Binary search for chain depth breakpoint +- `find_exact.sh` — Precise breakpoint isolation +- `isolate_cmath.sh` — Isolation of `` as the trigger diff --git a/benchmarks/pch_chain/bench_all_headers.sh b/benchmarks/pch_chain/bench_all_headers.sh new file mode 100755 index 000000000..b67ba7b1f --- /dev/null +++ b/benchmarks/pch_chain/bench_all_headers.sh @@ -0,0 +1,201 @@ +#!/bin/bash +# +# Exhaustive chained PCH test: try ALL C++ standard library headers. +# Build a chain where each link adds one header on top of the previous PCH. +# Report which header (if any) breaks the chain. +# +set -uo pipefail + +CLANG="${CLANG:-clang++}" +STD="${STD:--std=c++20}" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +echo "=== Exhaustive Chained PCH Test ===" +echo "clang: $($CLANG --version | head -1)" +echo "workdir: $WORKDIR" +echo "" + +# ALL C++20 standard library headers (sorted, no sub-directories) +HEADERS=( + algorithm + any + array + atomic + barrier + bit + bitset + cassert + cctype + cerrno + cfenv + cfloat + charconv + chrono + cinttypes + climits + clocale + cmath + codecvt + compare + complex + concepts + condition_variable + coroutine + csetjmp + csignal + cstdarg + cstddef + cstdint + cstdio + cstdlib + cstring + ctime + cuchar + cwchar + cwctype + deque + exception + execution + expected + filesystem + format + forward_list + fstream + functional + future + initializer_list + iomanip + ios + iosfwd + iostream + istream + iterator + latch + limits + list + locale + map + memory + memory_resource + mutex + new + numbers + numeric + optional + ostream + print + queue + random + ranges + ratio + regex + scoped_allocator + semaphore + set + shared_mutex + source_location + span + spanstream + sstream + stack + stdexcept + stop_token + streambuf + string + string_view + syncstream + system_error + thread + tuple + type_traits + typeindex + typeinfo + unordered_map + unordered_set + utility + valarray + variant + vector + version +) + +NUM=${#HEADERS[@]} +echo "Total headers: $NUM" +echo "" + +PREV_PCH="" +PASS=0 +FAIL=0 +FAIL_LIST="" +TOTAL_START=$(date +%s%N) + +for i in "${!HEADERS[@]}"; do + hdr="${HEADERS[$i]}" + idx=$((i + 1)) + HDR_FILE="$WORKDIR/link_${idx}.h" + PCH_FILE="$WORKDIR/link_${idx}.pch" + + echo "#include <$hdr>" > "$HDR_FILE" + + LINK_START=$(date +%s%N) + if [ -z "$PREV_PCH" ]; then + OUTPUT=$($CLANG -x c++-header $STD -o "$PCH_FILE" "$HDR_FILE" 2>&1) + RC=$? + else + OUTPUT=$($CLANG -x c++-header $STD -include-pch "$PREV_PCH" -o "$PCH_FILE" "$HDR_FILE" 2>&1) + RC=$? + fi + LINK_END=$(date +%s%N) + LINK_MS=$(( (LINK_END - LINK_START) / 1000000 )) + + if [ $RC -eq 0 ] && [ -f "$PCH_FILE" ]; then + SIZE=$(stat -c%s "$PCH_FILE") + SIZE_KB=$(( SIZE / 1024 )) + printf " [PASS] %3d/%d %-25s %5dms %5dKB\n" "$idx" "$NUM" "<$hdr>" "$LINK_MS" "$SIZE_KB" + PREV_PCH="$PCH_FILE" + PASS=$((PASS + 1)) + else + printf " [FAIL] %3d/%d %-25s %5dms\n" "$idx" "$NUM" "<$hdr>" "$LINK_MS" + if [ -n "$OUTPUT" ]; then + echo " Error: $(echo "$OUTPUT" | head -3)" + fi + FAIL=$((FAIL + 1)) + FAIL_LIST="$FAIL_LIST $hdr" + # Skip this header in the chain, continue with previous PCH + fi +done + +TOTAL_END=$(date +%s%N) +TOTAL_MS=$(( (TOTAL_END - TOTAL_START) / 1000000 )) + +echo "" +echo "=========================================" +echo "RESULTS" +echo "=========================================" +echo " Total headers: $NUM" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Total time: ${TOTAL_MS}ms" +if [ $FAIL -gt 0 ]; then + echo " Failed headers:$FAIL_LIST" +fi +echo "" + +# Final correctness check: compile a test file with the last successful PCH +if [ -n "$PREV_PCH" ] && [ -f "$PREV_PCH" ]; then + cat > "$WORKDIR/final_test.cpp" << 'SRCEOF' +#include +int main() { + static_assert(std::is_integral_v); + return 0; +} +SRCEOF + echo "Final PCH correctness check..." + if $CLANG $STD -include-pch "$PREV_PCH" -fsyntax-only "$WORKDIR/final_test.cpp" 2>&1; then + echo " Result: PASS" + else + echo " Result: FAIL" + fi + echo " Chain depth at end: $PASS links" +fi diff --git a/benchmarks/pch_chain/bench_pch_chain.sh b/benchmarks/pch_chain/bench_pch_chain.sh new file mode 100755 index 000000000..e6f25165c --- /dev/null +++ b/benchmarks/pch_chain/bench_pch_chain.sh @@ -0,0 +1,218 @@ +#!/bin/bash +# +# Benchmark: Monolithic PCH vs Chained PCH (one-include-per-link) +# +# Tests whether clang's chained PCH works correctly with 15+ chain links, +# and compares build time against a single monolithic PCH. +# +set -euo pipefail + +CLANG="${CLANG:-clang++}" +STD="${STD:--std=c++20}" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +echo "=== PCH Chain Benchmark ===" +echo "clang: $($CLANG --version | head -1)" +echo "workdir: $WORKDIR" +echo "" + +# --- Standard library headers to include (order matters for chaining) --- +HEADERS=( + "cstddef" + "cstdint" + "type_traits" + "utility" + "memory" + "string" + "string_view" + "vector" + "array" + "map" + "unordered_map" + "set" + "optional" + "variant" + "functional" + "algorithm" + "numeric" + "iostream" + "sstream" + "format" +) + +NUM_HEADERS=${#HEADERS[@]} +echo "Headers: $NUM_HEADERS" +echo "List: ${HEADERS[*]}" +echo "" + +# ============================================================ +# 1. Monolithic PCH: one file with all includes +# ============================================================ + +echo "=========================================" +echo "1. MONOLITHIC PCH" +echo "=========================================" + +# Generate the preamble header +MONO_HDR="$WORKDIR/mono_preamble.h" +for hdr in "${HEADERS[@]}"; do + echo "#include <$hdr>" >> "$MONO_HDR" +done + +echo "Building monolithic PCH ($NUM_HEADERS includes)..." +MONO_START=$(date +%s%N) +$CLANG -x c++-header $STD -o "$WORKDIR/mono.pch" "$MONO_HDR" 2>&1 +MONO_END=$(date +%s%N) +MONO_MS=$(( (MONO_END - MONO_START) / 1000000 )) +MONO_SIZE=$(stat -c%s "$WORKDIR/mono.pch") +echo " Time: ${MONO_MS}ms" +echo " Size: $(( MONO_SIZE / 1024 ))KB" + +# Test: compile a source file using the monolithic PCH +cat > "$WORKDIR/test_mono.cpp" << 'SRCEOF' +int main() { + std::vector v = {"hello", "world"}; + std::map m = {{1, "one"}, {2, "two"}}; + auto opt = std::make_optional(42); + std::cout << v[0] << " " << m[1] << " " << *opt << std::endl; + return 0; +} +SRCEOF + +echo " Compiling test with monolithic PCH..." +if $CLANG $STD -include-pch "$WORKDIR/mono.pch" -fsyntax-only "$WORKDIR/test_mono.cpp" 2>&1; then + echo " Result: PASS" +else + echo " Result: FAIL" +fi +echo "" + +# ============================================================ +# 2. Chained PCH: one PCH per include, each chaining the previous +# ============================================================ + +echo "=========================================" +echo "2. CHAINED PCH (one include per link)" +echo "=========================================" + +CHAIN_TOTAL_START=$(date +%s%N) + +PREV_PCH="" +for i in "${!HEADERS[@]}"; do + hdr="${HEADERS[$i]}" + idx=$((i + 1)) + CHAIN_HDR="$WORKDIR/chain_${idx}.h" + CHAIN_PCH="$WORKDIR/chain_${idx}.pch" + + echo "#include <$hdr>" > "$CHAIN_HDR" + + LINK_START=$(date +%s%N) + if [ -z "$PREV_PCH" ]; then + # First link: no previous PCH + if ! $CLANG -x c++-header $STD -o "$CHAIN_PCH" "$CHAIN_HDR" 2>&1; then + echo " FAIL at link $idx ($hdr) — build error" + break + fi + else + # Subsequent links: chain from previous PCH + if ! $CLANG -x c++-header $STD -include-pch "$PREV_PCH" -o "$CHAIN_PCH" "$CHAIN_HDR" 2>&1; then + echo " FAIL at link $idx ($hdr) — build error" + echo " (chained PCH may have a bug at this chain depth)" + break + fi + fi + LINK_END=$(date +%s%N) + LINK_MS=$(( (LINK_END - LINK_START) / 1000000 )) + LINK_SIZE=$(stat -c%s "$CHAIN_PCH" 2>/dev/null || echo 0) + + echo " Link $idx/$NUM_HEADERS: <$hdr> ${LINK_MS}ms $(( LINK_SIZE / 1024 ))KB" + PREV_PCH="$CHAIN_PCH" +done + +CHAIN_TOTAL_END=$(date +%s%N) +CHAIN_TOTAL_MS=$(( (CHAIN_TOTAL_END - CHAIN_TOTAL_START) / 1000000 )) + +echo "" +echo " Total chain build time: ${CHAIN_TOTAL_MS}ms" +if [ -n "$PREV_PCH" ] && [ -f "$PREV_PCH" ]; then + LAST_SIZE=$(stat -c%s "$PREV_PCH") + echo " Final PCH size: $(( LAST_SIZE / 1024 ))KB" +fi +echo "" + +# Test: compile the same source file using the final chained PCH +echo " Compiling test with chained PCH (final link)..." +if [ -n "$PREV_PCH" ] && [ -f "$PREV_PCH" ]; then + if $CLANG $STD -include-pch "$PREV_PCH" -fsyntax-only "$WORKDIR/test_mono.cpp" 2>&1; then + echo " Result: PASS" + else + echo " Result: FAIL" + fi +else + echo " Result: SKIP (chain build failed)" +fi +echo "" + +# ============================================================ +# 3. Incremental rebuild simulation: add one more include +# ============================================================ + +echo "=========================================" +echo "3. INCREMENTAL REBUILD (add )" +echo "=========================================" + +EXTRA_HDR="$WORKDIR/chain_extra.h" +EXTRA_PCH="$WORKDIR/chain_extra.pch" +echo "#include " > "$EXTRA_HDR" + +echo " Monolithic: rebuild entire PCH..." +echo "#include " >> "$MONO_HDR" +MONO_REBUILD_START=$(date +%s%N) +$CLANG -x c++-header $STD -o "$WORKDIR/mono.pch" "$MONO_HDR" 2>&1 +MONO_REBUILD_END=$(date +%s%N) +MONO_REBUILD_MS=$(( (MONO_REBUILD_END - MONO_REBUILD_START) / 1000000 )) +echo " Monolithic rebuild: ${MONO_REBUILD_MS}ms" + +if [ -n "$PREV_PCH" ] && [ -f "$PREV_PCH" ]; then + echo " Chained: append one link..." + INCR_START=$(date +%s%N) + if $CLANG -x c++-header $STD -include-pch "$PREV_PCH" -o "$EXTRA_PCH" "$EXTRA_HDR" 2>&1; then + INCR_END=$(date +%s%N) + INCR_MS=$(( (INCR_END - INCR_START) / 1000000 )) + echo " Chained incremental: ${INCR_MS}ms" + echo " Speedup: $(( MONO_REBUILD_MS / (INCR_MS > 0 ? INCR_MS : 1) ))x" + + # Verify the incremental PCH works + cat > "$WORKDIR/test_incr.cpp" << 'SRCEOF' +int main() { + auto now = std::chrono::system_clock::now(); + std::vector v = {"hello"}; + std::cout << v[0] << std::endl; + return 0; +} +SRCEOF + echo " Verifying incremental PCH..." + if $CLANG $STD -include-pch "$EXTRA_PCH" -fsyntax-only "$WORKDIR/test_incr.cpp" 2>&1; then + echo " Result: PASS" + else + echo " Result: FAIL" + fi + else + echo " Chained incremental: FAIL (build error)" + fi +else + echo " Skipped (chain build failed earlier)" +fi + +echo "" +echo "=========================================" +echo "SUMMARY" +echo "=========================================" +echo " Chain length: $NUM_HEADERS headers" +echo " Monolithic PCH build: ${MONO_MS}ms" +echo " Chained PCH total: ${CHAIN_TOTAL_MS}ms" +echo " Monolithic rebuild: ${MONO_REBUILD_MS}ms" +if [ -n "${INCR_MS:-}" ]; then +echo " Chained incremental: ${INCR_MS}ms" +fi diff --git a/benchmarks/pch_chain/find_breakpoint.sh b/benchmarks/pch_chain/find_breakpoint.sh new file mode 100644 index 000000000..7d903541b --- /dev/null +++ b/benchmarks/pch_chain/find_breakpoint.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -uo pipefail + +CLANG="${CLANG:-clang++}" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +HEADERS=(algorithm any array atomic barrier bit bitset cassert cctype cerrno cfenv cfloat charconv chrono cinttypes climits clocale cmath codecvt compare complex concepts condition_variable coroutine csetjmp csignal cstdarg cstddef cstdint cstdio cstdlib cstring ctime cuchar cwchar cwctype) + +echo "=== Finding chain depth breakpoint for ===" + +for depth in 1 2 3 5 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36; do + if [ $depth -gt ${#HEADERS[@]} ]; then + break + fi + + PREV="" + for ((i=0; i" > "$WORKDIR/l_${i}.h" + if [ -z "$PREV" ]; then + $CLANG -x c++-header -std=c++20 -o "$WORKDIR/l_${i}.pch" "$WORKDIR/l_${i}.h" 2>/dev/null + else + $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/l_${i}.pch" "$WORKDIR/l_${i}.h" 2>/dev/null + fi + PREV="$WORKDIR/l_${i}.pch" + done + + echo "#include " > "$WORKDIR/test_deque.h" + ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test_deque.pch" "$WORKDIR/test_deque.h" 2>&1) + RC=$? + if [ $RC -eq 0 ]; then + echo " depth=$depth: PASS" + else + ERRMSG=$(echo "$ERR" | grep "error:" | head -1) + echo " depth=$depth: FAIL — $ERRMSG" + fi + + rm -f "$WORKDIR"/l_*.h "$WORKDIR"/l_*.pch "$WORKDIR"/test_deque.* +done diff --git a/benchmarks/pch_chain/find_exact.sh b/benchmarks/pch_chain/find_exact.sh new file mode 100644 index 000000000..75a53988c --- /dev/null +++ b/benchmarks/pch_chain/find_exact.sh @@ -0,0 +1,85 @@ +#!/bin/bash +set -uo pipefail + +CLANG="${CLANG:-clang++}" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +HEADERS=(algorithm any array atomic barrier bit bitset cassert cctype cerrno cfenv cfloat charconv chrono cinttypes climits clocale cmath codecvt compare complex concepts condition_variable coroutine csetjmp csignal cstdarg cstddef cstdint cstdio cstdlib cstring ctime cuchar cwchar cwctype) + +echo "=== Exact breakpoint: depth 16-18 ===" + +for depth in 16 17 18; do + PREV="" + for ((i=0; i" > "$WORKDIR/l_${i}.h" + if [ -z "$PREV" ]; then + $CLANG -x c++-header -std=c++20 -o "$WORKDIR/l_${i}.pch" "$WORKDIR/l_${i}.h" 2>/dev/null + else + $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/l_${i}.pch" "$WORKDIR/l_${i}.h" 2>/dev/null + fi + PREV="$WORKDIR/l_${i}.pch" + done + + echo "#include " > "$WORKDIR/test.h" + ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test.pch" "$WORKDIR/test.h" 2>&1) + RC=$? + echo " depth=$depth (last hdr=${HEADERS[$((depth-1))]}): $([ $RC -eq 0 ] && echo PASS || echo FAIL)" + + rm -f "$WORKDIR"/l_*.h "$WORKDIR"/l_*.pch "$WORKDIR"/test.* +done + +echo "" +echo "=== Confirming: is it depth or is it the specific header? ===" +echo "depth=17 headers: ${HEADERS[*]:0:17}" +echo "" + +# Test: 17 links but with different ordering — is it just depth? +echo "Test: 17 trivial headers (cstddef repeated style)..." +PREV="" +for ((i=0; i<17; i++)); do + echo "#include " > "$WORKDIR/triv_${i}.h" + if [ -z "$PREV" ]; then + $CLANG -x c++-header -std=c++20 -o "$WORKDIR/triv_${i}.pch" "$WORKDIR/triv_${i}.h" 2>/dev/null + else + # Each link is a no-op (header guard prevents re-inclusion) + $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/triv_${i}.pch" "$WORKDIR/triv_${i}.h" 2>/dev/null + fi + PREV="$WORKDIR/triv_${i}.pch" +done +echo "#include " > "$WORKDIR/test.h" +ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test.pch" "$WORKDIR/test.h" 2>&1) +echo " 17 trivial links + deque: $([ $? -eq 0 ] && echo PASS || echo FAIL)" +rm -f "$WORKDIR"/triv_*.h "$WORKDIR"/triv_*.pch "$WORKDIR"/test.* + +# Test: 30 trivial headers +PREV="" +for ((i=0; i<30; i++)); do + echo "#include " > "$WORKDIR/triv_${i}.h" + if [ -z "$PREV" ]; then + $CLANG -x c++-header -std=c++20 -o "$WORKDIR/triv_${i}.pch" "$WORKDIR/triv_${i}.h" 2>/dev/null + else + $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/triv_${i}.pch" "$WORKDIR/triv_${i}.h" 2>/dev/null + fi + PREV="$WORKDIR/triv_${i}.pch" +done +echo "#include " > "$WORKDIR/test.h" +ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test.pch" "$WORKDIR/test.h" 2>&1) +echo " 30 trivial links + deque: $([ $? -eq 0 ] && echo PASS || echo FAIL)" +rm -f "$WORKDIR"/triv_*.h "$WORKDIR"/triv_*.pch "$WORKDIR"/test.* + +# Test: 50 trivial headers +PREV="" +for ((i=0; i<50; i++)); do + echo "#include " > "$WORKDIR/triv_${i}.h" + if [ -z "$PREV" ]; then + $CLANG -x c++-header -std=c++20 -o "$WORKDIR/triv_${i}.pch" "$WORKDIR/triv_${i}.h" 2>/dev/null + else + $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/triv_${i}.pch" "$WORKDIR/triv_${i}.h" 2>/dev/null + fi + PREV="$WORKDIR/triv_${i}.pch" +done +echo "#include " > "$WORKDIR/test.h" +ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test.pch" "$WORKDIR/test.h" 2>&1) +echo " 50 trivial links + deque: $([ $? -eq 0 ] && echo PASS || echo FAIL)" diff --git a/benchmarks/pch_chain/isolate_cmath.sh b/benchmarks/pch_chain/isolate_cmath.sh new file mode 100644 index 000000000..fdaf36806 --- /dev/null +++ b/benchmarks/pch_chain/isolate_cmath.sh @@ -0,0 +1,81 @@ +#!/bin/bash +set -uo pipefail + +CLANG="${CLANG:-clang++}" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +# We know: 17-link chain ending at clocale + deque = PASS +# 18-link chain ending at cmath + deque = FAIL +# But cmath -> deque alone = PASS +# So it's the combination of deep chain + cmath that causes __N loss + +# Test: just the "heavy" headers in chain, then cmath, then deque +echo "=== Test: minimal chain that breaks ===" + +# Which headers before cmath are "heavy" (pull in lots of stuff)? +# algorithm, atomic, barrier, charconv, chrono are the big ones +COMBOS=( + "algorithm cmath" + "chrono cmath" + "barrier cmath" + "algorithm chrono cmath" + "algorithm atomic barrier chrono cmath" + "algorithm any array atomic barrier bit bitset cassert cctype cerrno cfenv cfloat charconv chrono cinttypes climits clocale cmath" +) + +for combo_str in "${COMBOS[@]}"; do + read -ra combo <<< "$combo_str" + PREV="" + for hdr in "${combo[@]}"; do + F="$WORKDIR/t_${hdr}.h" + P="$WORKDIR/t_${hdr}.pch" + echo "#include <$hdr>" > "$F" + if [ -z "$PREV" ]; then + $CLANG -x c++-header -std=c++20 -o "$P" "$F" 2>/dev/null + else + $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$P" "$F" 2>/dev/null + fi + PREV="$P" + done + + echo "#include " > "$WORKDIR/test.h" + ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test.pch" "$WORKDIR/test.h" 2>&1) + RC=$? + echo " [${combo_str}] -> deque: $([ $RC -eq 0 ] && echo PASS || echo FAIL) (${#combo[@]} links)" + + rm -f "$WORKDIR"/t_*.h "$WORKDIR"/t_*.pch "$WORKDIR"/test.* +done + +echo "" +echo "=== Stripping headers one by one from the 18-header chain ===" +FULL=(algorithm any array atomic barrier bit bitset cassert cctype cerrno cfenv cfloat charconv chrono cinttypes climits clocale cmath) + +for skip in "${FULL[@]}"; do + CHAIN=() + for h in "${FULL[@]}"; do + if [ "$h" != "$skip" ]; then + CHAIN+=("$h") + fi + done + + PREV="" + for hdr in "${CHAIN[@]}"; do + F="$WORKDIR/s_${hdr}.h" + P="$WORKDIR/s_${hdr}.pch" + echo "#include <$hdr>" > "$F" + if [ -z "$PREV" ]; then + $CLANG -x c++-header -std=c++20 -o "$P" "$F" 2>/dev/null + else + $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$P" "$F" 2>/dev/null + fi + PREV="$P" + done + + echo "#include " > "$WORKDIR/test.h" + ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test.pch" "$WORKDIR/test.h" 2>&1) + RC=$? + echo " skip <$skip> (${#CHAIN[@]} links): $([ $RC -eq 0 ] && echo PASS || echo FAIL)" + + rm -f "$WORKDIR"/s_*.h "$WORKDIR"/s_*.pch "$WORKDIR"/test.* +done diff --git a/benchmarks/pch_chain/minimal_repro.sh b/benchmarks/pch_chain/minimal_repro.sh new file mode 100644 index 000000000..9bcd4a452 --- /dev/null +++ b/benchmarks/pch_chain/minimal_repro.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Minimal reproduction: Chained PCH loses macro defined in c++config.h +# +# Bug: When building a chained PCH with .pch -> .pch, +# the __N macro (defined in libstdc++'s bits/c++config.h line 742) +# becomes invisible to subsequent compilation units, even though +# other macros from the same file (e.g. _GLIBCXX_VISIBILITY) survive. +# +# This is NOT a chain-depth issue (50+ trivial chains work fine). +# It is specifically triggered by in a chained PCH context. +# Monolithic PCH with both and works correctly. +# +set -euo pipefail + +CLANG="${CLANG:-clang++}" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +echo "clang: $($CLANG --version | head -1)" +echo "" + +cat > "$WORKDIR/test.cpp" << 'EOF' +// Test: use __N macro from libstdc++'s c++config.h +#include +const char* f() { return __N("test"); } +EOF + +# Case 1: Solo PCH → PASS +echo "Case 1: solo PCH" +echo "#include " > "$WORKDIR/solo.h" +$CLANG -x c++-header -std=c++20 -o "$WORKDIR/solo.pch" "$WORKDIR/solo.h" 2>/dev/null +if $CLANG -std=c++20 -include-pch "$WORKDIR/solo.pch" -fsyntax-only "$WORKDIR/test.cpp" 2>/dev/null; then + echo " Result: PASS (expected)" +else + echo " Result: FAIL (unexpected)" +fi + +# Case 2: Monolithic + PCH → PASS +echo "Case 2: monolithic + PCH" +printf '#include \n#include \n' > "$WORKDIR/mono.h" +$CLANG -x c++-header -std=c++20 -o "$WORKDIR/mono.pch" "$WORKDIR/mono.h" 2>/dev/null +if $CLANG -std=c++20 -include-pch "$WORKDIR/mono.pch" -fsyntax-only "$WORKDIR/test.cpp" 2>/dev/null; then + echo " Result: PASS (expected)" +else + echo " Result: FAIL (unexpected)" +fi + +# Case 3: Chained .pch → .pch → FAIL +echo "Case 3: chained -> PCH" +echo "#include " > "$WORKDIR/link1.h" +$CLANG -x c++-header -std=c++20 -o "$WORKDIR/link1.pch" "$WORKDIR/link1.h" 2>/dev/null +echo "#include " > "$WORKDIR/link2.h" +$CLANG -x c++-header -std=c++20 -include-pch "$WORKDIR/link1.pch" -o "$WORKDIR/link2.pch" "$WORKDIR/link2.h" 2>/dev/null +if $CLANG -std=c++20 -include-pch "$WORKDIR/link2.pch" -fsyntax-only "$WORKDIR/test.cpp" 2>&1; then + echo " Result: PASS" +else + echo " Result: FAIL ← BUG: __N macro lost in chained PCH" +fi + +echo "" +echo "Note: _GLIBCXX_VISIBILITY (line 72 of c++config.h) survives." +echo " __N (line 742 of c++config.h) is lost." +echo " Both are unconditional #define, no #undef anywhere." diff --git a/benchmarks/pch_chain/test_versions.sh b/benchmarks/pch_chain/test_versions.sh new file mode 100644 index 000000000..d62eaa03a --- /dev/null +++ b/benchmarks/pch_chain/test_versions.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Test the chained PCH macro loss bug across clang versions. +# Usage: Pass the clang++ binary path as argument. +# +set -euo pipefail + +CLANG="$1" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +echo "clang: $($CLANG --version | head -1)" + +cat > "$WORKDIR/test.cpp" << 'EOF' +#include +const char* f() { return __N("test"); } +EOF + +echo "#include " > "$WORKDIR/link1.h" +$CLANG -std=c++20 -Xclang -emit-pch -o "$WORKDIR/link1.pch" "$WORKDIR/link1.h" 2>/dev/null + +echo "#include " > "$WORKDIR/link2.h" +$CLANG -std=c++20 -include-pch "$WORKDIR/link1.pch" -Xclang -emit-pch -o "$WORKDIR/link2.pch" "$WORKDIR/link2.h" 2>/dev/null + +if $CLANG -std=c++20 -include-pch "$WORKDIR/link2.pch" -fsyntax-only "$WORKDIR/test.cpp" 2>/dev/null; then + echo "Result: PASS (bug is fixed)" +else + echo "Result: FAIL (bug still present)" +fi diff --git a/src/server/compiler.h b/src/server/compiler.h index 6ece99853..cec95bf3d 100644 --- a/src/server/compiler.h +++ b/src/server/compiler.h @@ -58,10 +58,8 @@ class Compiler { llvm::DenseMap& sessions); ~Compiler(); - void init_compile_graph(); - /// Fill compile arguments for a file (CDB lookup + header context fallback). /// @param session If non-null, used for header context resolution on open files. bool fill_compile_args(llvm::StringRef path, @@ -69,12 +67,10 @@ class Compiler { std::vector& arguments, Session* session = nullptr); - /// Compile an open file's AST if dirty. On success, updates session's /// file_index, pch_ref, ast_deps, and publishes diagnostics. et::task ensure_compiled(Session& session); - using RawResult = et::task; RawResult forward_query(worker::QueryKind kind, Session& session); diff --git a/src/server/indexer.cpp b/src/server/indexer.cpp index c31b5a230..c20d15e42 100644 --- a/src/server/indexer.cpp +++ b/src/server/indexer.cpp @@ -21,7 +21,6 @@ namespace clice { namespace lsp = eventide::ipc::lsp; - void Indexer::merge(const void* tu_index_data, std::size_t size) { auto tu_index = index::TUIndex::from(tu_index_data); if(tu_index.graph.paths.empty()) { @@ -182,7 +181,6 @@ bool Indexer::need_update(llvm::StringRef file_path) { return merged_it->second.index.need_update(path_mapping); } - bool Indexer::find_symbol_info(index::SymbolHash hash, std::string& name, SymbolKind& kind) const { for(auto& [_, session]: sessions) { if(!session.file_index) @@ -367,7 +365,6 @@ std::optional return lookup_symbol(uri, path, range.start, session); } - void Indexer::collect_grouped_relations( index::SymbolHash hash, RelationKind kind, @@ -433,7 +430,6 @@ void Indexer::collect_unique_targets(index::SymbolHash hash, } } - /// Resolve a symbol hash into a SymbolInfo with definition location. /// Returns nullopt if the symbol or its definition cannot be found. std::optional Indexer::resolve_symbol(index::SymbolHash hash) { @@ -577,7 +573,6 @@ std::vector Indexer::search_symbols(llvm::StringRef return results; } - protocol::SymbolKind Indexer::to_lsp_symbol_kind(SymbolKind kind) { switch(kind) { case SymbolKind::Namespace: return protocol::SymbolKind::Namespace; diff --git a/src/server/protocol.h b/src/server/protocol.h index 7f79ee25c..1ba4f6bd5 100644 --- a/src/server/protocol.h +++ b/src/server/protocol.h @@ -103,7 +103,6 @@ struct BuildResult { eventide::serde::RawValue result_json; ///< Completion/SignatureHelp result }; - struct DocumentUpdateParams { std::string path; int version; @@ -119,7 +118,6 @@ struct EvictedParams { } // namespace clice::worker - namespace clice::ext { struct ContextItem { @@ -177,7 +175,6 @@ struct RequestTraits { constexpr inline static std::string_view method = "clice/worker/build"; }; - template <> struct NotificationTraits { constexpr inline static std::string_view method = "clice/worker/documentUpdate"; diff --git a/src/server/session.h b/src/server/session.h index 1ba04cae2..f6d66a59f 100644 --- a/src/server/session.h +++ b/src/server/session.h @@ -26,11 +26,9 @@ namespace et = eventide; /// Sessions may READ from Workspace (e.g. to obtain PCH/PCM paths, module /// mappings, include graph) but all compilation results stay here. struct Session { - /// Path ID of this file in PathPool. Set on creation, never changes. std::uint32_t path_id = 0; - /// LSP document version, incremented by the client on each edit. int version = 0; @@ -54,7 +52,6 @@ struct Session { std::shared_ptr compiling; - /// Reference to the PCH entry in Workspace.pch_cache, if any. /// The PCH itself is owned by Workspace (shared, content-addressed); /// Session only stores enough to locate and validate it. @@ -70,7 +67,6 @@ struct Session { /// Used for two-layer staleness detection (mtime + content hash). std::optional ast_deps; - /// Compilation context for header files that lack their own CDB entry. /// Stores the host source file and synthesized preamble for this header. std::optional header_context; @@ -79,7 +75,6 @@ struct Session { /// When set, overrides automatic header context resolution. std::optional active_context; - /// Symbol index built from the latest compilation of this file's buffer. /// Used for queries (hover, goto, references) on this file. /// NOT merged into Workspace.project_index — that only gets disk-derived diff --git a/src/server/worker_pool.h b/src/server/worker_pool.h index c4b80257e..6e83860d1 100644 --- a/src/server/worker_pool.h +++ b/src/server/worker_pool.h @@ -83,7 +83,6 @@ class WorkerPool { bool spawn_worker(const std::string& self_path, bool stateful, std::uint64_t memory_limit); }; - template RequestResult WorkerPool::send_stateful(std::uint32_t path_id, const Params& params, diff --git a/src/server/workspace.cpp b/src/server/workspace.cpp index b233aa03c..c08de7fd1 100644 --- a/src/server/workspace.cpp +++ b/src/server/workspace.cpp @@ -35,7 +35,6 @@ const static index::Occurrence* lookup_occurrence(const std::vector> OpenFileIndex::find_occurrence(std::uint32_t offset) const { if(!mapper) @@ -53,7 +52,6 @@ std::optional> }; } - std::optional> MergedIndexShard::find_occurrence(std::uint32_t offset) const { auto* m = mapper(); @@ -74,7 +72,6 @@ std::optional> return result; } - llvm::SmallVector Workspace::on_file_saved(std::uint32_t path_id) { llvm::SmallVector dirtied; if(compile_graph) { @@ -95,7 +92,6 @@ void Workspace::on_file_closed(std::uint32_t path_id) { pch_cache.erase(path_id); } - std::uint64_t hash_file(llvm::StringRef path) { auto buf = llvm::MemoryBuffer::getFile(path); if(!buf) @@ -141,7 +137,6 @@ bool deps_changed(const PathPool& pool, const DepsSnapshot& snap) { return false; } - namespace { struct CacheDepEntry { @@ -174,7 +169,6 @@ struct CacheData { } // namespace - void Workspace::load_cache() { if(config.cache_dir.empty()) return; diff --git a/src/server/workspace.h b/src/server/workspace.h index 1af414039..4f840cdb8 100644 --- a/src/server/workspace.h +++ b/src/server/workspace.h @@ -29,7 +29,6 @@ namespace et = eventide; namespace protocol = et::ipc::protocol; namespace lsp = et::ipc::lsp; - /// Two-layer staleness snapshot for compilation artifacts (PCH, AST, etc.). /// /// Layer 1 (fast): compare each file's current mtime against build_at. @@ -51,7 +50,6 @@ struct HeaderFileContext { std::uint64_t preamble_hash; ///< Hash of preamble content for staleness. }; - /// In-memory index for an open file. Kept separate from MergedIndex because /// open files change frequently, are based on unsaved buffer content, and only /// need to track the main file (headers are covered by PCH/PCM indexing). @@ -135,7 +133,6 @@ struct MergedIndexShard { } }; - /// Cached PCH state. Content-addressed by preamble hash — shared across all /// files (open or on-disk) that have the same preamble content. struct PCHState { @@ -153,7 +150,6 @@ struct PCMState { DepsSnapshot deps; }; - /// All persistent, project-wide state derived from files on disk. /// /// Workspace is the single source of truth for: @@ -171,14 +167,11 @@ struct PCMState { /// Sessions (open files) may READ from Workspace but must not WRITE to it /// except through the didSave path. struct Workspace { - CliceConfig config; CompilationDatabase cdb; - PathPool path_pool; - /// Include relationships between files on disk (#include edges). /// Built once at startup from CDB scan; updated incrementally on didSave. DependencyGraph dep_graph; @@ -192,7 +185,6 @@ struct Workspace { /// declarations change. llvm::DenseMap path_to_module; - /// PCH cache, keyed by file path_id. Content-addressed by preamble hash, /// so different files with identical preambles share the same PCH. /// Both open files and background indexing may reference entries here. @@ -205,7 +197,6 @@ struct Workspace { /// Maps to the .pcm file on disk used as -fmodule-file argument. llvm::DenseMap pcm_paths; - /// Global symbol table across all indexed translation units. index::ProjectIndex project_index; @@ -214,7 +205,6 @@ struct Workspace { /// for position mapping. llvm::DenseMap merged_indices; - /// Called when a file is saved to disk. Cascades invalidation through /// compile_graph and clears affected PCM caches. /// Returns path_ids of all files dirtied by the cascade. @@ -224,7 +214,6 @@ struct Workspace { /// is a module unit so dependents can be re-evaluated on next compile. void on_file_closed(std::uint32_t path_id); - /// Load PCH/PCM cache from cache.json on disk. void load_cache(); /// Save PCH/PCM cache to cache.json on disk. From de8749a7afeb259fd062229a7bc491df77023d10 Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 11:26:22 +0800 Subject: [PATCH 07/18] chore: remove stray benchmark shell scripts These were debug scripts from the chained PCH investigation, not part of the workspace/session refactor. Co-Authored-By: Claude Opus 4.6 (1M context) --- benchmarks/pch_chain/README.md | 56 ------ benchmarks/pch_chain/bench_all_headers.sh | 201 -------------------- benchmarks/pch_chain/bench_pch_chain.sh | 218 ---------------------- benchmarks/pch_chain/find_breakpoint.sh | 40 ---- benchmarks/pch_chain/find_exact.sh | 85 --------- benchmarks/pch_chain/isolate_cmath.sh | 81 -------- benchmarks/pch_chain/minimal_repro.sh | 64 ------- benchmarks/pch_chain/test_versions.sh | 29 --- 8 files changed, 774 deletions(-) delete mode 100644 benchmarks/pch_chain/README.md delete mode 100755 benchmarks/pch_chain/bench_all_headers.sh delete mode 100755 benchmarks/pch_chain/bench_pch_chain.sh delete mode 100644 benchmarks/pch_chain/find_breakpoint.sh delete mode 100644 benchmarks/pch_chain/find_exact.sh delete mode 100644 benchmarks/pch_chain/isolate_cmath.sh delete mode 100644 benchmarks/pch_chain/minimal_repro.sh delete mode 100644 benchmarks/pch_chain/test_versions.sh diff --git a/benchmarks/pch_chain/README.md b/benchmarks/pch_chain/README.md deleted file mode 100644 index 6e2bf1b29..000000000 --- a/benchmarks/pch_chain/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# Chained PCH Benchmark Results - -clang 20.1.8, libstdc++ 14.2.0, Linux x86_64 - -## Summary - -| Metric | Monolithic PCH | Chained PCH | -| ------------------------------- | -------------------- | ----------------------- | -| Initial build (20 headers) | 869ms | 1488ms (1.7x slower) | -| Incremental rebuild (+1 header) | 917ms (full rebuild) | 226ms (append one link) | -| Incremental speedup | — | **4x** | - -## Bug: `` breaks macro visibility in chained PCH - -**Reproduction**: Any PCH chain that includes `` followed by `` fails: - -``` -algorithm.pch -> cmath.pch -> deque.pch ← FAIL: undeclared '__N' -``` - -But both of these work: - -``` -cmath.pch -> deque.pch ← PASS (2 links, no prior chain) -algorithm.pch -> deque.pch ← PASS (no cmath) -``` - -**Affected headers** (all fail when chained after ``): - -- ``, ``, ``, ``, ``, - ``, ``, ``, ``, `` - -All use `__N` macro from libstdc++'s `bits/c++config.h`. - -**Root cause**: The `__N` macro IS present in the chained PCH (verified via `-E`), -but becomes invisible to certain headers during compilation with the chained PCH. -This is likely a clang serialization bug where macro visibility scope is lost -across PCH chain boundaries when the defining header (`c++config.h`) was included -in a link prior to ``. - -**Workaround for clice**: When building chained PCH, merge `` and its -predecessor into a single link, or use a monolithic PCH when `` is present. - -## Chain depth limit - -Tested up to **50 trivial links** and **90 real-header links** — no depth limit -observed. The bug is specific to `` + libstdc++ `__N` macro interaction, -not chain depth. - -## Files - -- `bench_pch_chain.sh` — Basic benchmark (20 headers, monolithic vs chain, incremental) -- `bench_all_headers.sh` — Exhaustive test with all 100 C++ standard headers -- `find_breakpoint.sh` — Binary search for chain depth breakpoint -- `find_exact.sh` — Precise breakpoint isolation -- `isolate_cmath.sh` — Isolation of `` as the trigger diff --git a/benchmarks/pch_chain/bench_all_headers.sh b/benchmarks/pch_chain/bench_all_headers.sh deleted file mode 100755 index b67ba7b1f..000000000 --- a/benchmarks/pch_chain/bench_all_headers.sh +++ /dev/null @@ -1,201 +0,0 @@ -#!/bin/bash -# -# Exhaustive chained PCH test: try ALL C++ standard library headers. -# Build a chain where each link adds one header on top of the previous PCH. -# Report which header (if any) breaks the chain. -# -set -uo pipefail - -CLANG="${CLANG:-clang++}" -STD="${STD:--std=c++20}" -WORKDIR="$(mktemp -d)" -trap 'rm -rf "$WORKDIR"' EXIT - -echo "=== Exhaustive Chained PCH Test ===" -echo "clang: $($CLANG --version | head -1)" -echo "workdir: $WORKDIR" -echo "" - -# ALL C++20 standard library headers (sorted, no sub-directories) -HEADERS=( - algorithm - any - array - atomic - barrier - bit - bitset - cassert - cctype - cerrno - cfenv - cfloat - charconv - chrono - cinttypes - climits - clocale - cmath - codecvt - compare - complex - concepts - condition_variable - coroutine - csetjmp - csignal - cstdarg - cstddef - cstdint - cstdio - cstdlib - cstring - ctime - cuchar - cwchar - cwctype - deque - exception - execution - expected - filesystem - format - forward_list - fstream - functional - future - initializer_list - iomanip - ios - iosfwd - iostream - istream - iterator - latch - limits - list - locale - map - memory - memory_resource - mutex - new - numbers - numeric - optional - ostream - print - queue - random - ranges - ratio - regex - scoped_allocator - semaphore - set - shared_mutex - source_location - span - spanstream - sstream - stack - stdexcept - stop_token - streambuf - string - string_view - syncstream - system_error - thread - tuple - type_traits - typeindex - typeinfo - unordered_map - unordered_set - utility - valarray - variant - vector - version -) - -NUM=${#HEADERS[@]} -echo "Total headers: $NUM" -echo "" - -PREV_PCH="" -PASS=0 -FAIL=0 -FAIL_LIST="" -TOTAL_START=$(date +%s%N) - -for i in "${!HEADERS[@]}"; do - hdr="${HEADERS[$i]}" - idx=$((i + 1)) - HDR_FILE="$WORKDIR/link_${idx}.h" - PCH_FILE="$WORKDIR/link_${idx}.pch" - - echo "#include <$hdr>" > "$HDR_FILE" - - LINK_START=$(date +%s%N) - if [ -z "$PREV_PCH" ]; then - OUTPUT=$($CLANG -x c++-header $STD -o "$PCH_FILE" "$HDR_FILE" 2>&1) - RC=$? - else - OUTPUT=$($CLANG -x c++-header $STD -include-pch "$PREV_PCH" -o "$PCH_FILE" "$HDR_FILE" 2>&1) - RC=$? - fi - LINK_END=$(date +%s%N) - LINK_MS=$(( (LINK_END - LINK_START) / 1000000 )) - - if [ $RC -eq 0 ] && [ -f "$PCH_FILE" ]; then - SIZE=$(stat -c%s "$PCH_FILE") - SIZE_KB=$(( SIZE / 1024 )) - printf " [PASS] %3d/%d %-25s %5dms %5dKB\n" "$idx" "$NUM" "<$hdr>" "$LINK_MS" "$SIZE_KB" - PREV_PCH="$PCH_FILE" - PASS=$((PASS + 1)) - else - printf " [FAIL] %3d/%d %-25s %5dms\n" "$idx" "$NUM" "<$hdr>" "$LINK_MS" - if [ -n "$OUTPUT" ]; then - echo " Error: $(echo "$OUTPUT" | head -3)" - fi - FAIL=$((FAIL + 1)) - FAIL_LIST="$FAIL_LIST $hdr" - # Skip this header in the chain, continue with previous PCH - fi -done - -TOTAL_END=$(date +%s%N) -TOTAL_MS=$(( (TOTAL_END - TOTAL_START) / 1000000 )) - -echo "" -echo "=========================================" -echo "RESULTS" -echo "=========================================" -echo " Total headers: $NUM" -echo " Passed: $PASS" -echo " Failed: $FAIL" -echo " Total time: ${TOTAL_MS}ms" -if [ $FAIL -gt 0 ]; then - echo " Failed headers:$FAIL_LIST" -fi -echo "" - -# Final correctness check: compile a test file with the last successful PCH -if [ -n "$PREV_PCH" ] && [ -f "$PREV_PCH" ]; then - cat > "$WORKDIR/final_test.cpp" << 'SRCEOF' -#include -int main() { - static_assert(std::is_integral_v); - return 0; -} -SRCEOF - echo "Final PCH correctness check..." - if $CLANG $STD -include-pch "$PREV_PCH" -fsyntax-only "$WORKDIR/final_test.cpp" 2>&1; then - echo " Result: PASS" - else - echo " Result: FAIL" - fi - echo " Chain depth at end: $PASS links" -fi diff --git a/benchmarks/pch_chain/bench_pch_chain.sh b/benchmarks/pch_chain/bench_pch_chain.sh deleted file mode 100755 index e6f25165c..000000000 --- a/benchmarks/pch_chain/bench_pch_chain.sh +++ /dev/null @@ -1,218 +0,0 @@ -#!/bin/bash -# -# Benchmark: Monolithic PCH vs Chained PCH (one-include-per-link) -# -# Tests whether clang's chained PCH works correctly with 15+ chain links, -# and compares build time against a single monolithic PCH. -# -set -euo pipefail - -CLANG="${CLANG:-clang++}" -STD="${STD:--std=c++20}" -WORKDIR="$(mktemp -d)" -trap 'rm -rf "$WORKDIR"' EXIT - -echo "=== PCH Chain Benchmark ===" -echo "clang: $($CLANG --version | head -1)" -echo "workdir: $WORKDIR" -echo "" - -# --- Standard library headers to include (order matters for chaining) --- -HEADERS=( - "cstddef" - "cstdint" - "type_traits" - "utility" - "memory" - "string" - "string_view" - "vector" - "array" - "map" - "unordered_map" - "set" - "optional" - "variant" - "functional" - "algorithm" - "numeric" - "iostream" - "sstream" - "format" -) - -NUM_HEADERS=${#HEADERS[@]} -echo "Headers: $NUM_HEADERS" -echo "List: ${HEADERS[*]}" -echo "" - -# ============================================================ -# 1. Monolithic PCH: one file with all includes -# ============================================================ - -echo "=========================================" -echo "1. MONOLITHIC PCH" -echo "=========================================" - -# Generate the preamble header -MONO_HDR="$WORKDIR/mono_preamble.h" -for hdr in "${HEADERS[@]}"; do - echo "#include <$hdr>" >> "$MONO_HDR" -done - -echo "Building monolithic PCH ($NUM_HEADERS includes)..." -MONO_START=$(date +%s%N) -$CLANG -x c++-header $STD -o "$WORKDIR/mono.pch" "$MONO_HDR" 2>&1 -MONO_END=$(date +%s%N) -MONO_MS=$(( (MONO_END - MONO_START) / 1000000 )) -MONO_SIZE=$(stat -c%s "$WORKDIR/mono.pch") -echo " Time: ${MONO_MS}ms" -echo " Size: $(( MONO_SIZE / 1024 ))KB" - -# Test: compile a source file using the monolithic PCH -cat > "$WORKDIR/test_mono.cpp" << 'SRCEOF' -int main() { - std::vector v = {"hello", "world"}; - std::map m = {{1, "one"}, {2, "two"}}; - auto opt = std::make_optional(42); - std::cout << v[0] << " " << m[1] << " " << *opt << std::endl; - return 0; -} -SRCEOF - -echo " Compiling test with monolithic PCH..." -if $CLANG $STD -include-pch "$WORKDIR/mono.pch" -fsyntax-only "$WORKDIR/test_mono.cpp" 2>&1; then - echo " Result: PASS" -else - echo " Result: FAIL" -fi -echo "" - -# ============================================================ -# 2. Chained PCH: one PCH per include, each chaining the previous -# ============================================================ - -echo "=========================================" -echo "2. CHAINED PCH (one include per link)" -echo "=========================================" - -CHAIN_TOTAL_START=$(date +%s%N) - -PREV_PCH="" -for i in "${!HEADERS[@]}"; do - hdr="${HEADERS[$i]}" - idx=$((i + 1)) - CHAIN_HDR="$WORKDIR/chain_${idx}.h" - CHAIN_PCH="$WORKDIR/chain_${idx}.pch" - - echo "#include <$hdr>" > "$CHAIN_HDR" - - LINK_START=$(date +%s%N) - if [ -z "$PREV_PCH" ]; then - # First link: no previous PCH - if ! $CLANG -x c++-header $STD -o "$CHAIN_PCH" "$CHAIN_HDR" 2>&1; then - echo " FAIL at link $idx ($hdr) — build error" - break - fi - else - # Subsequent links: chain from previous PCH - if ! $CLANG -x c++-header $STD -include-pch "$PREV_PCH" -o "$CHAIN_PCH" "$CHAIN_HDR" 2>&1; then - echo " FAIL at link $idx ($hdr) — build error" - echo " (chained PCH may have a bug at this chain depth)" - break - fi - fi - LINK_END=$(date +%s%N) - LINK_MS=$(( (LINK_END - LINK_START) / 1000000 )) - LINK_SIZE=$(stat -c%s "$CHAIN_PCH" 2>/dev/null || echo 0) - - echo " Link $idx/$NUM_HEADERS: <$hdr> ${LINK_MS}ms $(( LINK_SIZE / 1024 ))KB" - PREV_PCH="$CHAIN_PCH" -done - -CHAIN_TOTAL_END=$(date +%s%N) -CHAIN_TOTAL_MS=$(( (CHAIN_TOTAL_END - CHAIN_TOTAL_START) / 1000000 )) - -echo "" -echo " Total chain build time: ${CHAIN_TOTAL_MS}ms" -if [ -n "$PREV_PCH" ] && [ -f "$PREV_PCH" ]; then - LAST_SIZE=$(stat -c%s "$PREV_PCH") - echo " Final PCH size: $(( LAST_SIZE / 1024 ))KB" -fi -echo "" - -# Test: compile the same source file using the final chained PCH -echo " Compiling test with chained PCH (final link)..." -if [ -n "$PREV_PCH" ] && [ -f "$PREV_PCH" ]; then - if $CLANG $STD -include-pch "$PREV_PCH" -fsyntax-only "$WORKDIR/test_mono.cpp" 2>&1; then - echo " Result: PASS" - else - echo " Result: FAIL" - fi -else - echo " Result: SKIP (chain build failed)" -fi -echo "" - -# ============================================================ -# 3. Incremental rebuild simulation: add one more include -# ============================================================ - -echo "=========================================" -echo "3. INCREMENTAL REBUILD (add )" -echo "=========================================" - -EXTRA_HDR="$WORKDIR/chain_extra.h" -EXTRA_PCH="$WORKDIR/chain_extra.pch" -echo "#include " > "$EXTRA_HDR" - -echo " Monolithic: rebuild entire PCH..." -echo "#include " >> "$MONO_HDR" -MONO_REBUILD_START=$(date +%s%N) -$CLANG -x c++-header $STD -o "$WORKDIR/mono.pch" "$MONO_HDR" 2>&1 -MONO_REBUILD_END=$(date +%s%N) -MONO_REBUILD_MS=$(( (MONO_REBUILD_END - MONO_REBUILD_START) / 1000000 )) -echo " Monolithic rebuild: ${MONO_REBUILD_MS}ms" - -if [ -n "$PREV_PCH" ] && [ -f "$PREV_PCH" ]; then - echo " Chained: append one link..." - INCR_START=$(date +%s%N) - if $CLANG -x c++-header $STD -include-pch "$PREV_PCH" -o "$EXTRA_PCH" "$EXTRA_HDR" 2>&1; then - INCR_END=$(date +%s%N) - INCR_MS=$(( (INCR_END - INCR_START) / 1000000 )) - echo " Chained incremental: ${INCR_MS}ms" - echo " Speedup: $(( MONO_REBUILD_MS / (INCR_MS > 0 ? INCR_MS : 1) ))x" - - # Verify the incremental PCH works - cat > "$WORKDIR/test_incr.cpp" << 'SRCEOF' -int main() { - auto now = std::chrono::system_clock::now(); - std::vector v = {"hello"}; - std::cout << v[0] << std::endl; - return 0; -} -SRCEOF - echo " Verifying incremental PCH..." - if $CLANG $STD -include-pch "$EXTRA_PCH" -fsyntax-only "$WORKDIR/test_incr.cpp" 2>&1; then - echo " Result: PASS" - else - echo " Result: FAIL" - fi - else - echo " Chained incremental: FAIL (build error)" - fi -else - echo " Skipped (chain build failed earlier)" -fi - -echo "" -echo "=========================================" -echo "SUMMARY" -echo "=========================================" -echo " Chain length: $NUM_HEADERS headers" -echo " Monolithic PCH build: ${MONO_MS}ms" -echo " Chained PCH total: ${CHAIN_TOTAL_MS}ms" -echo " Monolithic rebuild: ${MONO_REBUILD_MS}ms" -if [ -n "${INCR_MS:-}" ]; then -echo " Chained incremental: ${INCR_MS}ms" -fi diff --git a/benchmarks/pch_chain/find_breakpoint.sh b/benchmarks/pch_chain/find_breakpoint.sh deleted file mode 100644 index 7d903541b..000000000 --- a/benchmarks/pch_chain/find_breakpoint.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -uo pipefail - -CLANG="${CLANG:-clang++}" -WORKDIR="$(mktemp -d)" -trap 'rm -rf "$WORKDIR"' EXIT - -HEADERS=(algorithm any array atomic barrier bit bitset cassert cctype cerrno cfenv cfloat charconv chrono cinttypes climits clocale cmath codecvt compare complex concepts condition_variable coroutine csetjmp csignal cstdarg cstddef cstdint cstdio cstdlib cstring ctime cuchar cwchar cwctype) - -echo "=== Finding chain depth breakpoint for ===" - -for depth in 1 2 3 5 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36; do - if [ $depth -gt ${#HEADERS[@]} ]; then - break - fi - - PREV="" - for ((i=0; i" > "$WORKDIR/l_${i}.h" - if [ -z "$PREV" ]; then - $CLANG -x c++-header -std=c++20 -o "$WORKDIR/l_${i}.pch" "$WORKDIR/l_${i}.h" 2>/dev/null - else - $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/l_${i}.pch" "$WORKDIR/l_${i}.h" 2>/dev/null - fi - PREV="$WORKDIR/l_${i}.pch" - done - - echo "#include " > "$WORKDIR/test_deque.h" - ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test_deque.pch" "$WORKDIR/test_deque.h" 2>&1) - RC=$? - if [ $RC -eq 0 ]; then - echo " depth=$depth: PASS" - else - ERRMSG=$(echo "$ERR" | grep "error:" | head -1) - echo " depth=$depth: FAIL — $ERRMSG" - fi - - rm -f "$WORKDIR"/l_*.h "$WORKDIR"/l_*.pch "$WORKDIR"/test_deque.* -done diff --git a/benchmarks/pch_chain/find_exact.sh b/benchmarks/pch_chain/find_exact.sh deleted file mode 100644 index 75a53988c..000000000 --- a/benchmarks/pch_chain/find_exact.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -set -uo pipefail - -CLANG="${CLANG:-clang++}" -WORKDIR="$(mktemp -d)" -trap 'rm -rf "$WORKDIR"' EXIT - -HEADERS=(algorithm any array atomic barrier bit bitset cassert cctype cerrno cfenv cfloat charconv chrono cinttypes climits clocale cmath codecvt compare complex concepts condition_variable coroutine csetjmp csignal cstdarg cstddef cstdint cstdio cstdlib cstring ctime cuchar cwchar cwctype) - -echo "=== Exact breakpoint: depth 16-18 ===" - -for depth in 16 17 18; do - PREV="" - for ((i=0; i" > "$WORKDIR/l_${i}.h" - if [ -z "$PREV" ]; then - $CLANG -x c++-header -std=c++20 -o "$WORKDIR/l_${i}.pch" "$WORKDIR/l_${i}.h" 2>/dev/null - else - $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/l_${i}.pch" "$WORKDIR/l_${i}.h" 2>/dev/null - fi - PREV="$WORKDIR/l_${i}.pch" - done - - echo "#include " > "$WORKDIR/test.h" - ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test.pch" "$WORKDIR/test.h" 2>&1) - RC=$? - echo " depth=$depth (last hdr=${HEADERS[$((depth-1))]}): $([ $RC -eq 0 ] && echo PASS || echo FAIL)" - - rm -f "$WORKDIR"/l_*.h "$WORKDIR"/l_*.pch "$WORKDIR"/test.* -done - -echo "" -echo "=== Confirming: is it depth or is it the specific header? ===" -echo "depth=17 headers: ${HEADERS[*]:0:17}" -echo "" - -# Test: 17 links but with different ordering — is it just depth? -echo "Test: 17 trivial headers (cstddef repeated style)..." -PREV="" -for ((i=0; i<17; i++)); do - echo "#include " > "$WORKDIR/triv_${i}.h" - if [ -z "$PREV" ]; then - $CLANG -x c++-header -std=c++20 -o "$WORKDIR/triv_${i}.pch" "$WORKDIR/triv_${i}.h" 2>/dev/null - else - # Each link is a no-op (header guard prevents re-inclusion) - $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/triv_${i}.pch" "$WORKDIR/triv_${i}.h" 2>/dev/null - fi - PREV="$WORKDIR/triv_${i}.pch" -done -echo "#include " > "$WORKDIR/test.h" -ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test.pch" "$WORKDIR/test.h" 2>&1) -echo " 17 trivial links + deque: $([ $? -eq 0 ] && echo PASS || echo FAIL)" -rm -f "$WORKDIR"/triv_*.h "$WORKDIR"/triv_*.pch "$WORKDIR"/test.* - -# Test: 30 trivial headers -PREV="" -for ((i=0; i<30; i++)); do - echo "#include " > "$WORKDIR/triv_${i}.h" - if [ -z "$PREV" ]; then - $CLANG -x c++-header -std=c++20 -o "$WORKDIR/triv_${i}.pch" "$WORKDIR/triv_${i}.h" 2>/dev/null - else - $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/triv_${i}.pch" "$WORKDIR/triv_${i}.h" 2>/dev/null - fi - PREV="$WORKDIR/triv_${i}.pch" -done -echo "#include " > "$WORKDIR/test.h" -ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test.pch" "$WORKDIR/test.h" 2>&1) -echo " 30 trivial links + deque: $([ $? -eq 0 ] && echo PASS || echo FAIL)" -rm -f "$WORKDIR"/triv_*.h "$WORKDIR"/triv_*.pch "$WORKDIR"/test.* - -# Test: 50 trivial headers -PREV="" -for ((i=0; i<50; i++)); do - echo "#include " > "$WORKDIR/triv_${i}.h" - if [ -z "$PREV" ]; then - $CLANG -x c++-header -std=c++20 -o "$WORKDIR/triv_${i}.pch" "$WORKDIR/triv_${i}.h" 2>/dev/null - else - $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/triv_${i}.pch" "$WORKDIR/triv_${i}.h" 2>/dev/null - fi - PREV="$WORKDIR/triv_${i}.pch" -done -echo "#include " > "$WORKDIR/test.h" -ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test.pch" "$WORKDIR/test.h" 2>&1) -echo " 50 trivial links + deque: $([ $? -eq 0 ] && echo PASS || echo FAIL)" diff --git a/benchmarks/pch_chain/isolate_cmath.sh b/benchmarks/pch_chain/isolate_cmath.sh deleted file mode 100644 index fdaf36806..000000000 --- a/benchmarks/pch_chain/isolate_cmath.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash -set -uo pipefail - -CLANG="${CLANG:-clang++}" -WORKDIR="$(mktemp -d)" -trap 'rm -rf "$WORKDIR"' EXIT - -# We know: 17-link chain ending at clocale + deque = PASS -# 18-link chain ending at cmath + deque = FAIL -# But cmath -> deque alone = PASS -# So it's the combination of deep chain + cmath that causes __N loss - -# Test: just the "heavy" headers in chain, then cmath, then deque -echo "=== Test: minimal chain that breaks ===" - -# Which headers before cmath are "heavy" (pull in lots of stuff)? -# algorithm, atomic, barrier, charconv, chrono are the big ones -COMBOS=( - "algorithm cmath" - "chrono cmath" - "barrier cmath" - "algorithm chrono cmath" - "algorithm atomic barrier chrono cmath" - "algorithm any array atomic barrier bit bitset cassert cctype cerrno cfenv cfloat charconv chrono cinttypes climits clocale cmath" -) - -for combo_str in "${COMBOS[@]}"; do - read -ra combo <<< "$combo_str" - PREV="" - for hdr in "${combo[@]}"; do - F="$WORKDIR/t_${hdr}.h" - P="$WORKDIR/t_${hdr}.pch" - echo "#include <$hdr>" > "$F" - if [ -z "$PREV" ]; then - $CLANG -x c++-header -std=c++20 -o "$P" "$F" 2>/dev/null - else - $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$P" "$F" 2>/dev/null - fi - PREV="$P" - done - - echo "#include " > "$WORKDIR/test.h" - ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test.pch" "$WORKDIR/test.h" 2>&1) - RC=$? - echo " [${combo_str}] -> deque: $([ $RC -eq 0 ] && echo PASS || echo FAIL) (${#combo[@]} links)" - - rm -f "$WORKDIR"/t_*.h "$WORKDIR"/t_*.pch "$WORKDIR"/test.* -done - -echo "" -echo "=== Stripping headers one by one from the 18-header chain ===" -FULL=(algorithm any array atomic barrier bit bitset cassert cctype cerrno cfenv cfloat charconv chrono cinttypes climits clocale cmath) - -for skip in "${FULL[@]}"; do - CHAIN=() - for h in "${FULL[@]}"; do - if [ "$h" != "$skip" ]; then - CHAIN+=("$h") - fi - done - - PREV="" - for hdr in "${CHAIN[@]}"; do - F="$WORKDIR/s_${hdr}.h" - P="$WORKDIR/s_${hdr}.pch" - echo "#include <$hdr>" > "$F" - if [ -z "$PREV" ]; then - $CLANG -x c++-header -std=c++20 -o "$P" "$F" 2>/dev/null - else - $CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$P" "$F" 2>/dev/null - fi - PREV="$P" - done - - echo "#include " > "$WORKDIR/test.h" - ERR=$($CLANG -x c++-header -std=c++20 -include-pch "$PREV" -o "$WORKDIR/test.pch" "$WORKDIR/test.h" 2>&1) - RC=$? - echo " skip <$skip> (${#CHAIN[@]} links): $([ $RC -eq 0 ] && echo PASS || echo FAIL)" - - rm -f "$WORKDIR"/s_*.h "$WORKDIR"/s_*.pch "$WORKDIR"/test.* -done diff --git a/benchmarks/pch_chain/minimal_repro.sh b/benchmarks/pch_chain/minimal_repro.sh deleted file mode 100644 index 9bcd4a452..000000000 --- a/benchmarks/pch_chain/minimal_repro.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# -# Minimal reproduction: Chained PCH loses macro defined in c++config.h -# -# Bug: When building a chained PCH with .pch -> .pch, -# the __N macro (defined in libstdc++'s bits/c++config.h line 742) -# becomes invisible to subsequent compilation units, even though -# other macros from the same file (e.g. _GLIBCXX_VISIBILITY) survive. -# -# This is NOT a chain-depth issue (50+ trivial chains work fine). -# It is specifically triggered by in a chained PCH context. -# Monolithic PCH with both and works correctly. -# -set -euo pipefail - -CLANG="${CLANG:-clang++}" -WORKDIR="$(mktemp -d)" -trap 'rm -rf "$WORKDIR"' EXIT - -echo "clang: $($CLANG --version | head -1)" -echo "" - -cat > "$WORKDIR/test.cpp" << 'EOF' -// Test: use __N macro from libstdc++'s c++config.h -#include -const char* f() { return __N("test"); } -EOF - -# Case 1: Solo PCH → PASS -echo "Case 1: solo PCH" -echo "#include " > "$WORKDIR/solo.h" -$CLANG -x c++-header -std=c++20 -o "$WORKDIR/solo.pch" "$WORKDIR/solo.h" 2>/dev/null -if $CLANG -std=c++20 -include-pch "$WORKDIR/solo.pch" -fsyntax-only "$WORKDIR/test.cpp" 2>/dev/null; then - echo " Result: PASS (expected)" -else - echo " Result: FAIL (unexpected)" -fi - -# Case 2: Monolithic + PCH → PASS -echo "Case 2: monolithic + PCH" -printf '#include \n#include \n' > "$WORKDIR/mono.h" -$CLANG -x c++-header -std=c++20 -o "$WORKDIR/mono.pch" "$WORKDIR/mono.h" 2>/dev/null -if $CLANG -std=c++20 -include-pch "$WORKDIR/mono.pch" -fsyntax-only "$WORKDIR/test.cpp" 2>/dev/null; then - echo " Result: PASS (expected)" -else - echo " Result: FAIL (unexpected)" -fi - -# Case 3: Chained .pch → .pch → FAIL -echo "Case 3: chained -> PCH" -echo "#include " > "$WORKDIR/link1.h" -$CLANG -x c++-header -std=c++20 -o "$WORKDIR/link1.pch" "$WORKDIR/link1.h" 2>/dev/null -echo "#include " > "$WORKDIR/link2.h" -$CLANG -x c++-header -std=c++20 -include-pch "$WORKDIR/link1.pch" -o "$WORKDIR/link2.pch" "$WORKDIR/link2.h" 2>/dev/null -if $CLANG -std=c++20 -include-pch "$WORKDIR/link2.pch" -fsyntax-only "$WORKDIR/test.cpp" 2>&1; then - echo " Result: PASS" -else - echo " Result: FAIL ← BUG: __N macro lost in chained PCH" -fi - -echo "" -echo "Note: _GLIBCXX_VISIBILITY (line 72 of c++config.h) survives." -echo " __N (line 742 of c++config.h) is lost." -echo " Both are unconditional #define, no #undef anywhere." diff --git a/benchmarks/pch_chain/test_versions.sh b/benchmarks/pch_chain/test_versions.sh deleted file mode 100644 index d62eaa03a..000000000 --- a/benchmarks/pch_chain/test_versions.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# -# Test the chained PCH macro loss bug across clang versions. -# Usage: Pass the clang++ binary path as argument. -# -set -euo pipefail - -CLANG="$1" -WORKDIR="$(mktemp -d)" -trap 'rm -rf "$WORKDIR"' EXIT - -echo "clang: $($CLANG --version | head -1)" - -cat > "$WORKDIR/test.cpp" << 'EOF' -#include -const char* f() { return __N("test"); } -EOF - -echo "#include " > "$WORKDIR/link1.h" -$CLANG -std=c++20 -Xclang -emit-pch -o "$WORKDIR/link1.pch" "$WORKDIR/link1.h" 2>/dev/null - -echo "#include " > "$WORKDIR/link2.h" -$CLANG -std=c++20 -include-pch "$WORKDIR/link1.pch" -Xclang -emit-pch -o "$WORKDIR/link2.pch" "$WORKDIR/link2.h" 2>/dev/null - -if $CLANG -std=c++20 -include-pch "$WORKDIR/link2.pch" -fsyntax-only "$WORKDIR/test.cpp" 2>/dev/null; then - echo "Result: PASS (bug is fixed)" -else - echo "Result: FAIL (bug still present)" -fi From 31a6e70548dda20a70b34c53180ffc5a00bb3256 Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 11:45:08 +0800 Subject: [PATCH 08/18] fix: address review findings from workspace/session refactor - Fix potential dangling pointer in ensure_compiled: cache version before finish_compile() which may invalidate sess pointer - Fix misleading pch_cache comment (not content-addressed yet, add TODO) - Remove unused includes from compiler.cpp (chrono, type_traits, variant, raw_ostream, Chrono.h) - Use try_emplace for Session creation in didOpen - Keep is_file_open callback in Indexer (server path_id vs project path_id are different pools, callback bridges the gap) Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/compiler.cpp | 8 ++------ src/server/master_server.cpp | 3 ++- src/server/workspace.h | 6 +++--- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/server/compiler.cpp b/src/server/compiler.cpp index 8c183cda7..05618bf03 100644 --- a/src/server/compiler.cpp +++ b/src/server/compiler.cpp @@ -1,11 +1,8 @@ #include "server/compiler.h" -#include #include #include #include -#include -#include #include "command/search_config.h" #include "eventide/ipc/lsp/position.h" @@ -18,11 +15,9 @@ #include "syntax/include_resolver.h" #include "syntax/scan.h" -#include "llvm/Support/Chrono.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/Support/xxhash.h" namespace clice { @@ -812,11 +807,12 @@ et::task Compiler::ensure_compiled(Session& session) { sess->file_index = std::move(ofi); } + auto version = sess->version; finish_compile(); // Publish diagnostics AFTER marking compile as done, so that concurrent // forward_query() calls can proceed immediately. - self->publish_diagnostics(uri_str, sess->version, result.value().diagnostics); + self->publish_diagnostics(uri_str, version, result.value().diagnostics); if(self->on_indexing_needed) self->on_indexing_needed(); }(this, path_id, pending_compile)); diff --git a/src/server/master_server.cpp b/src/server/master_server.cpp index 04be093e3..e1647f21a 100644 --- a/src/server/master_server.cpp +++ b/src/server/master_server.cpp @@ -371,7 +371,8 @@ void MasterServer::register_handlers() { auto path = uri_to_path(params.text_document.uri); auto path_id = workspace.path_pool.intern(path); - auto& session = sessions[path_id]; + auto [it, _] = sessions.try_emplace(path_id); + auto& session = it->second; session.path_id = path_id; session.version = params.text_document.version; session.text = params.text_document.text; diff --git a/src/server/workspace.h b/src/server/workspace.h index 4f840cdb8..fc5aab675 100644 --- a/src/server/workspace.h +++ b/src/server/workspace.h @@ -185,9 +185,9 @@ struct Workspace { /// declarations change. llvm::DenseMap path_to_module; - /// PCH cache, keyed by file path_id. Content-addressed by preamble hash, - /// so different files with identical preambles share the same PCH. - /// Both open files and background indexing may reference entries here. + /// PCH cache, keyed by file path_id. + /// TODO: re-key by preamble content hash to enable cross-file sharing and + /// add LRU eviction. Compile flags should also be part of the key. llvm::DenseMap pch_cache; /// PCM cache, keyed by module source path_id. From 2f5b394677025cd567cd0937e2907db62e6d3950 Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 11:56:41 +0800 Subject: [PATCH 09/18] refactor: move background indexing scheduling from MasterServer to Indexer Indexer now owns the entire background indexing lifecycle: - index_queue, scheduling state, idle timer - enqueue() to add files, schedule() to trigger - run_background_indexing() coroutine MasterServer no longer has any indexing-related state or methods. Compiler.on_indexing_needed callback now calls indexer.schedule(). Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/indexer.cpp | 76 +++++++++++++++++++++++++++ src/server/indexer.h | 42 ++++++++++----- src/server/master_server.cpp | 99 ++++++------------------------------ src/server/master_server.h | 15 ++---- 4 files changed, 124 insertions(+), 108 deletions(-) diff --git a/src/server/indexer.cpp b/src/server/indexer.cpp index c20d15e42..137831258 100644 --- a/src/server/indexer.cpp +++ b/src/server/indexer.cpp @@ -8,7 +8,10 @@ #include "eventide/ipc/lsp/protocol.h" #include "eventide/ipc/lsp/uri.h" #include "index/tu_index.h" +#include "server/compiler.h" +#include "server/protocol.h" #include "server/session.h" +#include "server/worker_pool.h" #include "support/filesystem.h" #include "support/logging.h" @@ -617,4 +620,77 @@ protocol::TypeHierarchyItem Indexer::build_type_hierarchy_item(const SymbolInfo& return item; } +void Indexer::enqueue(std::uint32_t server_path_id) { + index_queue.push_back(server_path_id); +} + +void Indexer::schedule() { + if(!workspace.config.enable_indexing || indexing_active || indexing_scheduled) + return; + indexing_scheduled = true; + + if(!index_idle_timer) { + index_idle_timer = std::make_shared(et::timer::create(loop)); + } + index_idle_timer->start(std::chrono::milliseconds(workspace.config.idle_timeout_ms)); + loop.schedule(run_background_indexing()); +} + +et::task<> Indexer::run_background_indexing() { + if(index_idle_timer) { + co_await index_idle_timer->wait(); + } + indexing_scheduled = false; + + if(index_queue_pos >= index_queue.size()) { + LOG_DEBUG("Background indexing: queue exhausted"); + co_return; + } + + indexing_active = true; + std::size_t processed = 0; + + while(index_queue_pos < index_queue.size()) { + auto server_path_id = index_queue[index_queue_pos]; + index_queue_pos++; + + auto file_path = std::string(workspace.path_pool.resolve(server_path_id)); + + if(sessions.contains(server_path_id)) + continue; + + if(!need_update(file_path)) + continue; + + worker::BuildParams params; + params.kind = worker::BuildKind::Index; + params.file = file_path; + if(!compiler.fill_compile_args(file_path, params.directory, params.arguments, nullptr)) + continue; + + workspace.fill_pcm_deps(params.pcms); + + LOG_INFO("Background indexing: {}", file_path); + + auto result = co_await pool.send_stateless(params); + if(result.has_value() && result.value().success && !result.value().tu_index_data.empty()) { + LOG_INFO("Background indexing got TUIndex for {}: {} bytes", + file_path, + result.value().tu_index_data.size()); + merge(result.value().tu_index_data.data(), result.value().tu_index_data.size()); + ++processed; + } else if(result.has_value() && !result.value().success) { + LOG_WARN("Background index failed for {}: {}", file_path, result.value().error); + } else if(result.has_value() && result.value().tu_index_data.empty()) { + LOG_WARN("Background index returned empty TUIndex for {}", file_path); + } else { + LOG_WARN("Background index IPC error for {}: {}", file_path, result.error().message); + } + } + + indexing_active = false; + LOG_INFO("Background indexing complete: {} files processed", processed); + save(workspace.config.index_dir); +} + } // namespace clice diff --git a/src/server/indexer.h b/src/server/indexer.h index 947e2081d..71e0ceff2 100644 --- a/src/server/indexer.h +++ b/src/server/indexer.h @@ -2,10 +2,12 @@ #include #include +#include #include #include #include +#include "eventide/async/async.h" #include "eventide/ipc/lsp/position.h" #include "eventide/ipc/lsp/protocol.h" #include "semantic/relation_kind.h" @@ -23,6 +25,8 @@ namespace protocol = et::ipc::protocol; namespace lsp = et::ipc::lsp; struct Session; +class Compiler; +class WorkerPool; /// Information about a symbol at a given position. struct SymbolInfo { @@ -33,26 +37,28 @@ struct SymbolInfo { protocol::Range range; }; -/// Provides query methods for cross-file navigation over Workspace index data -/// and open Session file indices. +/// Provides query methods for cross-file navigation and manages background +/// indexing scheduling. /// /// Indexer does NOT own any index state. All persistent data lives in /// Workspace (disk-derived ProjectIndex + MergedIndex shards) and per-file /// data lives in Session (OpenFileIndex from unsaved buffers). -/// -/// Background indexing scheduling is driven by MasterServer; Indexer is the -/// pure query layer. class Indexer { public: - /// @param workspace Project-wide persistent state (ProjectIndex, MergedIndex shards). - /// @param sessions Map of open editing sessions, keyed by server-level path_id. - /// @param is_file_open Callback that returns true if a *project-level* path_id - /// has an active Session. Used to skip stale MergedIndex shards when - /// fresher OpenFileIndex data is available. - Indexer(Workspace& workspace, + Indexer(et::event_loop& loop, + Workspace& workspace, llvm::DenseMap& sessions, + WorkerPool& pool, + Compiler& compiler, std::function is_file_open = {}) : - workspace(workspace), sessions(sessions), is_file_open(std::move(is_file_open)) {} + loop(loop), workspace(workspace), sessions(sessions), pool(pool), compiler(compiler), + is_file_open(std::move(is_file_open)) {} + + /// Add a file to the background indexing queue. + void enqueue(std::uint32_t server_path_id); + + /// Schedule background indexing (respects idle timeout and dedup). + void schedule(); /// Merge a TUIndex result into Workspace's ProjectIndex and MergedIndex shards. void merge(const void* tu_index_data, std::size_t size); @@ -150,13 +156,25 @@ class Indexer { } private: + et::event_loop& loop; Workspace& workspace; llvm::DenseMap& sessions; + WorkerPool& pool; + Compiler& compiler; /// Callback that checks if a *project-level* path_id has an active /// Session. Set by the owner (e.g. MasterServer) to bridge the /// server-path-id-keyed sessions map to project-level path_ids. std::function is_file_open; + + /// Background indexing queue and scheduling state. + std::vector index_queue; + std::size_t index_queue_pos = 0; + bool indexing_active = false; + bool indexing_scheduled = false; + std::shared_ptr index_idle_timer; + + et::task<> run_background_indexing(); }; } // namespace clice diff --git a/src/server/master_server.cpp b/src/server/master_server.cpp index e1647f21a..2e81df698 100644 --- a/src/server/master_server.cpp +++ b/src/server/master_server.cpp @@ -37,9 +37,14 @@ static serde_raw to_raw(const T& value) { } MasterServer::MasterServer(et::event_loop& loop, et::ipc::JsonPeer& peer, std::string self_path) : - loop(loop), peer(peer), pool(loop), - indexer(workspace, sessions, [this](uint32_t id) { return sessions.contains(id); }), - compiler(loop, peer, workspace, pool, sessions), self_path(std::move(self_path)) {} + loop(loop), peer(peer), pool(loop), compiler(loop, peer, workspace, pool, sessions), + indexer(loop, + workspace, + sessions, + pool, + compiler, + [this](uint32_t id) { return sessions.contains(id); }), + self_path(std::move(self_path)) {} MasterServer::~MasterServer() = default; @@ -129,88 +134,14 @@ et::task<> MasterServer::load_workspace() { for(auto& entry: workspace.cdb.get_entries()) { auto file = workspace.cdb.resolve_path(entry.file); auto server_id = workspace.path_pool.intern(file); - index_queue.push_back(server_id); - } - if(!index_queue.empty()) { - LOG_INFO("Queued {} files for background indexing", index_queue.size()); - schedule_indexing(); + indexer.enqueue(server_id); } + indexer.schedule(); } compiler.init_compile_graph(); } -void MasterServer::schedule_indexing() { - if(!workspace.config.enable_indexing || indexing_active || indexing_scheduled) - return; - indexing_scheduled = true; - - if(!index_idle_timer) { - index_idle_timer = std::make_shared(et::timer::create(loop)); - } - index_idle_timer->start(std::chrono::milliseconds(workspace.config.idle_timeout_ms)); - loop.schedule(run_background_indexing()); -} - -et::task<> MasterServer::run_background_indexing() { - if(index_idle_timer) { - co_await index_idle_timer->wait(); - } - indexing_scheduled = false; - - if(index_queue_pos >= index_queue.size()) { - LOG_DEBUG("Background indexing: queue exhausted"); - co_return; - } - - indexing_active = true; - std::size_t processed = 0; - - while(index_queue_pos < index_queue.size()) { - auto server_path_id = index_queue[index_queue_pos]; - index_queue_pos++; - - auto file_path = std::string(workspace.path_pool.resolve(server_path_id)); - - /// Skip open files — their index comes from the stateful worker. - if(sessions.contains(server_path_id)) { - continue; - } - - if(!indexer.need_update(file_path)) - continue; - - worker::BuildParams params; - params.kind = worker::BuildKind::Index; - params.file = file_path; - if(!compiler.fill_compile_args(file_path, params.directory, params.arguments, nullptr)) - continue; - - workspace.fill_pcm_deps(params.pcms); - - LOG_INFO("Background indexing: {}", file_path); - - auto result = co_await pool.send_stateless(params); - if(result.has_value() && result.value().success && !result.value().tu_index_data.empty()) { - LOG_INFO("Background indexing got TUIndex for {}: {} bytes", - file_path, - result.value().tu_index_data.size()); - indexer.merge(result.value().tu_index_data.data(), result.value().tu_index_data.size()); - ++processed; - } else if(result.has_value() && !result.value().success) { - LOG_WARN("Background index failed for {}: {}", file_path, result.value().error); - } else if(result.has_value() && result.value().tu_index_data.empty()) { - LOG_WARN("Background index returned empty TUIndex for {}", file_path); - } else { - LOG_WARN("Background index IPC error for {}: {}", file_path, result.error().message); - } - } - - indexing_active = false; - LOG_INFO("Background indexing complete: {} files processed", processed); - indexer.save(workspace.config.index_dir); -} - void MasterServer::register_handlers() { using StringVec = std::vector; @@ -335,7 +266,7 @@ void MasterServer::register_handlers() { lifecycle = ServerLifecycle::Ready; compiler.on_indexing_needed = [this]() { - schedule_indexing(); + indexer.schedule(); }; loop.schedule(load_workspace()); @@ -446,8 +377,8 @@ void MasterServer::register_handlers() { sessions.erase(path_id); - index_queue.push_back(path_id); - schedule_indexing(); + indexer.enqueue(path_id); + indexer.schedule(); LOG_DEBUG("didClose: {}", path); }); @@ -464,7 +395,7 @@ void MasterServer::register_handlers() { if(auto sit = sessions.find(dirty_id); sit != sessions.end()) { sit->second.ast_dirty = true; } else { - index_queue.push_back(dirty_id); + indexer.enqueue(dirty_id); } } @@ -476,7 +407,7 @@ void MasterServer::register_handlers() { } } - schedule_indexing(); + indexer.schedule(); LOG_DEBUG("didSave: {}", path); }); diff --git a/src/server/master_server.h b/src/server/master_server.h index 0ee4d2f09..ba41added 100644 --- a/src/server/master_server.h +++ b/src/server/master_server.h @@ -55,27 +55,18 @@ class MasterServer { /// Worker process pool for offloading compilation and queries. WorkerPool pool; - /// Index query layer (reads from workspace and sessions). - Indexer indexer; - /// Compilation lifecycle manager (reads/writes workspace and sessions). Compiler compiler; + /// Index query and background scheduling (reads from workspace and sessions). + Indexer indexer; + ServerLifecycle lifecycle = ServerLifecycle::Uninitialized; std::string self_path; std::string workspace_root; std::string session_log_dir; - /// Background indexing state. - std::vector index_queue; - std::size_t index_queue_pos = 0; - bool indexing_active = false; - bool indexing_scheduled = false; - std::shared_ptr index_idle_timer; - et::task<> load_workspace(); - void schedule_indexing(); - et::task<> run_background_indexing(); using RawResult = et::task; }; From 70218253f3d083440e8ba0eebee43c6fbab9feae Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 12:12:36 +0800 Subject: [PATCH 10/18] refactor: extract completion logic from Compiler to syntax/completion Move preamble completion detection, module import completion, and include path completion into standalone functions in syntax/completion.h. Compiler now calls these functions and wraps results in LSP types. Add 16 unit tests for detect_completion_context and complete_module_import. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/compiler.cpp | 183 ++++++------------------- src/server/compiler.h | 12 +- src/syntax/completion.cpp | 121 ++++++++++++++++ src/syntax/completion.h | 57 ++++++++ tests/unit/syntax/completion_tests.cpp | 125 +++++++++++++++++ 5 files changed, 343 insertions(+), 155 deletions(-) create mode 100644 src/syntax/completion.cpp create mode 100644 src/syntax/completion.h create mode 100644 tests/unit/syntax/completion_tests.cpp diff --git a/src/server/compiler.cpp b/src/server/compiler.cpp index 05618bf03..fcc2c2e64 100644 --- a/src/server/compiler.cpp +++ b/src/server/compiler.cpp @@ -26,48 +26,6 @@ namespace lsp = eventide::ipc::lsp; using serde_raw = et::serde::RawValue; /// Detect whether the cursor is inside a preamble directive (include/import). -static PreambleCompletionContext detect_completion_context(const std::string& text, - uint32_t offset) { - auto line_start = text.rfind('\n', offset > 0 ? offset - 1 : 0); - line_start = (line_start == std::string::npos) ? 0 : line_start + 1; - - auto line_end = text.find('\n', offset); - if(line_end == std::string::npos) - line_end = text.size(); - - auto line = llvm::StringRef(text).slice(line_start, offset); - auto trimmed = line.ltrim(); - - if(trimmed.starts_with("#")) { - auto directive = trimmed.drop_front(1).ltrim(); - if(directive.consume_front("include")) { - directive = directive.ltrim(); - if(directive.consume_front("\"")) { - return {CompletionContext::IncludeQuoted, directive.str()}; - } - if(directive.consume_front("<")) { - return {CompletionContext::IncludeAngled, directive.str()}; - } - } - return {}; - } - - auto import_check = trimmed; - if(import_check.consume_front("export") && !import_check.empty() && - !std::isalnum(import_check[0])) { - import_check = import_check.ltrim(); - } - if(import_check.consume_front("import") && - (import_check.empty() || !std::isalnum(import_check[0]))) { - import_check = import_check.ltrim(); - auto rest_of_line = llvm::StringRef(text).slice(line_start, line_end); - if(!rest_of_line.contains(';')) { - return {CompletionContext::Import, import_check.str()}; - } - } - - return {}; -} Compiler::Compiler(et::event_loop& loop, et::ipc::JsonPeer& peer, @@ -914,114 +872,51 @@ Compiler::RawResult Compiler::handle_completion(const protocol::Position& positi auto pctx = detect_completion_context(session.text, *offset); if(pctx.kind == CompletionContext::IncludeQuoted || pctx.kind == CompletionContext::IncludeAngled) { - co_return complete_include(pctx, path); + std::string directory; + std::vector arguments; + if(!fill_compile_args(path, directory, arguments)) + co_return serde_raw{"[]"}; + + std::vector args_ptrs; + args_ptrs.reserve(arguments.size()); + for(auto& arg: arguments) + args_ptrs.push_back(arg.c_str()); + + auto search_config = extract_search_config(args_ptrs, directory); + DirListingCache dir_cache; + auto resolved = resolve_search_config(search_config, dir_cache); + bool angled = (pctx.kind == CompletionContext::IncludeAngled); + auto candidates = complete_include_path(resolved, pctx.prefix, angled, dir_cache); + + std::vector items; + items.reserve(candidates.size()); + for(auto& c: candidates) { + protocol::CompletionItem item; + item.label = c.is_directory ? c.name + "/" : c.name; + item.kind = protocol::CompletionItemKind::File; + items.push_back(std::move(item)); + } + auto json = et::serde::json::to_json(items); + co_return serde_raw{json ? std::move(*json) : "[]"}; } if(pctx.kind == CompletionContext::Import) { - co_return complete_import(pctx); - } - } - - co_return co_await forward_build(worker::BuildKind::Completion, position, session); -} - -et::serde::RawValue Compiler::complete_include(const PreambleCompletionContext& ctx, - llvm::StringRef path) { - std::string directory; - std::vector arguments; - if(!fill_compile_args(path, directory, arguments)) - return serde_raw{"[]"}; - - std::vector args_ptrs; - args_ptrs.reserve(arguments.size()); - for(auto& arg: arguments) { - args_ptrs.push_back(arg.c_str()); - } - - auto search_config = extract_search_config(args_ptrs, directory); - DirListingCache dir_cache; - auto resolved = resolve_search_config(search_config, dir_cache); - - unsigned start_idx = 0; - if(ctx.kind == CompletionContext::IncludeAngled) { - start_idx = resolved.angled_start_idx; - } - - llvm::StringRef prefix_ref(ctx.prefix); - llvm::StringRef dir_prefix; - llvm::StringRef file_prefix = prefix_ref; - auto slash_pos = prefix_ref.rfind('/'); - if(slash_pos != llvm::StringRef::npos) { - dir_prefix = prefix_ref.slice(0, slash_pos); - file_prefix = prefix_ref.slice(slash_pos + 1, llvm::StringRef::npos); - } - - std::vector items; - llvm::StringSet<> seen; - - for(unsigned i = start_idx; i < resolved.dirs.size(); ++i) { - auto& search_dir = resolved.dirs[i]; - - const llvm::StringSet<>* entries = nullptr; - if(!dir_prefix.empty()) { - llvm::SmallString<256> sub_path(search_dir.path); - llvm::sys::path::append(sub_path, dir_prefix); - entries = resolve_dir(sub_path, dir_cache); - } else { - entries = search_dir.entries; - } - - if(!entries) - continue; - - for(auto& entry: *entries) { - auto name = entry.getKey(); - if(!name.starts_with(file_prefix)) - continue; - if(!seen.insert(name).second) - continue; - - llvm::SmallString<256> full_path(search_dir.path); - if(!dir_prefix.empty()) { - llvm::sys::path::append(full_path, dir_prefix); + auto module_names = complete_module_import(workspace.path_to_module, pctx.prefix); + + std::vector items; + items.reserve(module_names.size()); + for(auto& name: module_names) { + protocol::CompletionItem item; + item.label = name; + item.kind = protocol::CompletionItemKind::Module; + item.insert_text = name + ";"; + items.push_back(std::move(item)); } - llvm::sys::path::append(full_path, name); - - bool is_dir = false; - llvm::sys::fs::is_directory(llvm::Twine(full_path), is_dir); - - protocol::CompletionItem item; - if(is_dir) { - item.label = (name + "/").str(); - } else { - item.label = name.str(); - } - item.kind = protocol::CompletionItemKind::File; - items.push_back(std::move(item)); + auto json = et::serde::json::to_json(items); + co_return serde_raw{json ? std::move(*json) : "[]"}; } } - auto json = et::serde::json::to_json(items); - return serde_raw{json ? std::move(*json) : "[]"}; -} - -et::serde::RawValue Compiler::complete_import(const PreambleCompletionContext& ctx) { - std::vector items; - llvm::StringRef prefix_ref(ctx.prefix); - - for(auto& [path_id, module_name]: workspace.path_to_module) { - llvm::StringRef name_ref(module_name); - if(!name_ref.starts_with(prefix_ref)) - continue; - - protocol::CompletionItem item; - item.label = module_name; - item.kind = protocol::CompletionItemKind::Module; - item.insert_text = module_name + ";"; - items.push_back(std::move(item)); - } - - auto json = et::serde::json::to_json(items); - return serde_raw{json ? std::move(*json) : "[]"}; + co_return co_await forward_build(worker::BuildKind::Completion, position, session); } } // namespace clice diff --git a/src/server/compiler.h b/src/server/compiler.h index cec95bf3d..1c21d8177 100644 --- a/src/server/compiler.h +++ b/src/server/compiler.h @@ -15,6 +15,7 @@ #include "server/session.h" #include "server/worker_pool.h" #include "server/workspace.h" +#include "syntax/completion.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" @@ -26,13 +27,6 @@ namespace clice { namespace et = eventide; namespace protocol = et::ipc::protocol; -enum class CompletionContext { None, IncludeQuoted, IncludeAngled, Import }; - -struct PreambleCompletionContext { - CompletionContext kind = CompletionContext::None; - std::string prefix; -}; - /// Convert a file:// URI to a local file path. std::string uri_to_path(const std::string& uri); @@ -114,10 +108,6 @@ class Compiler { std::vector& arguments, Session* session); - et::serde::RawValue complete_include(const PreambleCompletionContext& ctx, - llvm::StringRef path); - et::serde::RawValue complete_import(const PreambleCompletionContext& ctx); - private: et::event_loop& loop; et::ipc::JsonPeer& peer; diff --git a/src/syntax/completion.cpp b/src/syntax/completion.cpp new file mode 100644 index 000000000..5d68f69f7 --- /dev/null +++ b/src/syntax/completion.cpp @@ -0,0 +1,121 @@ +#include "syntax/completion.h" + +#include "syntax/include_resolver.h" + +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" + +namespace clice { + +PreambleCompletionContext detect_completion_context(llvm::StringRef text, std::uint32_t offset) { + auto line_start = text.rfind('\n', offset > 0 ? offset - 1 : 0); + line_start = (line_start == llvm::StringRef::npos) ? 0 : line_start + 1; + + auto line = text.slice(line_start, offset); + auto trimmed = line.ltrim(); + + if(trimmed.starts_with("#")) { + auto directive = trimmed.drop_front(1).ltrim(); + if(directive.consume_front("include")) { + directive = directive.ltrim(); + if(directive.consume_front("\"")) { + return {CompletionContext::IncludeQuoted, directive.str()}; + } + if(directive.consume_front("<")) { + return {CompletionContext::IncludeAngled, directive.str()}; + } + } + return {}; + } + + auto import_check = trimmed; + if(import_check.consume_front("export") && !import_check.empty() && + !std::isalnum(import_check[0])) { + import_check = import_check.ltrim(); + } + if(import_check.consume_front("import") && + (import_check.empty() || !std::isalnum(import_check[0]))) { + import_check = import_check.ltrim(); + + auto line_end = text.find('\n', offset); + if(line_end == llvm::StringRef::npos) + line_end = text.size(); + auto rest_of_line = text.slice(line_start, line_end); + if(!rest_of_line.contains(';')) { + return {CompletionContext::Import, import_check.str()}; + } + } + + return {}; +} + +std::vector + complete_module_import(const llvm::DenseMap& modules, + llvm::StringRef prefix) { + std::vector results; + for(auto& [path_id, module_name]: modules) { + if(llvm::StringRef(module_name).starts_with(prefix)) { + results.push_back(module_name); + } + } + return results; +} + +std::vector complete_include_path(const ResolvedSearchConfig& resolved, + llvm::StringRef prefix, + bool angled, + DirListingCache& dir_cache) { + llvm::StringRef dir_prefix; + llvm::StringRef file_prefix = prefix; + auto slash_pos = prefix.rfind('/'); + if(slash_pos != llvm::StringRef::npos) { + dir_prefix = prefix.slice(0, slash_pos); + file_prefix = prefix.slice(slash_pos + 1, llvm::StringRef::npos); + } + + unsigned start_idx = angled ? resolved.angled_start_idx : 0; + + std::vector results; + llvm::StringSet<> seen; + + for(unsigned i = start_idx; i < resolved.dirs.size(); ++i) { + auto& search_dir = resolved.dirs[i]; + + const llvm::StringSet<>* entries = nullptr; + if(!dir_prefix.empty()) { + llvm::SmallString<256> sub_path(search_dir.path); + llvm::sys::path::append(sub_path, dir_prefix); + entries = resolve_dir(sub_path, dir_cache); + } else { + entries = search_dir.entries; + } + + if(!entries) + continue; + + for(auto& entry: *entries) { + auto name = entry.getKey(); + if(!name.starts_with(file_prefix)) + continue; + if(!seen.insert(name).second) + continue; + + llvm::SmallString<256> full_path(search_dir.path); + if(!dir_prefix.empty()) { + llvm::sys::path::append(full_path, dir_prefix); + } + llvm::sys::path::append(full_path, name); + + bool is_dir = false; + llvm::sys::fs::is_directory(llvm::Twine(full_path), is_dir); + + results.push_back({name.str(), is_dir}); + } + } + + return results; +} + +} // namespace clice diff --git a/src/syntax/completion.h b/src/syntax/completion.h new file mode 100644 index 000000000..80193e6a8 --- /dev/null +++ b/src/syntax/completion.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringRef.h" + +namespace clice { + +struct ResolvedSearchConfig; +struct DirListingCache; + +/// What kind of preamble-level completion is being requested. +enum class CompletionContext : std::uint8_t { + None, + IncludeQuoted, + IncludeAngled, + Import, +}; + +/// Result of detecting the completion context from source text. +struct PreambleCompletionContext { + CompletionContext kind = CompletionContext::None; + std::string prefix; +}; + +/// Detect whether the cursor is inside a #include or import directive. +/// Pure text parsing — no compiler state needed. +PreambleCompletionContext detect_completion_context(llvm::StringRef text, std::uint32_t offset); + +/// Return module names matching a prefix, suitable for `import` completion. +/// @param modules Module name map (path_id → module name). +/// @param prefix Partially-typed module name to match against. +std::vector + complete_module_import(const llvm::DenseMap& modules, + llvm::StringRef prefix); + +/// Entry in the include path completion result. +struct IncludeCandidate { + std::string name; + bool is_directory = false; +}; + +/// Return file/directory names matching a prefix in the given search paths. +/// @param resolved Pre-resolved search directories with cached directory listings. +/// @param angled_start Index where angled (<>) search dirs begin. +/// @param prefix Partially-typed include path (e.g. "vec" or "sys/"). +/// @param angled True for <> includes, false for "" includes. +/// @param dir_cache Shared directory listing cache (for subdirectory lookups). +std::vector complete_include_path(const ResolvedSearchConfig& resolved, + llvm::StringRef prefix, + bool angled, + DirListingCache& dir_cache); + +} // namespace clice diff --git a/tests/unit/syntax/completion_tests.cpp b/tests/unit/syntax/completion_tests.cpp new file mode 100644 index 000000000..aed64c2ed --- /dev/null +++ b/tests/unit/syntax/completion_tests.cpp @@ -0,0 +1,125 @@ +#include "test/test.h" +#include "syntax/completion.h" + +#include "llvm/ADT/DenseMap.h" + +namespace clice::testing { +namespace { + +TEST_SUITE(DetectCompletionContext) { + +TEST_CASE(IncludeAngled) { + auto ctx = detect_completion_context("#include modules; + modules[1] = "std"; + modules[2] = "std.io"; + modules[3] = "std.net"; + modules[4] = "my_lib"; + + auto results = complete_module_import(modules, "std"); + EXPECT_EQ(results.size(), 3u); + for(auto& name: results) { + EXPECT_TRUE(name.starts_with("std")); + } +} + +TEST_CASE(EmptyPrefix) { + llvm::DenseMap modules; + modules[1] = "std"; + modules[2] = "my_lib"; + + auto results = complete_module_import(modules, ""); + EXPECT_EQ(results.size(), 2u); +} + +TEST_CASE(NoMatch) { + llvm::DenseMap modules; + modules[1] = "std"; + modules[2] = "my_lib"; + + auto results = complete_module_import(modules, "xyz"); + EXPECT_TRUE(results.empty()); +} + +TEST_CASE(EmptyModules) { + llvm::DenseMap modules; + auto results = complete_module_import(modules, "std"); + EXPECT_TRUE(results.empty()); +} + +}; // TEST_SUITE(CompleteModuleImport) + +} // namespace +} // namespace clice::testing From 60408dbbe8215ba6cf8c4af5c5e4a5897f1243e1 Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 12:18:57 +0800 Subject: [PATCH 11/18] docs: add design principle comments to Workspace and Session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Open files are never depended upon by other files. Dependencies always point to disk files." — the core invariant of the two-layer Workspace/Session architecture. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/session.h | 9 ++++----- src/server/workspace.h | 9 ++++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/server/session.h b/src/server/session.h index f6d66a59f..be58aea99 100644 --- a/src/server/session.h +++ b/src/server/session.h @@ -16,13 +16,12 @@ namespace et = eventide; /// An editing session for a single file opened in the editor. /// +/// Design principle: open files are never depended upon by other files. +/// Dependencies always point to disk files. The only path from Session +/// to Workspace is didSave, which tells Workspace to rescan the disk file. +/// /// Created on didOpen, destroyed on didClose. All fields are local to this /// file's translation unit and NEVER leak to Workspace or other Sessions. -/// -/// The only path from Session to Workspace is didSave, which triggers -/// Workspace to rescan the disk file — Session itself does not write to -/// Workspace. -/// /// Sessions may READ from Workspace (e.g. to obtain PCH/PCM paths, module /// mappings, include graph) but all compilation results stay here. struct Session { diff --git a/src/server/workspace.h b/src/server/workspace.h index fc5aab675..fd8b7e6da 100644 --- a/src/server/workspace.h +++ b/src/server/workspace.h @@ -152,6 +152,12 @@ struct PCMState { /// All persistent, project-wide state derived from files on disk. /// +/// Design principle: open files are never depended upon by other files. +/// Dependencies always point to disk files. This enforces a clean two-layer +/// architecture: +/// - Global layer (Workspace): tracks disk truth, shared by all files +/// - Per-file layer (Session): tracks buffer truth, isolated per TU +/// /// Workspace is the single source of truth for: /// - dependency relationships (include graph, module DAG) /// - compilation artifacts shared across files (PCH/PCM caches) @@ -163,9 +169,6 @@ struct PCMState { /// - Initialization (load_workspace at startup) /// - didSave (on_file_saved: rescan disk, cascade invalidation) /// - Background index (merge TUIndex results from stateless workers) -/// -/// Sessions (open files) may READ from Workspace but must not WRITE to it -/// except through the didSave path. struct Workspace { CliceConfig config; CompilationDatabase cdb; From 83babe14b4d8b18cc9c7666751f2655ee974009d Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 12:35:53 +0800 Subject: [PATCH 12/18] fix: address CodeRabbit review findings 1. (Critical) Cache session fields before co_await in forward_query/ forward_build to prevent dangling reference if didClose erases the session during suspension. 2. (Major) Include compile arguments and directory in PCH cache hash so switchContext produces a different PCH instead of reusing one built with different flags. 3. (Major) Rescan module declarations on didSave and update path_to_module, so module renames are reflected without restart. 4. (Major) Bridge project-level path_id to server-level path_id in the is_file_open callback, since ProjectIndex and Workspace use separate PathPool instances with potentially different IDs. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/compiler.cpp | 25 ++++++++++++++++++++++--- src/server/master_server.cpp | 8 +++++++- src/server/workspace.cpp | 13 +++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/src/server/compiler.cpp b/src/server/compiler.cpp index fcc2c2e64..9d7dc4cd8 100644 --- a/src/server/compiler.cpp +++ b/src/server/compiler.cpp @@ -445,7 +445,14 @@ et::task Compiler::ensure_pch(Session& session, co_return true; } - auto preamble_hash = llvm::xxh3_64bits(llvm::StringRef(text).substr(0, bound)); + // Hash both the preamble text AND compile arguments so that different + // compilation contexts (e.g. after switchContext) produce different PCHs. + auto preamble_text = llvm::StringRef(text).substr(0, bound); + auto preamble_hash = llvm::xxh3_64bits(preamble_text); + for(auto& arg: arguments) { + preamble_hash ^= llvm::xxh3_64bits(arg); + } + preamble_hash ^= llvm::xxh3_64bits(directory); // Deterministic content-addressed PCH path. auto pch_path = path::join(workspace.config.cache_dir, @@ -807,16 +814,21 @@ Compiler::RawResult Compiler::forward_query(worker::QueryKind kind, Session& session) { auto path_id = session.path_id; auto path = std::string(workspace.path_pool.resolve(path_id)); + // Cache text before co_await — session reference may dangle if didClose + // erases the entry from the sessions map during suspension. + auto text = session.text; if(!co_await ensure_compiled(session)) { co_return serde_raw{"null"}; } - if(session.ast_dirty) { + // Re-lookup session after co_await (DenseMap may have been modified). + auto sit = sessions.find(path_id); + if(sit == sessions.end() || sit->second.ast_dirty) { co_return serde_raw{"null"}; } - lsp::PositionMapper mapper(session.text, lsp::PositionEncoding::UTF16); + lsp::PositionMapper mapper(text, lsp::PositionEncoding::UTF16); auto offset = mapper.to_offset(position); if(!offset) co_return serde_raw{"null"}; @@ -838,6 +850,8 @@ Compiler::RawResult Compiler::forward_build(worker::BuildKind kind, worker::BuildParams wp; wp.kind = kind; wp.file = path; + // Cache session fields before co_await — session reference may dangle + // if didClose erases the entry from the sessions map during suspension. wp.version = session.version; wp.text = session.text; if(!fill_compile_args(path, wp.directory, wp.arguments, &session)) { @@ -848,6 +862,11 @@ Compiler::RawResult Compiler::forward_build(worker::BuildKind kind, co_return serde_raw{}; } + // After co_await, verify session still exists. + if(sessions.find(path_id) == sessions.end()) { + co_return serde_raw{}; + } + lsp::PositionMapper mapper(wp.text, lsp::PositionEncoding::UTF16); auto offset = mapper.to_offset(position); if(!offset) diff --git a/src/server/master_server.cpp b/src/server/master_server.cpp index 2e81df698..a672ac873 100644 --- a/src/server/master_server.cpp +++ b/src/server/master_server.cpp @@ -43,7 +43,13 @@ MasterServer::MasterServer(et::event_loop& loop, et::ipc::JsonPeer& peer, std::s sessions, pool, compiler, - [this](uint32_t id) { return sessions.contains(id); }), + [this](uint32_t proj_path_id) { + // Bridge project-level path_id to server-level path_id. + // The two PathPools may assign different IDs to the same path. + auto path = workspace.project_index.path_pool.path(proj_path_id); + auto server_id = workspace.path_pool.intern(path); + return sessions.contains(server_id); + }), self_path(std::move(self_path)) {} MasterServer::~MasterServer() = default; diff --git a/src/server/workspace.cpp b/src/server/workspace.cpp index c08de7fd1..66f18315c 100644 --- a/src/server/workspace.cpp +++ b/src/server/workspace.cpp @@ -8,6 +8,7 @@ #include "eventide/serde/json/json.h" #include "support/filesystem.h" #include "support/logging.h" +#include "syntax/scan.h" #include "llvm/Support/Chrono.h" #include "llvm/Support/FileSystem.h" @@ -74,6 +75,18 @@ std::optional> llvm::SmallVector Workspace::on_file_saved(std::uint32_t path_id) { llvm::SmallVector dirtied; + + // Re-scan the saved file for module declarations and update path_to_module. + auto file_path = path_pool.resolve(path_id); + if(auto buf = llvm::MemoryBuffer::getFile(file_path)) { + auto result = scan((*buf)->getBuffer()); + if(!result.module_name.empty()) { + path_to_module[path_id] = std::move(result.module_name); + } else { + path_to_module.erase(path_id); + } + } + if(compile_graph) { auto result = compile_graph->update(path_id); for(auto id: result) { From 4bcd914aebcf2c86f6ed16a53115f6a988999d13 Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 12:38:40 +0800 Subject: [PATCH 13/18] docs: update Compiler and Indexer class-level comments Clarify responsibilities and non-responsibilities for each component to reflect the final state of the refactoring. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/compiler.h | 26 +++++++++++++++----------- src/server/indexer.h | 15 ++++++++++++--- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/server/compiler.h b/src/server/compiler.h index 1c21d8177..83bdcc7fe 100644 --- a/src/server/compiler.h +++ b/src/server/compiler.h @@ -30,19 +30,23 @@ namespace protocol = et::ipc::protocol; /// Convert a file:// URI to a local file path. std::string uri_to_path(const std::string& uri); -/// Stateless compilation service. +/// Compilation service — drives worker processes to build ASTs, PCHs, and PCMs. /// -/// Compiler does not own any persistent state. It holds references to -/// Workspace (project-wide shared state) and WorkerPool (child processes), -/// both owned by MasterServer. Per-file state lives in Session objects -/// managed by the caller. +/// Compiler holds no persistent state of its own. All project-wide data +/// lives in Workspace; per-file data lives in Session. Compiler reads from +/// both and writes compilation results back to Session (file_index, pch_ref, +/// ast_deps, diagnostics). /// -/// Document lifecycle (open/change/close) is handled directly by -/// MasterServer on the sessions map. Compiler is only responsible for: -/// - Compilation (ensure_compiled, ensure_pch, ensure_deps) -/// - Feature request forwarding to workers -/// - Compile argument resolution -/// - Compile graph initialization +/// Responsibilities: +/// - AST compilation lifecycle (ensure_compiled → ensure_pch → ensure_deps) +/// - Feature request forwarding to stateful/stateless workers +/// - Compile argument resolution (CDB lookup + header context fallback) +/// - Compile graph initialization (module DAG setup) +/// +/// NOT responsible for: +/// - Document lifecycle (didOpen/didChange/didClose) — handled by MasterServer +/// - Index queries — handled by Indexer +/// - Background indexing scheduling — handled by Indexer class Compiler { public: Compiler(et::event_loop& loop, diff --git a/src/server/indexer.h b/src/server/indexer.h index 71e0ceff2..921bc73e4 100644 --- a/src/server/indexer.h +++ b/src/server/indexer.h @@ -37,12 +37,21 @@ struct SymbolInfo { protocol::Range range; }; -/// Provides query methods for cross-file navigation and manages background -/// indexing scheduling. +/// Index query layer and background indexing scheduler. /// -/// Indexer does NOT own any index state. All persistent data lives in +/// Indexer holds no index data of its own. All persistent data lives in /// Workspace (disk-derived ProjectIndex + MergedIndex shards) and per-file /// data lives in Session (OpenFileIndex from unsaved buffers). +/// +/// Responsibilities: +/// - Cross-file navigation queries (definition, references, hierarchy) +/// - Symbol search (workspace/symbol) +/// - Background indexing scheduling (enqueue → idle timer → worker dispatch) +/// - Merging TUIndex results into Workspace's ProjectIndex +/// +/// NOT responsible for: +/// - Compilation — handled by Compiler +/// - Document lifecycle — handled by MasterServer class Indexer { public: Indexer(et::event_loop& loop, From f0e92f2f8a81404817dad24f3ea80b7f8a950d44 Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 12:40:58 +0800 Subject: [PATCH 14/18] docs: add comments to forward_query, forward_build, handle_completion, clear_diagnostics Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/compiler.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/server/compiler.h b/src/server/compiler.h index 83bdcc7fe..3f2d75b41 100644 --- a/src/server/compiler.h +++ b/src/server/compiler.h @@ -71,17 +71,27 @@ class Compiler { using RawResult = et::task; + /// Forward a query (hover, semantic tokens, etc.) to the stateful worker + /// that holds this file's AST. Ensures compilation first. RawResult forward_query(worker::QueryKind kind, Session& session); + + /// Forward a position-sensitive query (goto definition, references, etc.) + /// to the stateful worker. Converts LSP position to byte offset. RawResult forward_query(worker::QueryKind kind, const protocol::Position& position, Session& session); + /// Forward a build request (signature help, etc.) to a stateless worker. + /// Sends the full buffer content and compile arguments. RawResult forward_build(worker::BuildKind kind, const protocol::Position& position, Session& session); + /// Handle completion requests. Detects preamble context (include/import) + /// and serves those locally; delegates code completion to a stateless worker. RawResult handle_completion(const protocol::Position& position, Session& session); + /// Send an empty diagnostics notification to clear stale markers in the editor. void clear_diagnostics(const std::string& uri); /// Callback invoked when indexing should be scheduled. From 5294b05524b90db8883b4456d982ab28a70c2e8a Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 12:55:52 +0800 Subject: [PATCH 15/18] refactor: merge forward_query overloads and pass InlayHints range Merge the two forward_query overloads into one with optional position and range parameters. Pass InlayHintParams.range through to the worker so it only computes hints for the visible region. Also remove LocalSourceRange user-defined constructors so it works as an aggregate with bincode serialization. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/compiler.cpp | 49 +++++++++++++++------------------- src/server/compiler.h | 15 +++++------ src/server/master_server.cpp | 13 +++++---- src/server/protocol.h | 4 ++- src/server/stateful_worker.cpp | 4 ++- src/syntax/token.h | 4 --- 6 files changed, 43 insertions(+), 46 deletions(-) diff --git a/src/server/compiler.cpp b/src/server/compiler.cpp index 9d7dc4cd8..1449e02e0 100644 --- a/src/server/compiler.cpp +++ b/src/server/compiler.cpp @@ -789,29 +789,10 @@ et::task Compiler::ensure_compiled(Session& session) { co_return !session.ast_dirty; } -Compiler::RawResult Compiler::forward_query(worker::QueryKind kind, Session& session) { - auto path_id = session.path_id; - auto path = std::string(workspace.path_pool.resolve(path_id)); - - if(!co_await ensure_compiled(session)) { - co_return serde_raw{"null"}; - } - - if(session.ast_dirty) { - co_return serde_raw{"null"}; - } - - worker::QueryParams wp{kind, path}; - auto result = co_await pool.send_stateful(path_id, wp); - if(!result.has_value()) { - co_return serde_raw{}; - } - co_return std::move(result.value()); -} - Compiler::RawResult Compiler::forward_query(worker::QueryKind kind, - const protocol::Position& position, - Session& session) { + Session& session, + std::optional position, + std::optional range) { auto path_id = session.path_id; auto path = std::string(workspace.path_pool.resolve(path_id)); // Cache text before co_await — session reference may dangle if didClose @@ -822,18 +803,32 @@ Compiler::RawResult Compiler::forward_query(worker::QueryKind kind, co_return serde_raw{"null"}; } - // Re-lookup session after co_await (DenseMap may have been modified). auto sit = sessions.find(path_id); if(sit == sessions.end() || sit->second.ast_dirty) { co_return serde_raw{"null"}; } + worker::QueryParams wp; + wp.kind = kind; + wp.path = path; + lsp::PositionMapper mapper(text, lsp::PositionEncoding::UTF16); - auto offset = mapper.to_offset(position); - if(!offset) - co_return serde_raw{"null"}; - worker::QueryParams wp{kind, path, *offset}; + if(position) { + auto offset = mapper.to_offset(*position); + if(!offset) + co_return serde_raw{"null"}; + wp.offset = *offset; + } + + if(range) { + auto start = mapper.to_offset(range->start); + auto end = mapper.to_offset(range->end); + if(start && end) { + wp.range = {*start, *end}; + } + } + auto result = co_await pool.send_stateful(path_id, wp); if(!result.has_value()) { co_return serde_raw{}; diff --git a/src/server/compiler.h b/src/server/compiler.h index 3f2d75b41..05ba039a7 100644 --- a/src/server/compiler.h +++ b/src/server/compiler.h @@ -71,15 +71,14 @@ class Compiler { using RawResult = et::task; - /// Forward a query (hover, semantic tokens, etc.) to the stateful worker - /// that holds this file's AST. Ensures compilation first. - RawResult forward_query(worker::QueryKind kind, Session& session); - - /// Forward a position-sensitive query (goto definition, references, etc.) - /// to the stateful worker. Converts LSP position to byte offset. + /// Forward a query to the stateful worker that holds this file's AST. + /// Ensures compilation first. For position-sensitive queries (hover, + /// goto-definition), pass a Position. For range-sensitive queries + /// (inlay hints), pass a Range. RawResult forward_query(worker::QueryKind kind, - const protocol::Position& position, - Session& session); + Session& session, + std::optional position = {}, + std::optional range = {}); /// Forward a build request (signature help, etc.) to a stateless worker. /// Sends the full buffer content and compile arguments. diff --git a/src/server/master_server.cpp b/src/server/master_server.cpp index a672ac873..6a9302e04 100644 --- a/src/server/master_server.cpp +++ b/src/server/master_server.cpp @@ -427,8 +427,8 @@ void MasterServer::register_handlers() { if(sit == sessions.end()) co_return serde_raw{"null"}; co_return co_await compiler.forward_query(worker::QueryKind::Hover, - params.text_document_position_params.position, - sit->second); + sit->second, + params.text_document_position_params.position); }); peer.on_request([this](RequestContext& ctx, @@ -448,7 +448,10 @@ void MasterServer::register_handlers() { auto sit = sessions.find(path_id); if(sit == sessions.end()) co_return serde_raw{"null"}; - co_return co_await compiler.forward_query(worker::QueryKind::InlayHints, sit->second); + co_return co_await compiler.forward_query(worker::QueryKind::InlayHints, + sit->second, + {}, + params.range); }); peer.on_request( @@ -544,8 +547,8 @@ void MasterServer::register_handlers() { if(sit == sessions.end()) co_return serde_raw{"null"}; co_return co_await compiler.forward_query(worker::QueryKind::GoToDefinition, - pos, - sit->second); + sit->second, + pos); }); peer.on_request([this, query_at](RequestContext& ctx, diff --git a/src/server/protocol.h b/src/server/protocol.h index 1ba4f6bd5..dba9dd1dc 100644 --- a/src/server/protocol.h +++ b/src/server/protocol.h @@ -10,6 +10,7 @@ #include "eventide/ipc/lsp/protocol.h" #include "eventide/ipc/protocol.h" #include "eventide/serde/serde/raw_value.h" +#include "syntax/token.h" namespace clice::worker { @@ -32,7 +33,8 @@ enum class QueryKind : uint8_t { struct QueryParams { QueryKind kind; std::string path; - uint32_t offset = 0; ///< Used by Hover and GoToDefinition. + uint32_t offset = 0; ///< Byte offset for position-sensitive queries (Hover, GoToDefinition). + LocalSourceRange range; ///< Byte range for range-sensitive queries (InlayHints). }; /// Parameters for stateful compilation (builds AST, publishes diagnostics). diff --git a/src/server/stateful_worker.cpp b/src/server/stateful_worker.cpp index 9e93afa44..8d31838d0 100644 --- a/src/server/stateful_worker.cpp +++ b/src/server/stateful_worker.cpp @@ -249,7 +249,9 @@ void StatefulWorker::register_handlers() { }); case K::InlayHints: co_return co_await with_ast(params.path, [&](DocumentEntry& doc) { - LocalSourceRange range{0, static_cast(doc.text.size())}; + auto range = params.range; + if(range.begin == static_cast(-1)) + range = LocalSourceRange{0, static_cast(doc.text.size())}; return to_raw(feature::inlay_hints(doc.unit, range)); }); case K::FoldingRange: diff --git a/src/syntax/token.h b/src/syntax/token.h index 31ade4a69..ad844fadc 100644 --- a/src/syntax/token.h +++ b/src/syntax/token.h @@ -44,10 +44,6 @@ struct LocalSourceRange { std::uint32_t begin = static_cast(-1); std::uint32_t end = static_cast(-1); - constexpr LocalSourceRange() = default; - - constexpr LocalSourceRange(std::uint32_t begin, std::uint32_t end) : begin(begin), end(end) {} - constexpr bool operator==(const LocalSourceRange& other) const = default; constexpr std::uint32_t length() const { From f93ff4b0e46891bef6cbda40677cfa6188e8bf8e Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 12:59:05 +0800 Subject: [PATCH 16/18] docs: update MasterServer class comment with design principle Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/master_server.h | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/server/master_server.h b/src/server/master_server.h index ba41added..2a2088014 100644 --- a/src/server/master_server.h +++ b/src/server/master_server.h @@ -28,12 +28,20 @@ enum class ServerLifecycle : std::uint8_t { Exited, }; -/// Top-level LSP server. +/// Top-level LSP server — the single orchestration point for the language +/// server process. /// -/// Owns all persistent state and registers LSP handlers. Document lifecycle -/// (open/change/close/save) is handled directly here on the sessions map. -/// Compilation and feature-request forwarding are delegated to Compiler; -/// index queries are delegated to Indexer. +/// Responsibilities: +/// - Owns the two-layer state model: Workspace (disk truth) and Sessions +/// (per-open-file volatile state). +/// - Manages Session lifecycle directly: didOpen creates, didChange mutates, +/// didSave syncs to Workspace, didClose destroys. +/// - Dispatches compilation and feature queries to Compiler. +/// - Dispatches index lookups and background indexing to Indexer. +/// +/// Design principle: +/// Open files are never depended upon by other files. Dependencies always +/// point to disk files. The only path from Session to Workspace is didSave. class MasterServer { public: MasterServer(et::event_loop& loop, et::ipc::JsonPeer& peer, std::string self_path); From 67bccc306c75f0bc01d6a9e1a51186f8584e3084 Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 13:25:48 +0800 Subject: [PATCH 17/18] fix: restore preamble-only PCH hash to fix shared preamble test The previous change hashed compile arguments (including the source file path) into the preamble hash, causing files with identical preambles but different paths to generate separate PCH files. Revert to hashing only the preamble text, matching the behavior before this refactor. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/compiler.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/server/compiler.cpp b/src/server/compiler.cpp index 1449e02e0..291680b80 100644 --- a/src/server/compiler.cpp +++ b/src/server/compiler.cpp @@ -445,14 +445,13 @@ et::task Compiler::ensure_pch(Session& session, co_return true; } - // Hash both the preamble text AND compile arguments so that different - // compilation contexts (e.g. after switchContext) produce different PCHs. + // Hash only the preamble text. This allows files with identical preambles + // to share the same PCH file (content-addressed). Compile flags are NOT + // included because preamble content is what determines the PCH output. + // TODO: consider hashing flags that affect preprocessing (e.g. -D, -I) + // for correctness when files have same preamble text but different flags. auto preamble_text = llvm::StringRef(text).substr(0, bound); auto preamble_hash = llvm::xxh3_64bits(preamble_text); - for(auto& arg: arguments) { - preamble_hash ^= llvm::xxh3_64bits(arg); - } - preamble_hash ^= llvm::xxh3_64bits(directory); // Deterministic content-addressed PCH path. auto pch_path = path::join(workspace.config.cache_dir, From 8b6f77b0413e62bfcf648eef1d034f5cfbf39fad Mon Sep 17 00:00:00 2001 From: ykiko Date: Wed, 8 Apr 2026 13:26:43 +0800 Subject: [PATCH 18/18] chore: upgrade PCH hash comment from TODO to FIXME Co-Authored-By: Claude Opus 4.6 (1M context) --- src/server/compiler.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server/compiler.cpp b/src/server/compiler.cpp index 291680b80..afdd72075 100644 --- a/src/server/compiler.cpp +++ b/src/server/compiler.cpp @@ -445,11 +445,11 @@ et::task Compiler::ensure_pch(Session& session, co_return true; } - // Hash only the preamble text. This allows files with identical preambles - // to share the same PCH file (content-addressed). Compile flags are NOT - // included because preamble content is what determines the PCH output. - // TODO: consider hashing flags that affect preprocessing (e.g. -D, -I) - // for correctness when files have same preamble text but different flags. + // FIXME: hash should also include compile flags that affect preprocessing + // (e.g. -D, -I, -isystem, -std) so that files with the same preamble text + // but different flags produce separate PCHs. Currently only the preamble + // text is hashed — the source file path must be excluded from the hash + // to allow sharing across files with identical preambles. auto preamble_text = llvm::StringRef(text).substr(0, bound); auto preamble_hash = llvm::xxh3_64bits(preamble_text);