diff --git a/src/clice.cc b/src/clice.cc index 7630a57f6..32d061a9e 100644 --- a/src/clice.cc +++ b/src/clice.cc @@ -30,7 +30,7 @@ int main(int argc, const char** argv) { driver::add_serve(clice, exit_code, self_path); driver::add_query(clice, exit_code); driver::add_worker(clice, exit_code); - driver::add_index(clice, exit_code); + driver::add_index(clice, exit_code, self_path); driver::add_doc(clice, exit_code); driver::add_lint(clice, exit_code); driver::add_format(clice, exit_code); diff --git a/src/command/command.h b/src/command/command.h index 52c1d95f9..867865a2f 100644 --- a/src/command/command.h +++ b/src/command/command.h @@ -238,6 +238,12 @@ class CompilationDatabase { /// All compilation entries (sorted by path_id). llvm::ArrayRef get_entries() const; + /// Map each file's path_id to the sorted canonical command hashes of its + /// entries (a file may own several entries with different flags). The + /// entry identity reload_and_diff() diffs on and the indexer persists to + /// catch command changes across sessions. + llvm::DenseMap> command_hash_snapshot() const; + /// A group of files that share the same compilation configuration. /// CDB internally deduplicates by (directory, canonical flags, user-content flags), /// so each group corresponds to one unique CompilationInfo — files within a group @@ -295,11 +301,6 @@ class CompilationDatabase { llvm::StringRef directory, llvm::StringRef command); - /// Map each file's path_id to the sorted canonical command hashes of its - /// entries (a file may own several entries with different flags). Used by - /// reload_and_diff() to compare the database before and after a reload. - llvm::DenseMap> command_hash_snapshot() const; - /// The memory pool which holds all elements of compilation database. /// Heap-allocated so its address is stable across moves. std::unique_ptr allocator = std::make_unique(); diff --git a/src/driver/driver.h b/src/driver/driver.h index 9d5151c57..0da622eb8 100644 --- a/src/driver/driver.h +++ b/src/driver/driver.h @@ -20,7 +20,7 @@ namespace clice::driver { void add_serve(kota::deco::cli::SubCommander& root, int& exit_code, const char* self_path); void add_query(kota::deco::cli::SubCommander& root, int& exit_code); void add_worker(kota::deco::cli::SubCommander& root, int& exit_code); -void add_index(kota::deco::cli::SubCommander& root, int& exit_code); +void add_index(kota::deco::cli::SubCommander& root, int& exit_code, const char* self_path); void add_doc(kota::deco::cli::SubCommander& root, int& exit_code); void add_lint(kota::deco::cli::SubCommander& root, int& exit_code); void add_format(kota::deco::cli::SubCommander& root, int& exit_code); diff --git a/src/driver/index.cc b/src/driver/index.cc index 00a8e0c7c..a06568658 100644 --- a/src/driver/index.cc +++ b/src/driver/index.cc @@ -1,4 +1,18 @@ +#include +#include +#include +#include +#include +#include + #include "driver/driver.h" +#include "index/serialization.h" +#include "server/transport/master_server.h" +#include "support/filesystem.h" +#include "support/timer.h" + +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/FileSystem.h" namespace clice::driver { @@ -7,26 +21,428 @@ namespace { struct IndexOptions { DecoFlag(names = {"-h", "--help"}, help = "Show help", required = false) help; + + DecoKV(style = deco::decl::KVStyle::JoinedOrSeparate, + help = "Workspace root directory (default: current directory)", + required = false) + workspace; + + DecoKV(style = deco::decl::KVStyle::JoinedOrSeparate, + help = "Number of indexing workers (default: from config)", + required = false) + workers; + + DecoFlag(names = {"--stats"}, + help = "Print statistics of the persisted index instead of indexing", + required = false) + stats; + + DecoKV(style = deco::decl::KVStyle::JoinedOrSeparate, + help = "How many of the largest file shards --stats lists", + required = false) + top; + + DecoKV(style = deco::decl::KVStyle::JoinedOrSeparate, + names = {"--log-level", "--log-level="}, + help = "Log level: trace, debug, info, warn, error, off", + required = false) + log_level; }; auto make_command() { return kota::deco::cli::command("clice index [OPTIONS]"); } +std::string format_size(std::uint64_t bytes) { + if(bytes >= 1024 * 1024) { + return std::format("{:.1f} MB", bytes / (1024.0 * 1024.0)); + } + if(bytes >= 1024) { + return std::format("{:.1f} KB", bytes / 1024.0); + } + return std::format("{} B", bytes); +} + +/// Poll until the background indexer has drained every round (requeue +/// rounds included) and persisted its results. +kota::task<> wait_until_indexed(const MasterServer& server) { + while(!server.indexer.is_idle()) { + co_await kota::sleep(200); + } +} + +/// The first signal asks for a graceful stop: in-flight files are +/// abandoned, finished ones are persisted, and a rerun resumes from +/// there. A second signal — of either watched kind, hence the shared +/// flag — exits immediately. +kota::task<> watch_signal(MasterServer& server, int signum, bool& stop_requested) { + auto watcher = kota::signal::create(); + if(!watcher || watcher->start(signum).has_error()) { + co_return; + } + while(true) { + co_await watcher->wait(); + if(stop_requested) { + std::_Exit(130); + } + stop_requested = true; + LOG_INFO("Interrupted; saving indexing progress"); + server.schedule_shutdown(); + } +} + +kota::task<> run_indexing_task(MasterServer& server, std::string root, int& exit_code) { + ScopedTimer timer; + server.initialize(root); + if(server.lifecycle != ServerLifecycle::Ready) { + exit_code = 1; + co_await server.shutdown_and_cleanup(); + co_return; + } + // The command's whole product is the persisted index: without storage + // (cache failed to open, another process holds the index writer lock, + // or an unreadable global blob disabled persistence) the run would + // only warm this process's memory and a rerun would start from + // nothing — fail instead of pretending. + if(!server.workspace.index_storage) { + LOG_ERROR("Cannot persist the index at {}; see the log for the cause and rerun", + std::string_view(server.workspace.config.project.cache_dir)); + exit_code = 1; + co_await server.shutdown_and_cleanup(); + co_return; + } + if(server.workspace.cdb.get_entries().empty()) { + LOG_ERROR("Nothing to index: no compile_commands.json found under {}", root); + exit_code = 1; + co_await server.shutdown_and_cleanup(); + co_return; + } + + bool stop_requested = false; + kota::task_group<> aux(server.loop); + aux.spawn(watch_signal(server, SIGINT, stop_requested)); + aux.spawn(watch_signal(server, SIGTERM, stop_requested)); + + co_await kota::with_token(wait_until_indexed(server), server.shutdown_token()); + co_await server.shutdown_and_cleanup(); + aux.cancel(); + co_await aux.join(); + + // Judged only after the watchers settle: a signal arriving while the + // final save/teardown ran must still report an interruption, not a + // normal completion with exit code 0. + if(stop_requested) { + std::println("Indexing interrupted; progress saved. Rerun `clice index` to resume."); + exit_code = 130; + co_return; + } + auto& workspace = server.workspace; + std::uint64_t total_bytes = 0; + for(auto& shard: llvm::make_second_range(workspace.shards)) { + total_bytes += shard.bytes().size(); + } + std::println("Indexed {} translation units in {:.1f}s: {} file shards ({}), {} symbols.", + workspace.project_index.manifests.size(), + timer.ms() / 1000.0, + workspace.shards.size(), + format_size(total_bytes), + workspace.project_index.symbols.size()); + if(auto failed = server.indexer.failed_files()) { + std::println("{} translation units failed to index (see the log); the index is partial.", + failed); + exit_code = 1; + } + // The shutdown save was the last retry for failed writes; whatever is + // still dirty never reached disk and a rerun cannot resume from it. + if(server.indexer.has_unsaved_state()) { + std::println("Part of the index could not be persisted (see the log)."); + exit_code = 1; + } +} + +int run_indexing(std::string root, std::uint32_t workers, const char* self_path) { + kota::event_loop loop; + MasterServer server(loop, self_path); + + // A one-shot batch run: rounds start immediately, disk polling stays + // off, and indexing happens even when the config keeps the background + // index disabled — running `clice index` is the request itself. + std::string worker_overlay; + if(workers != 0) { + worker_overlay = std::format( + R"(, "stateless_worker_count": {0}, "min_stateless_worker_count": {0}, "max_stateless_worker_count": {0})", + workers); + } + server.init_options_json = std::format( + R"({{"project": {{"idle_timeout_ms": 0, "enable_indexing": true{}}}, "tracker": {{"cdb_poll_seconds": 0, "workspace_poll_seconds": 0}}}})", + worker_overlay); + + int exit_code = 0; + loop.schedule(run_indexing_task(server, std::move(root), exit_code)); + loop.run(); + return exit_code; +} + +/// Sentinel of run_stats_once: the load raced a live writer's batch; +/// the caller retries instead of reporting over the mid-write state. +constexpr int stats_retry = -1; + +int run_stats_once(llvm::StringRef root, std::uint32_t top, bool allow_retry) { + auto config = Config::load_from_workspace(root); + // Read-only: the default cache directory exists as soon as the config + // resolves it, so only the versioned store inside it proves an index + // was ever built — and a live server (even one on an older layout) + // must not lose blobs to a stats reader. + auto store = + CacheStore::open(config.project.cache_dir, cache_format_version, /*read_only=*/true); + if(!store) { + if(store.error() == std::errc::no_such_file_or_directory) { + LOG_ERROR("No index cache at {}; run `clice index` first", + std::string_view(config.project.cache_dir)); + } else { + LOG_ERROR("Failed to open cache store at {}: {}", + std::string_view(config.project.cache_dir), + store.error().message()); + } + return 1; + } + + kota::event_loop loop; + Workspace workspace; + workspace.config = std::move(config); + workspace.store.emplace(std::move(*store)); + workspace.index_storage = index::make_fs_index_storage(*workspace.store); + // A namespace whose directory scan failed looks empty while its blobs + // exist — reporting "Index is empty" with exit code 0 would be a lie. + if(auto ec = workspace.store->scan_error()) { + LOG_ERROR("Failed to read the index cache at {}: {}", + std::string_view(workspace.config.project.cache_dir), + ec.message()); + return 1; + } + WorkerPool pool(loop); + ContextResolver contexts(workspace); + SessionStore sessions; + Indexer indexer(loop, workspace, pool, contexts, sessions); + if(!indexer.load(/*read_only=*/true)) { + LOG_ERROR("Index cache at {} is in an old or corrupt format; run `clice index` to rebuild", + std::string_view(workspace.config.project.cache_dir)); + return 1; + } + // load() detaches the storage when the global blob exists but cannot + // be read — a transient IO error, not an empty index. + if(workspace.index_storage == nullptr) { + LOG_ERROR("Failed to read the index cache at {}; the cache was left untouched", + std::string_view(workspace.config.project.cache_dir)); + return 1; + } + // A live writer's save publishes shards and manifests before the + // replacement global blob, so a read racing the batch can capture the + // old global next to newer blobs; the load drops those as stale and + // the verdicts below misread the mid-write state as damage. The writer + // may already have finished and unlocked by the time any post-load + // probe runs, so retry on the drops themselves; genuine damage merely + // spends the bounded retries before the final no-retry pass reports it. + if(allow_retry && indexer.pending_files() != 0) { + return stats_retry; + } + + auto& project = workspace.project_index; + if(project.manifests.empty() && workspace.shards.empty()) { + // Nothing else fills the queue here, so pending files can only be + // load()'s recovery drops: every TU's blobs were missing, stale, + // or corrupt — a damaged cache, not a legitimately empty one. + if(indexer.pending_files() != 0) { + LOG_ERROR( + "Index cache at {} has no servable data ({} translation units need " + "reindexing); run `clice index` to rebuild", + std::string_view(workspace.config.project.cache_dir), + indexer.pending_files()); + return 1; + } + std::println("Index is empty; run `clice index` to build it."); + return 0; + } + + struct ShardStat { + llvm::StringRef path; + std::uint64_t bytes; + std::size_t variants; + std::uint64_t occurrences = 0; + std::uint64_t relations = 0; + }; + + struct ColumnBytes { + std::uint64_t content = 0; + std::uint64_t variants = 0; + std::uint64_t symbols = 0; + std::uint64_t local_names = 0; + std::uint64_t occ_rows = 0; + std::uint64_t occ_masks = 0; + std::uint64_t rel_rows = 0; + std::uint64_t rel_masks = 0; + } columns; + + auto row_bytes = [](const index::RowRanges& rr) -> std::uint64_t { + return rr.packed.size() * 4 + rr.begins.size() * 4 + rr.lengths.size() + + rr.long_rows.size() * 4 + rr.long_ends.size() * 4; + }; + auto mask_bytes = [](const index::RowRanges& rr) -> std::uint64_t { + return rr.masks32.size() * 4 + rr.masks64.size() * 8 + rr.roaring_offsets.size() * 4 + + rr.roaring.size(); + }; + + std::vector files; + files.reserve(workspace.shards.size()); + std::uint64_t total_bytes = 0, total_occurrences = 0, total_relations = 0; + for(auto& [path_id, shard]: workspace.shards) { + ShardStat stat{.path = workspace.path_pool.resolve(path_id), + .bytes = shard.bytes().size(), + .variants = shard.variants().size()}; + shard.for_each_occurrence([&](const index::Occurrence&) { + stat.occurrences += 1; + return true; + }); + shard.for_each_relation([&](index::SymbolHash, const index::Relation&) { + stat.relations += 1; + return true; + }); + total_bytes += stat.bytes; + total_occurrences += stat.occurrences; + total_relations += stat.relations; + files.push_back(stat); + + index::ShardBlob blob; + if(index::deserialize_blob(shard.bytes(), blob)) { + columns.content += blob.content.size() + blob.line_lengths.size() + + blob.long_line_rows.size() * 4 + blob.long_line_lengths.size() * 4; + columns.variants += blob.variants.size() * 8; + columns.symbols += blob.sym_hashes.size() * 8 + blob.sym_rel_offsets.size() * 4; + columns.local_names += + blob.local_syms.size() * 4 + blob.local_kinds.size() + blob.local_scopes.size(); + for(auto& name: blob.local_names) { + columns.local_names += name.size(); + } + columns.occ_rows += row_bytes(blob.occs) + blob.occ_syms8.size() + + blob.occ_syms16.size() * 2 + blob.occ_syms32.size() * 4; + columns.occ_masks += mask_bytes(blob.occs); + columns.rel_rows += row_bytes(blob.rels) + blob.rel_kinds.size() + + blob.rel_sym_rows.size() * 4 + blob.rel_sym8.size() + + blob.rel_sym16.size() * 2 + blob.rel_sym32.size() * 4 + + blob.rel_def_rows.size() * 4 + blob.rel_def_begins.size() * 4 + + blob.rel_def_ends.size() * 4; + columns.rel_masks += mask_bytes(blob.rels); + } + } + std::ranges::sort(files, std::ranges::greater{}, &ShardStat::bytes); + + std::println("Index cache: {}", std::string_view(workspace.store->base_dir())); + std::println("Translation units: {}", project.manifests.size()); + std::println("File shards: {} ({}), {} occurrences, {} relations", + files.size(), + format_size(total_bytes), + total_occurrences, + total_relations); + std::println("Global symbols: {}, file versions: {}", + project.symbols.size(), + project.file_versions.size()); + + auto payload = columns.content + columns.variants + columns.symbols + columns.local_names + + columns.occ_rows + columns.occ_masks + columns.rel_rows + columns.rel_masks; + auto share = [&](std::uint64_t bytes) { + return payload != 0 ? 100.0 * static_cast(bytes) / static_cast(payload) + : 0.0; + }; + std::println(); + std::println("Payload by column ({}; the rest of the file size is format framing):", + format_size(payload)); + std::println(" occurrence rows {:>10} {:>5.1f}%", + format_size(columns.occ_rows), + share(columns.occ_rows)); + std::println(" occurrence masks {:>10} {:>5.1f}%", + format_size(columns.occ_masks), + share(columns.occ_masks)); + std::println(" relation rows {:>10} {:>5.1f}%", + format_size(columns.rel_rows), + share(columns.rel_rows)); + std::println(" relation masks {:>10} {:>5.1f}%", + format_size(columns.rel_masks), + share(columns.rel_masks)); + std::println(" symbol tables {:>10} {:>5.1f}%", + format_size(columns.symbols), + share(columns.symbols)); + std::println(" local symbol names {:>10} {:>5.1f}%", + format_size(columns.local_names), + share(columns.local_names)); + std::println(" content + line maps {:>10} {:>5.1f}%", + format_size(columns.content), + share(columns.content)); + std::println(" variant tables {:>10} {:>5.1f}%", + format_size(columns.variants), + share(columns.variants)); + if(indexer.pending_files() != 0) { + std::println( + "Translation units pending reindex (stale or partially written): {}; " + "run `clice index` to repair", + indexer.pending_files()); + } + std::println(); + std::println("Top {} file shards by size:", std::min(top, files.size())); + for(auto& stat: files | std::views::take(top)) { + std::println(" {:>10} {:>4} variants {:>9} occs {:>9} rels {}", + format_size(stat.bytes), + stat.variants, + stat.occurrences, + stat.relations, + std::string_view(stat.path)); + } + // Partial damage is still damage: automation must not read exit 0 as + // "the cache is healthy" just because some TUs remained servable. + return indexer.pending_files() == 0 ? 0 : 1; +} + +int run_stats(llvm::StringRef root, std::uint32_t top) { + constexpr std::uint32_t stats_attempts = 5; + for(std::uint32_t attempt = 1; attempt < stats_attempts; attempt += 1) { + int rc = run_stats_once(root, top, /*allow_retry=*/true); + if(rc != stats_retry) { + return rc; + } + LOG_DEBUG("Index cache is mid-save; retrying the stats read"); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + return run_stats_once(root, top, /*allow_retry=*/false); +} + } // namespace -void add_index(kota::deco::cli::SubCommander& root, int& exit_code) { +void add_index(kota::deco::cli::SubCommander& root, int& exit_code, const char* self_path) { auto cmd = make_command(); - cmd.matchAll([&exit_code](IndexOptions opts) { + cmd.matchAll([&exit_code, self_path](IndexOptions opts) { if(opts.help) { auto help = make_command(); print_usage(help); exit_code = 0; return; } - std::println( - stderr, - "clice index is not implemented yet: it will build the workspace symbol index ahead of time."); + if(!apply_log_level(opts.log_level.value_or("info"))) + return; + logging::stderr_logger("index", logging::options); + + llvm::SmallString<256> workspace(opts.workspace.value_or("")); + if(workspace.empty()) { + llvm::sys::fs::current_path(workspace); + } else { + llvm::sys::fs::make_absolute(workspace); + } + std::string ws(workspace.str()); + path::canonicalize(ws); + + if(opts.stats) { + exit_code = run_stats(ws, opts.top.value_or(20)); + return; + } + exit_code = run_indexing(std::move(ws), opts.workers.value_or(0), self_path); }) .on_error([](auto err) { LOG_ERROR("{}", err.message); }); diff --git a/src/index/serialization.h b/src/index/serialization.h index c400db162..f8781f6d7 100644 --- a/src/index/serialization.h +++ b/src/index/serialization.h @@ -100,8 +100,10 @@ namespace clice::index { /// On-disk index blob schema version. Every persisted blob carries it as a /// regular field and every loader discards blobs with a different value — /// including version-less blobs from older builds, which read back as 0. -/// Bump it whenever a persisted type's reflected layout changes. -constexpr inline std::uint32_t index_format_version = 6; +/// Bump it whenever a persisted type's reflected layout changes, or when +/// symbol-hash semantics change (stored rows keyed by old hashes would +/// silently stop matching newly computed ones). +constexpr inline std::uint32_t index_format_version = 7; /// Serialize a reflected index blob to `os` as a verified-readable /// flatbuffer. Encoding only fails on structural impossibilities (e.g. more diff --git a/src/index/storage.cpp b/src/index/storage.cpp index fd52293c4..4264414ee 100644 --- a/src/index/storage.cpp +++ b/src/index/storage.cpp @@ -1,27 +1,35 @@ #include "index/storage.h" #include "support/cache_store.h" +#include "support/filesystem.h" #include "support/logging.h" +#include "llvm/Support/Process.h" #include "llvm/Support/raw_ostream.h" namespace clice::index { namespace { +constexpr llvm::StringLiteral index_lock_name = "index.lock"; + llvm::StringRef namespace_of(IndexBlobKind kind) { switch(kind) { case IndexBlobKind::Shard: return "index"; case IndexBlobKind::Manifest: return "index-manifest"; case IndexBlobKind::Global: return "index-global"; + case IndexBlobKind::Cdb: return "index-cdb"; } std::unreachable(); } class FsIndexStorage final : public IndexStorage { public: - explicit FsIndexStorage(CacheStore& store) : store(store) { - for(auto kind: {IndexBlobKind::Shard, IndexBlobKind::Manifest, IndexBlobKind::Global}) { + FsIndexStorage(CacheStore& store, int lock_fd) : store(store), lock_fd(lock_fd) { + for(auto kind: {IndexBlobKind::Shard, + IndexBlobKind::Manifest, + IndexBlobKind::Global, + IndexBlobKind::Cdb}) { store.register_namespace({ .name = std::string(namespace_of(kind)), .extension = ".idx", @@ -30,6 +38,13 @@ class FsIndexStorage final : public IndexStorage { } } + ~FsIndexStorage() override { + if(lock_fd != -1) { + llvm::sys::fs::unlockFile(lock_fd); + llvm::sys::Process::SafelyCloseFileDescriptor(lock_fd); + } + } + std::unique_ptr read(IndexBlobKind kind, llvm::StringRef key) override { auto path = store.lookup(namespace_of(kind), key); if(!path) { @@ -47,7 +62,18 @@ class FsIndexStorage final : public IndexStorage { } llvm::SmallVector write(llvm::ArrayRef batch) override { - llvm::SmallVector failed; + // Batch order encodes dependency (shards → manifests → global → + // CDB snapshot), so the first failure fails the rest of the batch: + // continuing would publish an entry whose prerequisites never + // landed — e.g. a CDB snapshot vouching for a global that failed — + // and load paths only tolerate a committed prefix, the crash shape. + auto fail_from = [&](std::size_t i) { + llvm::SmallVector failed; + for(; i < batch.size(); i += 1) { + failed.push_back(i); + } + return failed; + }; for(std::size_t i = 0; i < batch.size(); i += 1) { auto& blob = batch[i]; auto ns = namespace_of(blob.kind); @@ -56,8 +82,7 @@ class FsIndexStorage final : public IndexStorage { llvm::raw_fd_ostream os(pending.tmp_path, ec); if(ec) { LOG_WARN("Failed to write index blob {}/{}: {}", ns, blob.key, ec.message()); - failed.push_back(i); - continue; + return fail_from(i); } os.write(blob.bytes.data(), blob.bytes.size()); os.close(); @@ -69,19 +94,17 @@ class FsIndexStorage final : public IndexStorage { blob.key, os.error().message()); os.clear_error(); - failed.push_back(i); - continue; + return fail_from(i); } if(auto committed = store.commit(std::move(pending)); !committed) { LOG_WARN("Failed to commit index blob {}/{}: {}", ns, blob.key, committed.error().message()); - failed.push_back(i); - continue; + return fail_from(i); } } - return failed; + return {}; } void remove(IndexBlobKind kind, llvm::StringRef key) override { @@ -94,12 +117,39 @@ class FsIndexStorage final : public IndexStorage { private: CacheStore& store; + int lock_fd; }; } // namespace std::unique_ptr make_fs_index_storage(CacheStore& store) { - return std::make_unique(store); + int lock_fd = -1; + if(!store.read_only()) { + // Atomic per-blob replacement cannot serialize the mutable + // global/manifest lineage: two writers (an LSP server plus a batch + // `clice index`) derive the same next generation from the same + // loaded state, so a manifest written by one passes the other's + // generation pin with FileVersion ids allocated against a different + // table, loading rows under the wrong files. An OS advisory lock + // dies with its process, so a crash leaves nothing stale behind. + auto lock_path = path::join(store.base_dir(), index_lock_name); + if(auto ec = llvm::sys::fs::openFileForReadWrite(lock_path, + lock_fd, + llvm::sys::fs::CD_OpenAlways, + llvm::sys::fs::OF_None)) { + LOG_WARN("Failed to open the index writer lock {}: {}", lock_path, ec.message()); + return nullptr; + } + if(llvm::sys::fs::tryLockFile(lock_fd)) { + LOG_WARN( + "Another clice process is writing the index cache at {}; " + "index persistence is disabled for this process", + store.base_dir()); + llvm::sys::Process::SafelyCloseFileDescriptor(lock_fd); + return nullptr; + } + } + return std::make_unique(store, lock_fd); } } // namespace clice::index diff --git a/src/index/storage.h b/src/index/storage.h index d4286c0f4..18ed83d22 100644 --- a/src/index/storage.h +++ b/src/index/storage.h @@ -19,7 +19,7 @@ class CacheStore; namespace clice::index { -/// The three blob families the index persists. +/// The blob families the index persists. enum class IndexBlobKind : std::uint8_t { /// Per-file row blobs (ShardBlob), keyed by a path hash. Shard, @@ -27,6 +27,10 @@ enum class IndexBlobKind : std::uint8_t { Manifest, /// The single global blob (FileVersion table + symbols), key "global". Global, + /// The single CDB command-hash snapshot the persisted index was built + /// against, key "cdb" — how a cold start detects compile-command + /// changes that happened while no server was running. + Cdb, }; /// Storage backend for index blobs. The filesystem implementation below is @@ -54,10 +58,11 @@ class IndexStorage { }; /// Persist a batch in order. Atomicity is per blob, not per batch — a - /// crash can land a prefix; every load path treats any partially - /// written combination as stale data to rebuild. An entry that fails - /// to persist is logged and skipped, and its batch index returned so - /// the caller can re-dirty it for a later save. + /// crash can land a prefix, and every load path treats a committed + /// prefix as stale data to rebuild. A failed entry therefore fails the + /// rest of the batch too: what lands is always a prefix, never a + /// suffix without its prerequisites. The failed indices are logged and + /// returned so the caller can re-dirty them for a later save. virtual llvm::SmallVector write(llvm::ArrayRef batch) = 0; virtual void remove(IndexBlobKind kind, llvm::StringRef key) = 0; @@ -66,7 +71,11 @@ class IndexStorage { }; /// Filesystem implementation over the cache store; registers the index -/// namespaces on construction. +/// namespaces on construction. On a writable store this takes an exclusive +/// cross-process writer lock for the index (held until destruction) and +/// returns nullptr when another clice process already holds it — the +/// global/manifest blobs form one mutable lineage that tolerates no second +/// writer. Read-only stores skip the lock. std::unique_ptr make_fs_index_storage(CacheStore& store); } // namespace clice::index diff --git a/src/index/usr_generation.cpp b/src/index/usr_generation.cpp index 19205695e..29dc338a9 100644 --- a/src/index/usr_generation.cpp +++ b/src/index/usr_generation.cpp @@ -836,8 +836,13 @@ void USRGenerator::VisitTemplateName(TemplateName Name) { if(auto DT = Name.getAsDependentTemplateName()) { Out << '^'; printQualifier(Out, LangOpts, DT->getQualifier()); - /// FIXME: This may be operators. - Out << ':' << DT->getName().getIdentifier(); + Out << ':'; + auto name = DT->getName(); + if(const IdentifierInfo* II = name.getIdentifier()) { + Out << II->getName(); + } else { + Out << clang::getOperatorSpelling(name.getOperator()); + } } } diff --git a/src/server/compiler/context_resolver.cpp b/src/server/compiler/context_resolver.cpp index e3deef724..a0699bd81 100644 --- a/src/server/compiler/context_resolver.cpp +++ b/src/server/compiler/context_resolver.cpp @@ -222,7 +222,8 @@ bool ContextResolver::fill_header_context_args(llvm::StringRef path, std::uint32_t path_id, std::string& directory, std::vector& arguments, - Session* session) { + Session* session, + std::uint32_t* host_path_id) { // Opening one of our own synthesized files (prefix/suffix/snapshot): // it is a fragment of the host TU it was synthesized for, so compile // it with that host's command, treated as self-contained. It must not @@ -247,6 +248,9 @@ bool ContextResolver::fill_header_context_args(llvm::StringRef path, artifact_cmd.source_file = workspace.path_pool.resolve(path_id).data(); directory = artifact_cmd.resolved.directory.str(); arguments = artifact_cmd.to_string_argv(); + if(host_path_id) { + *host_path_id = it->second; + } return true; } @@ -332,6 +336,9 @@ bool ContextResolver::fill_header_context_args(llvm::StringRef path, } arguments = header_cmd.to_string_argv(); + if(host_path_id) { + *host_path_id = ctx_ptr->host_path_id; + } LOG_INFO("resolve_command: header context for {} (host={}, preamble={})", path, @@ -343,7 +350,8 @@ bool ContextResolver::fill_header_context_args(llvm::StringRef path, CommandSource ContextResolver::resolve_command(llvm::StringRef path, std::string& directory, std::vector& arguments, - Session* session) { + Session* session, + std::uint32_t* host_path_id) { auto path_id = workspace.path_pool.intern(path); llvm::SmallVector tried; @@ -379,7 +387,7 @@ CommandSource ContextResolver::resolve_command(llvm::StringRef path, // host source's CDB entry with file path replaced and preamble injected. if(has_host_choice) { tried.push_back("switch_context"); - if(fill_header_context_args(path, path_id, directory, arguments, session)) { + if(fill_header_context_args(path, path_id, directory, arguments, session, host_path_id)) { log_command_decision(path, tried, CommandSource::IncludeGraph, arguments); return CommandSource::IncludeGraph; } @@ -397,7 +405,7 @@ CommandSource ContextResolver::resolve_command(llvm::StringRef path, // 3. No CDB entry — try automatic header context resolution. if(!has_host_choice) { tried.push_back("include_graph"); - if(fill_header_context_args(path, path_id, directory, arguments, session)) { + if(fill_header_context_args(path, path_id, directory, arguments, session, host_path_id)) { log_command_decision(path, tried, CommandSource::IncludeGraph, arguments); return CommandSource::IncludeGraph; } diff --git a/src/server/compiler/context_resolver.h b/src/server/compiler/context_resolver.h index 53ec99fe3..596612e07 100644 --- a/src/server/compiler/context_resolver.h +++ b/src/server/compiler/context_resolver.h @@ -186,10 +186,13 @@ class ContextResolver { /// and finally a synthesized fallback command — so it always succeeds. /// Emits a per-file decision log (tiers tried, tier hit, command hash). /// @param session If non-null, used for header context resolution on open files. + /// @param host_path_id If non-null, receives the host source whose + /// command an IncludeGraph resolution borrowed (untouched otherwise). CommandSource resolve_command(llvm::StringRef path, std::string& directory, std::vector& arguments, - Session* session = nullptr); + Session* session = nullptr, + std::uint32_t* host_path_id = nullptr); /// Append the header context's suffix as one trailing #include line: the /// suffix content (everything after the include position along the chain) @@ -206,7 +209,8 @@ class ContextResolver { std::uint32_t path_id, std::string& directory, std::vector& arguments, - Session* session); + Session* session, + std::uint32_t* host_path_id); /// Validate a context choice persisted from an earlier run against the /// current CDB and include graph, dropping it when stale. Called on diff --git a/src/server/compiler/indexer.cpp b/src/server/compiler/indexer.cpp index dcbda26b4..5fed090ef 100644 --- a/src/server/compiler/indexer.cpp +++ b/src/server/compiler/indexer.cpp @@ -19,8 +19,10 @@ #include "support/logging.h" #include "support/timer.h" +#include "kota/codec/json/json.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" @@ -28,13 +30,108 @@ namespace clice { +namespace { + /// Stable blob key for a file's shard or a TU's manifest: runtime pool ids /// are per-session, so blobs are named by a hash of the path instead. -static std::string blob_key(llvm::StringRef path) { +std::string blob_key(llvm::StringRef path) { return std::format("{:016x}", llvm::xxh3_64bits(path)); } -void Indexer::merge(const void* tu_index_data, std::size_t size) { +/// JSON layout of the persisted CDB snapshot (blob kind Cdb): per source +/// file, the sorted canonical command hashes of its entries and a hash of +/// its matched config rules when the index state was last saved. A +/// standalone-indexed header gets an entry too (empty hashes): its own +/// matched rules plus the host source whose command its rows borrowed. +struct CdbSnapshotEntry { + std::string file; + std::vector hashes; + std::string rules; + std::string host; +}; + +struct CdbSnapshot { + std::vector entries; +}; + +/// clice.toml append/remove rules change the effective indexing command +/// without touching the CDB entry, so the snapshot must cover them too — +/// an offline rule edit is as stale-making as an offline command edit. +std::string rules_hash(const Config& config, llvm::StringRef file) { + std::vector append, remove; + config.match_rules(file, append, remove); + if(append.empty() && remove.empty()) { + return {}; + } + std::string joined; + for(auto& arg: append) { + joined += 'a'; + joined += arg; + joined += '\0'; + } + for(auto& arg: remove) { + joined += 'r'; + joined += arg; + joined += '\0'; + } + return std::format("{:016x}", llvm::xxh3_64bits(joined)); +} + +CdbSnapshot build_cdb_snapshot(Workspace& workspace, + const llvm::DenseMap& header_hosts, + llvm::ArrayRef standalone_debt) { + CdbSnapshot snapshot; + for(auto& [path_id, hashes]: workspace.cdb.command_hash_snapshot()) { + auto file = workspace.cdb.resolve_path(path_id).str(); + auto rules = rules_hash(workspace.config, file); + snapshot.entries.push_back({ + .file = std::move(file), + .hashes = {hashes.begin(), hashes.end()}, + .rules = std::move(rules), + }); + } + // Standalone-indexed TUs have no CDB entry, yet their effective command + // depends on their own matched rules and their borrowed host's command + // — both must be snapshot to detect offline changes. + auto add_standalone = [&](std::uint32_t tu) { + auto file = workspace.path_pool.resolve(tu); + if(workspace.cdb.has_entry(file)) { + return; + } + auto host_it = header_hosts.find(tu); + snapshot.entries.push_back({ + .file = file.str(), + .rules = rules_hash(workspace.config, file), + .host = host_it != header_hosts.end() + ? workspace.path_pool.resolve(host_it->second).str() + : std::string(), + }); + }; + for(auto tu: llvm::make_first_range(workspace.project_index.manifests)) { + add_standalone(tu); + } + // A dropped standalone TU whose rebuild has not landed keeps its entry: + // with no manifest pin and no CDB entry, the snapshot is the only + // record that an index is owed (reconcile's debt pass retries it). + for(auto tu: standalone_debt) { + add_standalone(tu); + } + // Deterministic bytes: save() decides "unchanged" by byte equality. + std::ranges::sort(snapshot.entries, {}, &CdbSnapshotEntry::file); + return snapshot; +} + +std::string serialize_cdb_snapshot(Workspace& workspace, + const llvm::DenseMap& header_hosts, + llvm::ArrayRef standalone_debt) { + auto json = + kota::codec::json::to_string(build_cdb_snapshot(workspace, header_hosts, standalone_debt)); + return json ? std::move(*json) : std::string(); +} + +} // namespace + +bool Indexer::merge(const void* tu_index_data, std::size_t size) { // Zero-copy consumption: the wire stays serialized; a new variant's // blob bytes are sliced out and installed or merged without decoding // the envelope, and only genuinely new symbol names are materialized. @@ -42,7 +139,7 @@ void Indexer::merge(const void* tu_index_data, std::size_t size) { index::TUIndex::from_bytes(llvm::StringRef(static_cast(tu_index_data), size)); if(!view.loaded()) { LOG_WARN("Ignoring TUIndex that failed verification"); - return; + return false; } auto main_local_id = view.path_count() - 1; llvm::StringRef main_tu_path = view.path(main_local_id); @@ -104,7 +201,7 @@ void Indexer::merge(const void* tu_index_data, std::size_t size) { // one membership test is the whole check — no IO, no bytes read. if(shard && shard->loaded() && shard->has_variant(blob_hash)) { if(!record_consumed(local_id, shard->content_hash())) { - return; + return false; } section_contributions.emplace_back(local_id, blob_hash); hits += 1; @@ -124,17 +221,17 @@ void Indexer::merge(const void* tu_index_data, std::size_t size) { LOG_WARN("Reject merge for {}: rows section for {} failed verification", main_tu_path, workspace.path_pool.resolve(global_id)); - return; + return false; } auto fresh = index::Shard::from_buffer(llvm::MemoryBuffer::getMemBufferCopy(bytes)); if(!fresh.loaded()) { LOG_WARN("Reject merge for {}: rows for {} do not form a valid shard", main_tu_path, workspace.path_pool.resolve(global_id)); - return; + return false; } if(!record_consumed(local_id, fresh.content_hash())) { - return; + return false; } index::Shard replacement; @@ -168,7 +265,7 @@ void Indexer::merge(const void* tu_index_data, std::size_t size) { // rebuilt. if(!project.merge(view, file_ids_map)) { LOG_WARN("Reject merge for {}: symbol reference bitmap failed verification", main_tu_path); - return; + return false; } // Intern a FileVersion per file of the parse. The freshness baseline is @@ -268,6 +365,7 @@ void Indexer::merge(const void* tu_index_data, std::size_t size) { appended, rebuilt_ids.size(), workspace.shards.size()); + return true; } void Indexer::drop_index(std::uint32_t tu_path_id) { @@ -387,10 +485,10 @@ kota::task<> Indexer::save() { } auto manifest_count = batch.size() - shard_count; - // The global blob goes last: a crash mid-batch then strands only - // manifests, which load() drops by their generation stamp — the - // reverse order would strand a global claiming symbols in files whose - // rows never landed. + // The global blob follows the shards and manifests: a crash mid-batch + // then strands only manifests, which load() drops by their generation + // stamp — the reverse order would strand a global claiming symbols in + // files whose rows never landed. if(global_dirty) { std::string bytes; llvm::raw_string_ostream os(bytes); @@ -398,6 +496,26 @@ kota::task<> Indexer::save() { batch.push_back({index::IndexBlobKind::Global, "global", std::move(bytes)}); } + // The CDB snapshot goes last, after the whole index state it + // describes: landed before the global, a crash between the two would + // leave the old global still pinning old-command manifests under a + // snapshot that already matches the live CDB — reconcile would then + // never drop them. A save that commits nothing skips the recompute + // (unless the snapshot itself is owed a rewrite): a pure CDB change + // dirties no blob on its own — the invalidator's drops do — so the + // next dirtying save carries the fresh snapshot. + std::string cdb_bytes; + std::optional cdb_index; + if(!batch.empty() || !removals.empty() || cdb_dirty) { + cdb_bytes = serialize_cdb_snapshot(workspace, header_hosts, standalone_debt()); + if(!cdb_bytes.empty() && cdb_bytes != persisted_cdb_snapshot) { + cdb_index = batch.size(); + batch.push_back({index::IndexBlobKind::Cdb, "cdb", cdb_bytes}); + } else { + cdb_dirty = false; + } + } + dirty_shards.clear(); dirty_manifests.clear(); global_dirty = false; @@ -431,10 +549,19 @@ kota::task<> Indexer::save() { dirty_shards.insert(shard_ids[i]); } else if(i - shard_count < manifest_count) { dirty_manifests.insert(manifest_ids[i - shard_count]); + } else if(cdb_index && i == *cdb_index) { + // Keep the old persisted bytes and stay dirty so the next + // save retries even when nothing else changes by then. + cdb_dirty = true; + cdb_index.reset(); } else { global_dirty = true; } } + if(cdb_index) { + persisted_cdb_snapshot = std::move(cdb_bytes); + cdb_dirty = false; + } saved_shards = shard_count - failed_shards; saving_shards = 0; @@ -446,14 +573,17 @@ kota::task<> Indexer::save() { timer.ms()); } -void Indexer::load() { +bool Indexer::load(bool read_only) { if(!workspace.index_storage) - return; + return true; auto& storage = *workspace.index_storage; auto& project = workspace.project_index; ScopedTimer timer; auto sweep_all = [&] { + if(read_only) { + return; + } for(auto kind: {index::IndexBlobKind::Shard, index::IndexBlobKind::Manifest}) { llvm::SmallVector keys; storage.for_each_key(kind, [&](llvm::StringRef key) { keys.push_back(key.str()); }); @@ -474,19 +604,21 @@ void Indexer::load() { if(storage.contains(index::IndexBlobKind::Global, "global")) { LOG_WARN("Index global blob unreadable; disabling index persistence this session"); workspace.index_storage.reset(); - return; + return true; } // No global table means no resolvable manifests: everything else // is unreachable data, swept so it cannot survive as orphans. sweep_all(); - return; + return true; } llvm::DenseMap manifest_pins; if(!project.load_global(global->getBuffer(), workspace.path_pool, manifest_pins)) { LOG_INFO("Discarding old-format index global blob"); sweep_all(); - storage.remove(index::IndexBlobKind::Global, "global"); - return; + if(!read_only) { + storage.remove(index::IndexBlobKind::Global, "global"); + } + return false; } // Adopt exactly the manifests the global blob pins, at exactly the @@ -519,8 +651,10 @@ void Indexer::load() { auto tu_path_id = project.file_versions.find(manifest->tu_fv)->second.path_id; project.apply_manifest(tu_path_id, std::move(*manifest)); }); - for(auto& key: dead_manifests) { - storage.remove(index::IndexBlobKind::Manifest, key); + if(!read_only) { + for(auto& key: dead_manifests) { + storage.remove(index::IndexBlobKind::Manifest, key); + } } // A pinned TU without an adopted manifest lost it to a failed write // that the landed global outran, or to a lost removal; re-enqueue it — @@ -601,8 +735,10 @@ void Indexer::load() { // the mask refresh below, like the merge path's. auto affected = project.remove_manifest(tu); mask_refresh.append(affected.begin(), affected.end()); - storage.remove(index::IndexBlobKind::Manifest, - blob_key(workspace.path_pool.resolve(tu))); + if(!read_only) { + storage.remove(index::IndexBlobKind::Manifest, + blob_key(workspace.path_pool.resolve(tu))); + } enqueue(tu, ReindexReason::ContentChanged); } } @@ -614,14 +750,17 @@ void Indexer::load() { } // Sweep shard blobs nothing references any more. - llvm::SmallVector orphans; - storage.for_each_key(index::IndexBlobKind::Shard, [&](llvm::StringRef key) { - if(!expected_keys.contains(key)) { - orphans.push_back(key.str()); + if(!read_only) { + llvm::SmallVector orphans; + storage.for_each_key(index::IndexBlobKind::Shard, [&](llvm::StringRef key) { + if(!expected_keys.contains(key)) { + orphans.push_back(key.str()); + } + }); + for(auto& key: orphans) { + storage.remove(index::IndexBlobKind::Shard, key); } - }); - for(auto& key: orphans) { - storage.remove(index::IndexBlobKind::Shard, key); + reconcile_cdb_snapshot(); } if(!workspace.shards.empty()) { @@ -636,6 +775,169 @@ void Indexer::load() { workspace.shards.size(), project.manifests.size(), timer.ms()); + return true; +} + +llvm::SmallVector Indexer::standalone_debt() { + llvm::SmallVector debt; + llvm::DenseSet seen; + auto add = [&](std::uint32_t id) { + if(workspace.project_index.manifests.contains(id) || + workspace.cdb.has_entry(workspace.path_pool.resolve(id)) || !seen.insert(id).second) { + return; + } + debt.push_back(id); + }; + for(auto id: failed_ids) { + add(id); + } + for(auto id: llvm::make_first_range(reindex_reasons)) { + add(id); + } + return debt; +} + +void Indexer::reconcile_cdb_snapshot() { + auto blob = workspace.index_storage->read(index::IndexBlobKind::Cdb, "cdb"); + CdbSnapshot persisted; + if(!blob || !kota::codec::json::from_string(std::string_view(blob->getBuffer()), persisted)) { + // Unknown baseline: nothing to diff against. Dirty the snapshot so + // the next save recreates it even when it commits nothing else — + // after a crash that lost only the CDB blob, waiting for an + // unrelated dirtying merge would leave offline command edits + // undetectable across every following session. + cdb_dirty = true; + return; + } + persisted_cdb_snapshot = blob->getBuffer().str(); + + llvm::StringMap before; + for(auto& entry: persisted.entries) { + before[entry.file] = &entry; + } + auto& project = workspace.project_index; + llvm::DenseSet cdb_ids; + llvm::SmallVector changed_ids; + auto snapshot = build_cdb_snapshot(workspace, header_hosts, {}); + for(auto& entry: snapshot.entries) { + if(entry.hashes.empty()) { + continue; + } + auto server_id = workspace.path_pool.intern(entry.file); + cdb_ids.insert(server_id); + auto it = before.find(entry.file); + if(it != before.end() && it->second->hashes == entry.hashes && + it->second->rules == entry.rules) { + continue; + } + changed_ids.push_back(server_id); + if(!project.manifests.contains(server_id)) { + continue; + } + LOG_INFO("Compile command changed since the last session; reindexing {}", entry.file); + drop_index(server_id); + enqueue(server_id, ReindexReason::ContentChanged); + } + + // A standalone-indexed header borrowed a host source's command and + // applied its own matched rules on top, so an offline change to either + // staled its rows exactly like the live CDB path's hosted-header + // invalidation. A header whose recorded host survives unchanged and + // still includes it is pinned fresh; one with no recorded host (older + // snapshot) falls back to the include-reachability approximation below. + llvm::DenseSet pinned_fresh; + for(auto& entry: snapshot.entries) { + if(!entry.hashes.empty()) { + continue; + } + auto it = before.find(entry.file); + if(it == before.end()) { + // Not in the persisted snapshot: the next save adopts it, and + // offline changes against an unknown baseline are undetectable + // anyway. + continue; + } + auto& old = *it->second; + // The header's own CDB entry vanished: keep the index — last-known + // content still serves navigation, mirroring the live treatment. + if(!old.hashes.empty()) { + continue; + } + auto server_id = workspace.path_pool.intern(entry.file); + if(old.rules != entry.rules) { + LOG_INFO("Config rules changed since the last session; reindexing {}", entry.file); + drop_index(server_id); + enqueue(server_id, ReindexReason::ContentChanged); + continue; + } + if(old.host.empty()) { + continue; + } + auto host_id = workspace.path_pool.intern(old.host); + if(!workspace.cdb.has_entry(old.host) || llvm::is_contained(changed_ids, host_id)) { + LOG_INFO("Host compile command changed since the last session; reindexing {}", + entry.file); + drop_index(server_id); + enqueue(server_id, ReindexReason::ContentChanged); + continue; + } + // The dependency scan preceding this load saw the offline edits, so + // a recorded host that no longer includes the header cannot vouch + // for the borrowed command any more. Keep the rows serving (last + // known good, like the vanished-entry case above) while a rebuild + // re-selects a host; a Fallback resolution then changes nothing. + // The retained rows were still built through the old host, so keep + // that association until a landed rebuild overwrites it — an empty + // host persisted after a Fallback or failed rebuild would hit the + // `old.host.empty()` gate next session and never retry. + if(workspace.dep_graph.find_include_chain(host_id, server_id).empty()) { + LOG_INFO("Recorded host no longer includes {}; reindexing", entry.file); + header_hosts[server_id] = host_id; + enqueue(server_id, ReindexReason::ContentChanged); + continue; + } + header_hosts[server_id] = host_id; + pinned_fresh.insert(server_id); + } + + // A persisted standalone entry whose file has no manifest is recorded + // debt: its index was dropped for a command or rule change and no + // rebuild has landed since — with no manifest pin and no CDB entry, + // nothing else would ever retry it. Re-enqueue while the file exists; + // a vanished file's debt dies with its entry at the next save. + for(auto& old: persisted.entries) { + if(!old.hashes.empty() || workspace.cdb.has_entry(old.file)) { + continue; + } + auto server_id = workspace.path_pool.intern(old.file); + if(project.manifests.contains(server_id) || reindex_reasons.contains(server_id) || + !fs::exists(old.file)) { + continue; + } + LOG_INFO("Index owed from the last session; reindexing {}", old.file); + enqueue(server_id, ReindexReason::ContentChanged); + } + if(changed_ids.empty()) { + return; + } + + llvm::SmallVector hosted; + for(auto tu: llvm::make_first_range(project.manifests)) { + if(cdb_ids.contains(tu) || pinned_fresh.contains(tu)) { + continue; + } + if(llvm::any_of(changed_ids, [&](std::uint32_t host) { + return !workspace.dep_graph.find_include_chain(host, tu).empty(); + })) { + hosted.push_back(tu); + } + } + for(auto header_id: hosted) { + LOG_INFO("Host compile command changed since the last session; reindexing {}", + workspace.path_pool.resolve(header_id)); + drop_index(header_id); + enqueue(header_id, ReindexReason::ContentChanged); + } } bool Indexer::file_version_stale(std::uint32_t fv_id) { @@ -815,9 +1117,26 @@ kota::task<> Indexer::index_one(std::uint32_t server_path_id, params.file = file_path; // Bulk background indexing sticks to real commands; synthesized fallback // commands would fill the index with guesses. - if(contexts.resolve_command(file_path, params.directory, params.arguments, nullptr) == - CommandSource::Fallback) + std::uint32_t host_path_id = no_path_id; + auto source = contexts.resolve_command(file_path, + params.directory, + params.arguments, + nullptr, + &host_path_id); + if(source == CommandSource::Fallback) { + // A file whose manifest survives keeps serving its last-known rows, + // so skipping it loses nothing. One without a manifest (dropped or + // never built) stays unindexed — count that as a failure so a batch + // run reports the debt instead of exiting clean. + if(!workspace.project_index.manifests.contains(server_path_id)) { + LOG_WARN("[{}/{}] No compile command found for {}; it stays unindexed", + index, + total, + file_path); + failed_ids.insert(server_path_id); + } co_return; + } workspace.fill_pcm_deps(params.pcms); @@ -841,7 +1160,21 @@ kota::task<> Indexer::index_one(std::uint32_t server_path_id, co_return; } ScopedTimer merge_timer; - merge(result.value().tu_index_data.data(), result.value().tu_index_data.size()); + if(!merge(result.value().tu_index_data.data(), result.value().tu_index_data.size())) { + // Rejected wholesale: the file's rows are missing or stale, + // which is a failure, not a completed index. + failed_ids.insert(server_path_id); + co_return; + } + failed_ids.erase(server_path_id); + // Record the borrowed host only for rows that landed: written at + // dispatch, a failed rebuild would leave the persisted CDB + // snapshot naming the new host while the retained rows were built + // through the old one — an unchanged new host then pins those + // stale rows fresh across restarts. + if(source == CommandSource::IncludeGraph) { + header_hosts[server_path_id] = host_path_id; + } LOG_PERF("index", "progress={}/{} file={} bytes={} index_ms={} merge_ms={}", index, @@ -852,8 +1185,10 @@ kota::task<> Indexer::index_one(std::uint32_t server_path_id, merge_timer.ms()); } else if(result.has_value() && !result.value().success) { LOG_WARN("[{}/{}] Index failed for {}: {}", index, total, file_path, result.value().error); + failed_ids.insert(server_path_id); } else if(result.has_value() && result.value().tu_index_data.empty()) { LOG_WARN("[{}/{}] Index returned empty TUIndex for {}", index, total, file_path); + failed_ids.insert(server_path_id); } else if(result.error().code == worker::dispatch_errc::cancelled || result.error().code == worker::dispatch_errc::worker_crashed || (result.error().code == worker::dispatch_errc::worker_unavailable && @@ -891,6 +1226,7 @@ kota::task<> Indexer::index_one(std::uint32_t server_path_id, file_path, max_requeue_attempts, result.error().message); + failed_ids.insert(server_path_id); break; } case RequeueVerdict::Requeued: { @@ -908,6 +1244,7 @@ kota::task<> Indexer::index_one(std::uint32_t server_path_id, total, file_path, result.error().message); + failed_ids.insert(server_path_id); } } diff --git a/src/server/compiler/indexer.h b/src/server/compiler/indexer.h index 2f9e549c7..bbf07bf86 100644 --- a/src/server/compiler/indexer.h +++ b/src/server/compiler/indexer.h @@ -12,6 +12,7 @@ #include "kota/async/async.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" namespace clice { @@ -139,8 +140,10 @@ class Indexer { /// Merge a TUIndex result: intern FileVersions, replace the TU's /// manifest, and write row blobs only for variants no shard stores yet /// — a re-index whose rows are unchanged records its contributions and - /// touches nothing else. - void merge(const void* tu_index_data, std::size_t size); + /// touches nothing else. Returns false when the result failed + /// verification and nothing was committed — the caller must count the + /// file as failed, not indexed. + bool merge(const void* tu_index_data, std::size_t size); /// Drop a TU's index wholesale: manifest and contributions now (the /// affected shards' live masks follow), persisted blobs at the next @@ -158,7 +161,13 @@ class Indexer { /// Load the global blob, adopt every resolvable manifest, fetch the /// shard blobs the contributions expect, and sweep the rest. - void load(); + /// `read_only` keeps the sweeps in memory only: an out-of-process + /// reader (`clice index --stats`) must not delete blobs a concurrently + /// running server may be about to reference. Returns false when a + /// global blob existed but could not be decoded (old format or + /// corrupt): the server rebuilds from scratch, but a read-only reader + /// must report an unusable cache instead of an empty index. + bool load(bool read_only = false); /// Shard blobs whose write has not durably completed: dirty since the /// last save plus the batch a running save is committing. The gauge @@ -187,6 +196,15 @@ class Indexer { return index_queue.size(); } + /// Files whose latest index attempt failed for good — rejected by the + /// worker, an empty or unverifiable result, a spent crash budget, or a + /// dead IPC path — with no retry pending. Their rows are missing or + /// stale; a later successful pass removes them again. The one-shot + /// `clice index` reports a partial build from this. + std::size_t failed_files() const { + return failed_ids.size(); + } + /// How many shard blobs the last save() durably committed. A /// steady-state save commits 0 — only variant-set changes rewrite a /// blob — so the stats endpoint can pin full-rewrite regressions. @@ -194,6 +212,13 @@ class Indexer { return saved_shards; } + /// Whether index state remains that no save() committed. After a final + /// save this means write failures whose retry never came — the one-shot + /// `clice index` must not report a durable index from this. + bool has_unsaved_state() const { + return !dirty_shards.empty() || !dirty_manifests.empty() || global_dirty || cdb_dirty; + } + /// Progress of the current (or last) indexing round. The reporter reads /// this on each on_progress_changed emission — the signal only wakes it, /// the numbers live here. @@ -310,6 +335,43 @@ class Indexer { llvm::DenseSet dirty_manifests; bool global_dirty = false; + /// The persisted CDB snapshot blob's bytes as last read or written; + /// empty when none exists. save() rewrites the blob whenever the live + /// CDB serializes differently. + std::string persisted_cdb_snapshot; + + /// The persisted CDB snapshot needs a rewrite no dirty blob will + /// trigger: its write failed while the rest of the batch may have + /// landed, or load() found it missing or corrupt next to a valid + /// global. Without this flag the rewrite would wait for an unrelated + /// dirtying merge: a save with nothing else to commit skips the + /// snapshot recompute entirely. + bool cdb_dirty = false; + + /// Host source whose command each standalone-indexed header's retained + /// rows borrowed, recorded when a merge lands and persisted in the CDB + /// snapshot. + /// The offline invalidator checks the recorded host directly — the + /// include graph is rebuilt from the NEW commands before load(), so + /// reachability alone cannot see a change that removed or redirected + /// the very include edge the header's context came through. + llvm::DenseMap header_hosts; + + /// Standalone TUs owed an index that nothing else records: no manifest + /// (dropped for a command or rule change, the rebuild failed or is + /// still pending) and no CDB entry the startup sweep would retry. The + /// snapshot keeps an entry for each so reconcile's debt pass retries + /// them next session instead of the index staying silently partial. + llvm::SmallVector standalone_debt(); + + /// Diff the persisted CDB snapshot against the live CDB and drop the + /// index of every TU whose compile command changed while no server was + /// running — content-based freshness cannot see command changes, so an + /// adopted manifest would keep serving the old-command rows forever. + /// Entries that vanished keep their index (last-known content still + /// serves navigation), mirroring the live CDB-reload treatment. + void reconcile_cdb_snapshot(); + /// Per-round FileVersion staleness verdicts: many TUs share the same /// versions, and one stat (or repair) per version per round is enough. /// Cleared when a round starts. @@ -326,6 +388,7 @@ class Indexer { bool need_update(llvm::StringRef file_path); llvm::DenseMap reindex_reasons; + llvm::DenseSet failed_ids; std::uint64_t reindex_ticket = 0; bool indexing_active = false; bool indexing_scheduled = false; diff --git a/src/support/cache_store.cpp b/src/support/cache_store.cpp index 4e9d778ec..a5969a05a 100644 --- a/src/support/cache_store.cpp +++ b/src/support/cache_store.cpp @@ -25,6 +25,7 @@ #include "support/logging.h" #include "kota/codec/json/json.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/Support/Error.h" @@ -38,6 +39,10 @@ namespace { /// Manifest checkpoint is forced after this many commits/invalidates. constexpr std::uint32_t checkpoint_interval = 16; +/// Root-level lock file serializing open()'s version sweep against layout +/// startup; lives beside the version directories, exempt from the sweep. +constexpr llvm::StringLiteral store_lock_name = "store.lock"; + /// JSON layout of manifest.json. Only an acceleration structure: blob /// presence and size always come from the filesystem; the manifest merely /// carries last-accessed times across restarts. @@ -138,6 +143,22 @@ void sweep_dead_pid_dirs(llvm::StringRef dir, std::uint32_t self_pid) { } } +/// Whether a live process is working inside this cache layout: every +/// writable open creates `tmp/{pid}` and keeps it until shutdown. +bool has_live_instance(llvm::StringRef base) { + std::error_code ec; + auto tmp_parent = path::join(base, "tmp"); + for(auto it = llvm::sys::fs::directory_iterator(tmp_parent, ec); + !ec && it != llvm::sys::fs::directory_iterator(); + it.increment(ec)) { + std::uint32_t pid = 0; + if(!path::filename(it->path()).getAsInteger(10, pid) && is_pid_alive(pid)) { + return true; + } + } + return false; +} + std::int64_t now_ms() { auto now = std::chrono::system_clock::now().time_since_epoch(); return std::chrono::duration_cast(now).count(); @@ -185,6 +206,15 @@ struct CacheStore::State { std::uint32_t changes_since_checkpoint = 0; bool dirty = false; + /// Inspection mode (open's read_only): no directory is created or + /// swept, and writes are a caller bug. + bool read_only = false; + + /// First namespace directory scan that failed (permissions, IO): + /// the affected namespace looks empty while its blobs exist, so + /// readers must not mistake the store for empty. + std::error_code scan_failure; + /// Logical clock: strictly increasing per issued stamp so that LRU /// ordering is deterministic even within one millisecond. std::int64_t last_stamp = 0; @@ -239,35 +269,86 @@ CacheStore& CacheStore::operator=(CacheStore&&) noexcept = default; CacheStore::~CacheStore() = default; std::expected CacheStore::open(llvm::StringRef root, - std::uint32_t version) { + std::uint32_t version, + bool read_only) { assert(!root.empty() && "cache root must not be empty"); auto state = std::make_unique(); state->self_pid = static_cast(llvm::sys::Process::getProcessId()); + state->read_only = read_only; auto parent = path::join(root, "cache"); auto version_dir = std::format("v{}", version); state->base = path::join(parent, version_dir); + if(read_only) { + if(!llvm::sys::fs::is_directory(state->base)) { + return std::unexpected(std::make_error_code(std::errc::no_such_file_or_directory)); + } + return CacheStore(std::move(state)); + } + + // Only the parent may exist before the lock is held: it hosts the lock + // file and is never swept. + if(auto ec = llvm::sys::fs::create_directories(parent)) { + return std::unexpected(ec); + } + + // The pid marker created below is what shields a layout from another + // version's sweep, but it only exists at the end of this function: a + // sweeper scanning between our create_directories and the marker would + // reclaim the layout we are initializing. This root-level lock covers + // exactly that window — every writable open holds it from before its + // version directory exists until its marker does. Best effort: on a + // filesystem without advisory locks the window stays open, as before. + int lock_fd = -1; + auto lock_path = path::join(parent, store_lock_name); + if(auto ec = llvm::sys::fs::openFileForReadWrite(lock_path, + lock_fd, + llvm::sys::fs::CD_OpenAlways, + llvm::sys::fs::OF_None)) { + LOG_WARN("CacheStore: cannot open {}: {}", lock_path, ec.message()); + lock_fd = -1; + } else if(auto ec2 = llvm::sys::fs::lockFile(lock_fd)) { + LOG_WARN("CacheStore: cannot lock {}: {}", lock_path, ec2.message()); + llvm::sys::Process::SafelyCloseFileDescriptor(lock_fd); + lock_fd = -1; + } + auto unlock = llvm::make_scope_exit([&] { + if(lock_fd != -1) { + llvm::sys::fs::unlockFile(lock_fd); + llvm::sys::Process::SafelyCloseFileDescriptor(lock_fd); + } + }); + if(auto ec = llvm::sys::fs::create_directories(state->base)) { return std::unexpected(ec); } // Discard anything that isn't the current layout version: older version - // directories and any stray files. + // directories and any stray files. A layout still holding a live + // instance stays: a clice of another version is serving from it, and + // its writer locks live inside the directory, so they cannot protect + // it from us — a later open reclaims it once that process exits. std::error_code ec; for(auto it = llvm::sys::fs::directory_iterator(parent, ec); !ec && it != llvm::sys::fs::directory_iterator(); it.increment(ec)) { - if(path::filename(it->path()) == version_dir) { + auto name = path::filename(it->path()); + if(name == version_dir || name == store_lock_name) { continue; } - LOG_INFO("CacheStore: discarding stale cache layout {}", it->path()); - if(llvm::sys::fs::is_directory(it->path())) { - fs::remove_all(it->path()); - } else { + if(!llvm::sys::fs::is_directory(it->path())) { + LOG_INFO("CacheStore: discarding stale cache layout {}", it->path()); llvm::sys::fs::remove(it->path()); + continue; } + if(has_live_instance(it->path())) { + LOG_INFO("CacheStore: keeping cache layout {}, still in use", it->path()); + continue; + } + LOG_INFO("CacheStore: discarding stale cache layout {}", it->path()); + fs::remove_all(it->path()); } // Load the manifest. Corrupt or missing is fine: registration falls @@ -306,7 +387,9 @@ void CacheStore::register_namespace(CacheNamespace ns) { std::lock_guard guard(state->mutex); auto ns_dir = path::join(state->base, ns.name); - llvm::sys::fs::create_directories(ns_dir); + if(!state->read_only) { + llvm::sys::fs::create_directories(ns_dir); + } auto [it, inserted] = state->namespaces.try_emplace(ns.name); assert(inserted && "namespace registered twice"); @@ -317,10 +400,13 @@ void CacheStore::register_namespace(CacheNamespace ns) { ns_state.config = std::move(ns); if(ns_state.config.policy == CachePolicy::Scratch) { + ns_state.dir = path::join(ns_dir, std::to_string(state->self_pid)); + if(state->read_only) { + return; + } // Scratch directories are per-instance; reclaim those left behind // by crashed instances and start with a fresh one of our own. sweep_dead_pid_dirs(ns_dir, state->self_pid); - ns_state.dir = path::join(ns_dir, std::to_string(state->self_pid)); fs::remove_all(ns_state.dir); llvm::sys::fs::create_directories(ns_state.dir); return; @@ -375,6 +461,13 @@ void CacheStore::register_namespace(CacheNamespace ns) { ns_state.total_size += status.getSize(); primary_mtimes[filename] = status.getLastModificationTime(); } + // A read-only open of a store whose namespace was never created is a + // legitimately empty scan; any other failure hides existing blobs. + if(ec && !(state->read_only && ec == std::errc::no_such_file_or_directory) && + !state->scan_failure) { + LOG_WARN("CacheStore: failed to scan namespace {}: {}", ns_state.dir, ec.message()); + state->scan_failure = ec; + } // Attach aux blobs to their entries. An aux without a primary is crash // residue (a pair evicted halfway); an aux OLDER than its primary is @@ -390,7 +483,7 @@ void CacheStore::register_namespace(CacheNamespace ns) { if(it != ns_state.entries.end() && entry.getValue().mtime >= primary_mtimes[key]) { it->second.aux_size = entry.getValue().size; ns_state.total_size += entry.getValue().size; - } else { + } else if(!state->read_only) { llvm::sys::fs::remove(state->aux_blob_path(ns_state, key)); LOG_DEBUG("CacheStore: removed stale aux blob {} in {}", key, ns_state.config.name); } @@ -438,6 +531,11 @@ std::optional CacheStore::lookup_aux(llvm::StringRef ns, llvm::Stri CacheStore::PendingEntry CacheStore::begin_store(llvm::StringRef ns, llvm::StringRef key) { std::lock_guard guard(state->mutex); + assert(!state->read_only && "write on a read-only store"); + if(state->read_only) { + LOG_ERROR("CacheStore: begin_store on a read-only store"); + return {}; + } auto* ns_state = state->find_namespace(ns); assert(ns_state && "begin_store on unregistered namespace"); @@ -459,6 +557,11 @@ CacheStore::PendingEntry CacheStore::begin_store(llvm::StringRef ns, llvm::Strin CacheStore::PendingEntry CacheStore::begin_store_aux(llvm::StringRef ns, llvm::StringRef key) { std::lock_guard guard(state->mutex); + assert(!state->read_only && "write on a read-only store"); + if(state->read_only) { + LOG_ERROR("CacheStore: begin_store_aux on a read-only store"); + return {}; + } auto* ns_state = state->find_namespace(ns); assert(ns_state && "begin_store_aux on unregistered namespace"); @@ -617,6 +720,9 @@ void CacheStore::PendingEntry::remove_tmp() { void CacheStore::invalidate(llvm::StringRef ns, llvm::StringRef key) { { std::lock_guard guard(state->mutex); + if(state->read_only) { + return; + } auto* ns_state = state->find_namespace(ns); if(!ns_state) { @@ -684,8 +790,17 @@ llvm::StringRef CacheStore::base_dir() const { return state->base; } +bool CacheStore::read_only() const { + return state->read_only; +} + +std::error_code CacheStore::scan_error() const { + std::lock_guard guard(state->mutex); + return state->scan_failure; +} + void CacheStore::State::evict_locked(Namespace& ns, llvm::StringRef keep_key) { - if(ns.config.policy != CachePolicy::LRU || ns.config.max_bytes == 0 || + if(read_only || ns.config.policy != CachePolicy::LRU || ns.config.max_bytes == 0 || ns.total_size <= ns.config.max_bytes) { return; } @@ -749,7 +864,10 @@ void CacheStore::State::evict_locked(Namespace& ns, llvm::StringRef keep_key) { } void CacheStore::State::checkpoint_locked() { - if(!dirty) { + // Read-only: lookups still bump in-memory atimes (setting dirty), but + // nothing may reach the disk — with no tmp_dir the write below would + // even land a manifest in the process working directory. + if(read_only || !dirty) { return; } @@ -798,6 +916,9 @@ void CacheStore::maybe_checkpoint() { void CacheStore::shutdown() { std::lock_guard guard(state->mutex); + if(state->read_only) { + return; + } state->checkpoint_locked(); fs::remove_all(state->tmp_dir); diff --git a/src/support/cache_store.h b/src/support/cache_store.h index 42a4025ad..61dd1e29e 100644 --- a/src/support/cache_store.h +++ b/src/support/cache_store.h @@ -140,8 +140,15 @@ class CacheStore { /// Open (creating if necessary) the store under `root`. Any sibling /// version directory other than v{version} is deleted. Loads the /// manifest if present and sweeps tmp directories of dead instances. + /// + /// `read_only` opens for inspection without touching the disk: nothing + /// is created, swept or discarded — a store another process (possibly + /// an older layout version) is live on must survive being inspected. + /// Fails with `no_such_file_or_directory` when the versioned directory + /// does not exist; only lookups and enumeration may be used. static std::expected open(llvm::StringRef root, - std::uint32_t version); + std::uint32_t version, + bool read_only = false); CacheStore(CacheStore&&) noexcept; CacheStore& operator=(CacheStore&&) noexcept; @@ -214,6 +221,16 @@ class CacheStore { /// manages namespace subdirectories); they die with the version. llvm::StringRef base_dir() const; + /// Whether the store was opened in read-only inspection mode. + bool read_only() const; + + /// First namespace directory scan that failed since open (default + /// error_code when none did). A failed scan makes the namespace look + /// empty while its blobs exist, so a reader that would report "no + /// data" must check this first. A read-only open of a namespace that + /// was never created scans empty legitimately and is not a failure. + std::error_code scan_error() const; + /// Atomically persist the manifest (key sizes and last-accessed times) /// if anything changed. Also runs automatically every few commits; /// the owner should additionally schedule it periodically and call diff --git a/tests/integration/lifecycle/subcommands.test.ts b/tests/integration/lifecycle/subcommands.test.ts index be0f3848b..387df97b3 100644 --- a/tests/integration/lifecycle/subcommands.test.ts +++ b/tests/integration/lifecycle/subcommands.test.ts @@ -1,13 +1,26 @@ import { spawnSync } from "node:child_process"; +import type { CliceClient } from "@clice/tools/client"; +import { sleep } from "@clice/tools/client"; import { cliceExecutable, expect, test } from "../fixtures.ts"; const SUBCOMMANDS = ["serve", "query", "worker", "index", "doc", "lint", "format"]; -const STUBS = ["index", "doc", "lint", "format"]; +const STUBS = ["doc", "lint", "format"]; function runClice(...args: string[]) { return spawnSync(cliceExecutable(), args, { encoding: "utf8", timeout: 30_000 }); } +async function waitSymbol(client: CliceClient, name: string): Promise { + for (let i = 0; i < 30; i += 1) { + const symbols = await client.workspaceSymbols(name); + if (symbols?.some((s) => s.name === name)) { + return true; + } + await sleep(1_000); + } + return false; +} + test("root usage lists subcommands", () => { // Both the bare invocation and --help print the root usage and succeed. for (const args of [[], ["--help"]]) { @@ -40,3 +53,77 @@ test("subcommand help", () => { test("unknown subcommand fails", () => { expect(runClice("bogus").status).not.toBe(0); }); + +test("index subcommand builds and resumes", ({ session }) => { + const ws = session.tmpdir(); + ws.pinCacheDir(); + ws.write("main.cpp", "int add(int a, int b) { return a + b; }\n"); + ws.writeCDB(["main.cpp"]); + + // A stats query before any index run reports the missing cache. + const empty = runClice("index", "--stats", "--workspace", ws.root); + expect(empty.status).toBe(1); + expect(empty.stderr).toContain("No index cache"); + + const args = ["index", "--workspace", ws.root, "--workers", "2"]; + const first = runClice(...args); + expect(first.status, `stderr: ${first.stderr}`).toBe(0); + expect(first.stderr).toContain("] Indexing "); + expect(first.stdout).toContain("Indexed 1 translation units"); + + // The second run resumes from the persisted index: the hash gate + // skips the fresh TU without recompiling it. + const second = runClice(...args); + expect(second.status, `stderr: ${second.stderr}`).toBe(0); + expect(second.stderr).not.toContain("] Indexing "); + + const stats = runClice("index", "--stats", "--workspace", ws.root); + expect(stats.status, `stderr: ${stats.stderr}`).toBe(0); + expect(stats.stdout).toContain("Translation units: 1"); +}); + +test("index reports header losing host", async ({ session }) => { + const ws = session.tmpdir(); + ws.pinCacheDir(); + ws.write("a.h", "#pragma once\ninline int alpha() { return 1; }\n"); + ws.write("main.cpp", '#include "a.h"\nint app_entry() { return alpha(); }\n'); + ws.writeCDB(["main.cpp"]); + + // Standalone-index a.h: edit it on disk while its buffer is open, so the + // close sees the shard/disk mismatch and reindexes the header with + // main.cpp as its borrowed host. `beta` can only come from that reindex — + // the tracker loops are off and main.cpp is never touched again. + const client = await session.spawn(ws).initialize(ws); + expect(await waitSymbol(client, "alpha"), "TU never indexed").toBe(true); + const [headerUri] = await client.openAndWait("a.h"); + ws.write( + "a.h", + "#pragma once\ninline int alpha() { return 1; }\ninline int beta() { return 2; }\n", + ); + client.close(headerUri); + expect(await waitSymbol(client, "beta"), "header never standalone-indexed").toBe(true); + await client.shutdown(); + + // Offline, the host's command changes and its include of a.h vanishes: + // reconciliation drops the header's index and no TU can host it any more, + // so the batch run must report the header as lost coverage. + ws.write("main.cpp", "int app_entry() { return 0; }\n"); + ws.writeCDB(["main.cpp"], { extraArgs: ["-DHOST_V2"] }); + + const second = runClice("index", "--workspace", ws.root, "--workers", "2"); + expect(second.status, `stderr: ${second.stderr}`).toBe(1); + expect(second.stderr).toContain("stays unindexed"); + expect(second.stdout).toContain("failed to index"); + + // The debt persists across runs: the snapshot keeps recording the + // dropped header, so every rerun retries it and reports the partial + // index rather than going silently clean. + const third = runClice("index", "--workspace", ws.root, "--workers", "2"); + expect(third.status, `stderr: ${third.stderr}`).toBe(1); + expect(third.stderr).toContain("stays unindexed"); + + // Only deleting the file settles the debt. + ws.rm("a.h"); + const fourth = runClice("index", "--workspace", ws.root, "--workers", "2"); + expect(fourth.status, `stderr: ${fourth.stderr}`).toBe(0); +}); diff --git a/tests/unit/index/tu_index_tests.cpp b/tests/unit/index/tu_index_tests.cpp index bec718d45..90ec6914e 100644 --- a/tests/unit/index/tu_index_tests.cpp +++ b/tests/unit/index/tu_index_tests.cpp @@ -240,6 +240,56 @@ TEST_CASE(FunctionTemplate) { GO_TO_DEFINITION("implicit_spec", "spec"); } +TEST_CASE(InstantiationLocalCollapse) { + build_index(R"( + struct Fn { + template + int operator()(T §(parm)x) const { + int §(local)loc = 1; + return §(local_ref)loc + static_cast(§(parm_ref)x); + } + }; + constexpr Fn fn{}; + int a = fn(1); + int b = fn(2.0); + )"); + + for(auto pos: {"parm", "local", "local_ref", "parm_ref"}) { + auto occurrences = select(pos); + std::set targets; + for(auto& occurrence: occurrences) { + targets.insert(occurrence.target); + } + EXPECT_EQ(targets.size(), 1U); + } +} + +TEST_CASE(LambdaCaptureCollapse) { + build_index(R"( + struct Fn { + template + int operator()(T §(parm)x) const { + auto lambda = [&] { + return static_cast(§(parm_ref)x); + }; + return lambda(); + } + }; + constexpr Fn fn{}; + int a = fn(1); + int b = fn(2.0); + )"); + + for(auto pos: {"parm", "parm_ref"}) { + auto occurrences = select(pos); + std::set targets; + for(auto& occurrence: occurrences) { + targets.insert(occurrence.target); + } + EXPECT_EQ(targets.size(), 1U); + } +} + TEST_CASE(AliasTemplate) { build_index(R"( template diff --git a/tests/unit/index/usr_tests.cpp b/tests/unit/index/usr_tests.cpp index f18616ac4..ec5a234c7 100644 --- a/tests/unit/index/usr_tests.cpp +++ b/tests/unit/index/usr_tests.cpp @@ -325,6 +325,14 @@ struct X { auto usr2 = tester.lookup("2"); ASSERT_NE(usr1, usr2); + + /// The dependent name is encoded as text: an AST-node address here + /// (IdentifierInfo streamed as a pointer) made every hash touching a + /// dependent template name unique per compile. + EXPECT_TRUE(usr1.contains(":apply")); + EXPECT_TRUE(usr2.contains(":apply2")); + EXPECT_FALSE(usr1.contains("0x")); + EXPECT_FALSE(usr2.contains("0x")); }; TEST_CASE(USRDeducingThis) { diff --git a/tests/unit/server/indexer_tests.cpp b/tests/unit/server/indexer_tests.cpp index e7a78e2ad..d1c9a7aae 100644 --- a/tests/unit/server/indexer_tests.cpp +++ b/tests/unit/server/indexer_tests.cpp @@ -13,6 +13,7 @@ #include "index/tu_index.h" #include "server/compiler/context_resolver.h" #include "server/compiler/indexer.h" +#include "server/state/config.h" #include "server/state/session_store.h" #include "server/state/workspace.h" #include "server/worker/worker_pool.h" @@ -88,6 +89,12 @@ struct IndexerFixture { indexer.global_dirty = false; } + /// Record a standalone header's borrowed host, as index_one's dispatch + /// does (these tests merge worker results directly). + void set_header_host(std::uint32_t header_id, std::uint32_t host_id) { + indexer.header_hosts[header_id] = host_id; + } + /// Run one save() to completion on the fixture's loop. void save() { auto body = [this]() -> kota::task<> { @@ -243,7 +250,7 @@ TEST_CASE(MergeRejectsGarbage) { ASSERT_TRUE(workspace.project_index.symbols.empty()); std::string garbage = "definitely not a flatbuffer, but long enough to try"; - indexer.merge(garbage.data(), garbage.size()); + ASSERT_FALSE(indexer.merge(garbage.data(), garbage.size())); ASSERT_TRUE(workspace.shards.empty()); ASSERT_TRUE(workspace.project_index.symbols.empty()); @@ -746,6 +753,30 @@ TEST_CASE(FailedWriteNotCounted) { ASSERT_EQ(indexer.pending_shard_writes(), 0u); } +TEST_CASE(WriteFailureStopsBatch) { + TempDir tmp; + open_store(tmp, workspace); + auto& storage = *workspace.index_storage; + + // Wedge the manifest's destination with a non-empty directory so its + // commit fails while the shard before it lands. + tmp.touch("cache/cache/v1/index-manifest/k.idx/wedge"); + + auto failures = storage.write({ + {index::IndexBlobKind::Shard, "k", "shard bytes" }, + {index::IndexBlobKind::Manifest, "k", "manifest bytes"}, + {index::IndexBlobKind::Global, "global", "global bytes" }, + }); + + // The failure fails the rest of the batch: a global must never land + // above a manifest that did not. + ASSERT_EQ(failures.size(), 2u); + ASSERT_EQ(failures[0], 1u); + ASSERT_EQ(failures[1], 2u); + ASSERT_TRUE(storage.contains(index::IndexBlobKind::Shard, "k")); + ASSERT_FALSE(storage.contains(index::IndexBlobKind::Global, "global")); +} + }; // TEST_SUITE(IndexerMerge) TEST_SUITE(IndexerStaleness) { @@ -906,6 +937,54 @@ TEST_CASE(LoadHealsBrokenShard) { ASSERT_FALSE(orphan_alive); } +TEST_CASE(ReadOnlyLoadKeepsDisk) { + TempDir tmp; + tmp.touch("dep.h", "#pragma once\ninline int dep() { return 1; }\n"); + tmp.touch("main.cpp", "#include \"dep.h\"\nint use() { return dep(); }\n"); + auto src = tmp.path("main.cpp"); + std::string header_key; + std::string manifest_key; + + { + IndexerFixture f; + open_store(tmp, f.workspace); + auto indexed = index_file(tmp, src); + ASSERT_FALSE(indexed.data.empty()); + f.indexer.merge(indexed.data.data(), indexed.data.size()); + f.save(); + header_key = blob_key( + f.workspace.path_pool.resolve(f.workspace.path_pool.intern(tmp.path("dep.h")))); + manifest_key = blob_key(f.workspace.path_pool.resolve(f.workspace.path_pool.intern(src))); + } + + tmp.touch("cache/cache/v1/index/" + header_key + ".idx", "corrupted beyond verification"); + tmp.touch("cache/cache/v1/index/deadbeefdeadbeef.idx", "orphan"); + + IndexerFixture f; + open_store(tmp, f.workspace); + f.indexer.load(/*read_only=*/true); + + // The in-memory sweeps still run: the unservable header drops its + // contributing TU's manifest and re-enqueues the TU. + auto tu_id = f.workspace.path_pool.intern(src); + ASSERT_TRUE(f.workspace.project_index.manifests.empty()); + ASSERT_TRUE(f.indexer.pending_reason(tu_id).has_value()); + + // But every blob survives on disk — a server running concurrently may + // still reference what this reader judged stale. + bool header_alive = false, orphan_alive = false, manifest_alive = false; + f.workspace.index_storage->for_each_key(index::IndexBlobKind::Shard, [&](llvm::StringRef key) { + header_alive |= key == header_key; + orphan_alive |= key == "deadbeefdeadbeef"; + }); + f.workspace.index_storage->for_each_key( + index::IndexBlobKind::Manifest, + [&](llvm::StringRef key) { manifest_alive |= key == manifest_key; }); + ASSERT_TRUE(header_alive); + ASSERT_TRUE(orphan_alive); + ASSERT_TRUE(manifest_alive); +} + TEST_CASE(LoadHealsMissingVariant) { TempDir tmp; tmp.touch("dep.h", "#pragma once\ninline int dep() { return 1; }\n"); @@ -1220,6 +1299,465 @@ TEST_CASE(DropIndexEvictsPersisted) { ASSERT_TRUE(f.need_update(src)); } +TEST_CASE(OfflineCommandChangeReindexed) { + TempDir tmp; + tmp.touch("main.cpp", "int value() { return 1; }\n"); + auto src = tmp.path("main.cpp"); + + { + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=1 -c main.cpp")); + auto indexed = index_file(tmp, src); + ASSERT_FALSE(indexed.data.empty()); + f.indexer.merge(indexed.data.data(), indexed.data.size()); + f.save(); + } + + // The command changed while no server ran: content freshness cannot + // see it, so the persisted CDB snapshot must catch it at load. + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=2 -c main.cpp")); + f.indexer.load(); + + auto tu_id = f.workspace.path_pool.intern(src); + ASSERT_FALSE(f.workspace.project_index.manifests.contains(tu_id)); + ASSERT_TRUE(f.indexer.pending_reason(tu_id) == ReindexReason::ContentChanged); +} + +TEST_CASE(UnchangedCommandKept) { + TempDir tmp; + tmp.touch("main.cpp", "int value() { return 1; }\n"); + auto src = tmp.path("main.cpp"); + + { + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=1 -c main.cpp")); + auto indexed = index_file(tmp, src); + ASSERT_FALSE(indexed.data.empty()); + f.indexer.merge(indexed.data.data(), indexed.data.size()); + f.save(); + } + + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=1 -c main.cpp")); + f.indexer.load(); + + auto tu_id = f.workspace.path_pool.intern(src); + ASSERT_TRUE(f.workspace.project_index.manifests.contains(tu_id)); + ASSERT_FALSE(f.indexer.pending_reason(tu_id).has_value()); +} + +TEST_CASE(RemovedEntryKeepsIndex) { + TempDir tmp; + tmp.touch("main.cpp", "int value() { return 1; }\n"); + auto src = tmp.path("main.cpp"); + + { + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=1 -c main.cpp")); + auto indexed = index_file(tmp, src); + ASSERT_FALSE(indexed.data.empty()); + f.indexer.merge(indexed.data.data(), indexed.data.size()); + f.save(); + } + + // The entry vanished from the CDB: the last-known rows still serve + // navigation, same conservative semantics as the live reload path. + IndexerFixture f; + open_store(tmp, f.workspace); + f.indexer.load(); + + auto tu_id = f.workspace.path_pool.intern(src); + ASSERT_TRUE(f.workspace.project_index.manifests.contains(tu_id)); + ASSERT_FALSE(f.indexer.pending_reason(tu_id).has_value()); +} + +TEST_CASE(RuleChangeReindexed) { + TempDir tmp; + tmp.touch("main.cpp", "int value() { return 1; }\n"); + auto src = tmp.path("main.cpp"); + + { + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -c main.cpp")); + auto indexed = index_file(tmp, src); + ASSERT_FALSE(indexed.data.empty()); + ASSERT_TRUE(f.indexer.merge(indexed.data.data(), indexed.data.size())); + f.save(); + } + + // The CDB entry is unchanged, but a clice.toml rule now adjusts the + // effective command — as invisible to content freshness as a command + // edit, so the snapshot must cover it too. + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -c main.cpp")); + f.workspace.config.rules.push_back(ConfigRule{ + .patterns = {"**/*.cpp"}, + .append = {"-DFOO=1"}, + }); + f.workspace.config.finalize(tmp.root); + f.indexer.load(); + + auto tu_id = f.workspace.path_pool.intern(src); + ASSERT_FALSE(f.workspace.project_index.manifests.contains(tu_id)); + ASSERT_TRUE(f.indexer.pending_reason(tu_id) == ReindexReason::ContentChanged); +} + +TEST_CASE(HostChangeDropsHeader) { + TempDir tmp; + tmp.touch("dep.h", "#pragma once\ninline int dep() { return 1; }\n"); + tmp.touch("main.cpp", "#include \"dep.h\"\nint use() { return dep(); }\n"); + auto src = tmp.path("main.cpp"); + auto header = tmp.path("dep.h"); + + { + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=1 -c main.cpp")); + // A standalone pass over the header, as a borrowed-context index + // produces it; the host source itself was never indexed. + auto indexed = index_file(tmp, header); + ASSERT_FALSE(indexed.data.empty()); + ASSERT_TRUE(f.indexer.merge(indexed.data.data(), indexed.data.size())); + f.save(); + } + + // The host's command changed while no server ran: the header has no + // CDB entry of its own, so only include reachability from the changed + // source can catch its borrowed-command manifest. + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=2 -c main.cpp")); + auto src_id = f.workspace.path_pool.intern(src); + auto header_id = f.workspace.path_pool.intern(header); + f.workspace.dep_graph.set_includes(src_id, 0, {header_id}); + f.workspace.dep_graph.build_reverse_map(); + f.indexer.load(); + + ASSERT_FALSE(f.workspace.project_index.manifests.contains(header_id)); + ASSERT_TRUE(f.indexer.pending_reason(header_id) == ReindexReason::ContentChanged); +} + +TEST_CASE(HeaderRuleChangeReindexed) { + TempDir tmp; + tmp.touch("dep.h", "#pragma once\ninline int dep() { return 1; }\n"); + auto header = tmp.path("dep.h"); + + { + IndexerFixture f; + open_store(tmp, f.workspace); + auto indexed = index_file(tmp, header); + ASSERT_FALSE(indexed.data.empty()); + ASSERT_TRUE(f.indexer.merge(indexed.data.data(), indexed.data.size())); + f.save(); + } + + // A clice.toml rule matching the header itself changed offline: the + // header has no CDB entry, so only its own snapshot entry can see it. + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.config.rules.push_back(ConfigRule{ + .patterns = {"**/*.h"}, + .append = {"-DFOO=1"}, + }); + f.workspace.config.finalize(tmp.root); + f.indexer.load(); + + auto header_id = f.workspace.path_pool.intern(header); + ASSERT_FALSE(f.workspace.project_index.manifests.contains(header_id)); + ASSERT_TRUE(f.indexer.pending_reason(header_id) == ReindexReason::ContentChanged); +} + +TEST_CASE(RecordedHostChangeDrops) { + TempDir tmp; + tmp.touch("dep.h", "#pragma once\ninline int dep() { return 1; }\n"); + tmp.touch("main.cpp", "int use() { return 0; }\n"); + auto src = tmp.path("main.cpp"); + auto header = tmp.path("dep.h"); + + { + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=1 -c main.cpp")); + auto indexed = index_file(tmp, header); + ASSERT_FALSE(indexed.data.empty()); + ASSERT_TRUE(f.indexer.merge(indexed.data.data(), indexed.data.size())); + f.set_header_host(f.workspace.path_pool.intern(header), f.workspace.path_pool.intern(src)); + f.save(); + } + + // The host's command changed offline AND the new command no longer + // includes the header, so the rebuilt include graph cannot reach it — + // only the recorded host association can catch the stale borrow. + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=2 -c main.cpp")); + f.indexer.load(); + + auto header_id = f.workspace.path_pool.intern(header); + ASSERT_FALSE(f.workspace.project_index.manifests.contains(header_id)); + ASSERT_TRUE(f.indexer.pending_reason(header_id) == ReindexReason::ContentChanged); +} + +TEST_CASE(PinnedHostKeepsHeader) { + TempDir tmp; + tmp.touch("dep.h", "#pragma once\ninline int dep() { return 1; }\n"); + tmp.touch("main.cpp", "#include \"dep.h\"\nint use() { return dep(); }\n"); + tmp.touch("other.cpp", "#include \"dep.h\"\nint more() { return dep(); }\n"); + auto src = tmp.path("main.cpp"); + auto other = tmp.path("other.cpp"); + auto header = tmp.path("dep.h"); + + { + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -c main.cpp")); + f.workspace.cdb.add_command(tmp.root, other, llvm::StringRef("clang++ -c other.cpp")); + auto indexed = index_file(tmp, header); + ASSERT_FALSE(indexed.data.empty()); + ASSERT_TRUE(f.indexer.merge(indexed.data.data(), indexed.data.size())); + f.set_header_host(f.workspace.path_pool.intern(header), f.workspace.path_pool.intern(src)); + f.save(); + } + + // Another includer's command changed, but the header borrowed the + // unchanged host's — the recorded association beats the reachability + // approximation's over-drop. + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -c main.cpp")); + f.workspace.cdb.add_command(tmp.root, other, llvm::StringRef("clang++ -DBAR=1 -c other.cpp")); + auto src_id = f.workspace.path_pool.intern(src); + auto other_id = f.workspace.path_pool.intern(other); + auto header_id = f.workspace.path_pool.intern(header); + f.workspace.dep_graph.set_includes(src_id, 0, {header_id}); + f.workspace.dep_graph.set_includes(other_id, 0, {header_id}); + f.workspace.dep_graph.build_reverse_map(); + f.indexer.load(); + + ASSERT_TRUE(f.workspace.project_index.manifests.contains(header_id)); + ASSERT_FALSE(f.indexer.pending_reason(header_id).has_value()); +} + +TEST_CASE(UnreachableHostRebuilds) { + TempDir tmp; + tmp.touch("dep.h", "#pragma once\ninline int dep() { return 1; }\n"); + tmp.touch("main.cpp", "int use() { return 0; }\n"); + auto src = tmp.path("main.cpp"); + auto header = tmp.path("dep.h"); + + { + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -c main.cpp")); + auto indexed = index_file(tmp, header); + ASSERT_FALSE(indexed.data.empty()); + ASSERT_TRUE(f.indexer.merge(indexed.data.data(), indexed.data.size())); + f.set_header_host(f.workspace.path_pool.intern(header), f.workspace.path_pool.intern(src)); + f.save(); + } + + // The host's command is untouched but an offline edit removed its + // include of the header: the rows keep serving while a queued rebuild + // re-selects a host. + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -c main.cpp")); + f.indexer.load(); + + auto header_id = f.workspace.path_pool.intern(header); + ASSERT_TRUE(f.workspace.project_index.manifests.contains(header_id)); + ASSERT_TRUE(f.indexer.pending_reason(header_id) == ReindexReason::ContentChanged); +} + +TEST_CASE(CdbWriteFailureRetried) { + TempDir tmp; + tmp.touch("main.cpp", "int value() { return 1; }\n"); + auto src = tmp.path("main.cpp"); + + // A storage that fails only the CDB snapshot blob, with everything + // else landing normally. + struct CdbFailingStorage final : index::IndexStorage { + std::unique_ptr real; + bool fail_cdb = true; + llvm::SmallVector written; + + std::unique_ptr read(index::IndexBlobKind kind, + llvm::StringRef key) override { + return real->read(kind, key); + } + + bool contains(index::IndexBlobKind kind, llvm::StringRef key) override { + return real->contains(kind, key); + } + + llvm::SmallVector write(llvm::ArrayRef batch) override { + llvm::SmallVector failed; + for(std::size_t i = 0; i < batch.size(); i += 1) { + written.push_back(batch[i].kind); + if(fail_cdb && batch[i].kind == index::IndexBlobKind::Cdb) { + failed.push_back(i); + } else if(!real->write(llvm::ArrayRef(batch[i])).empty()) { + failed.push_back(i); + } + } + return failed; + } + + void remove(index::IndexBlobKind kind, llvm::StringRef key) override { + real->remove(kind, key); + } + + void for_each_key(index::IndexBlobKind kind, + llvm::function_ref fn) override { + real->for_each_key(kind, fn); + } + }; + + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -c main.cpp")); + auto failing = std::make_unique(); + failing->real = std::move(f.workspace.index_storage); + auto* storage = failing.get(); + f.workspace.index_storage = std::move(failing); + + auto indexed = index_file(tmp, src); + ASSERT_FALSE(indexed.data.empty()); + ASSERT_TRUE(f.indexer.merge(indexed.data.data(), indexed.data.size())); + f.save(); + + // The snapshot rides the batch behind the index state it describes. + ASSERT_EQ(int(storage->written.back()), int(index::IndexBlobKind::Cdb)); + ASSERT_FALSE(storage->real->contains(index::IndexBlobKind::Cdb, "cdb")); + + // Nothing else is dirty any more, yet the failed snapshot alone must + // drive the next save until it lands — and until it does, the state + // counts as unsaved (`clice index` fails on it after the final save). + ASSERT_TRUE(f.indexer.has_unsaved_state()); + storage->fail_cdb = false; + f.save(); + ASSERT_TRUE(storage->real->contains(index::IndexBlobKind::Cdb, "cdb")); + ASSERT_FALSE(f.indexer.has_unsaved_state()); +} + +TEST_CASE(MissingSnapshotRewritten) { + TempDir tmp; + tmp.touch("main.cpp", "int value() { return 1; }\n"); + auto src = tmp.path("main.cpp"); + + { + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -c main.cpp")); + auto indexed = index_file(tmp, src); + ASSERT_FALSE(indexed.data.empty()); + ASSERT_TRUE(f.indexer.merge(indexed.data.data(), indexed.data.size())); + f.save(); + // The global landed but the final CDB write never did: the rest of + // the index is intact. + f.workspace.index_storage->remove(index::IndexBlobKind::Cdb, "cdb"); + } + + // A rerun that dirties nothing must still recreate the baseline — + // without it, every later offline command edit would go undetected. + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -c main.cpp")); + f.indexer.load(); + ASSERT_TRUE(f.indexer.has_unsaved_state()); + f.save(); + ASSERT_TRUE(f.workspace.index_storage->contains(index::IndexBlobKind::Cdb, "cdb")); + ASSERT_FALSE(f.indexer.has_unsaved_state()); +} + +TEST_CASE(DroppedHeaderDebtRetried) { + TempDir tmp; + tmp.touch("dep.h", "#pragma once\ninline int dep() { return 1; }\n"); + tmp.touch("main.cpp", "int use() { return 0; }\n"); + auto src = tmp.path("main.cpp"); + auto header = tmp.path("dep.h"); + + { + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=1 -c main.cpp")); + auto indexed = index_file(tmp, header); + ASSERT_FALSE(indexed.data.empty()); + ASSERT_TRUE(f.indexer.merge(indexed.data.data(), indexed.data.size())); + f.set_header_host(f.workspace.path_pool.intern(header), f.workspace.path_pool.intern(src)); + f.save(); + } + + { + // The host's command changed offline: the header's index is dropped + // and queued — but the rebuild never lands this session. The saved + // snapshot must keep recording the header, or nothing would ever + // retry it. + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=2 -c main.cpp")); + f.indexer.load(); + auto header_id = f.workspace.path_pool.intern(header); + ASSERT_FALSE(f.workspace.project_index.manifests.contains(header_id)); + ASSERT_TRUE(f.indexer.pending_reason(header_id) == ReindexReason::ContentChanged); + f.save(); + } + + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=2 -c main.cpp")); + f.indexer.load(); + + auto header_id = f.workspace.path_pool.intern(header); + ASSERT_FALSE(f.workspace.project_index.manifests.contains(header_id)); + ASSERT_TRUE(f.indexer.pending_reason(header_id) == ReindexReason::ContentChanged); +} + +TEST_CASE(VanishedHeaderDebtDies) { + TempDir tmp; + tmp.touch("dep.h", "#pragma once\ninline int dep() { return 1; }\n"); + tmp.touch("main.cpp", "int use() { return 0; }\n"); + auto src = tmp.path("main.cpp"); + auto header = tmp.path("dep.h"); + + { + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=1 -c main.cpp")); + auto indexed = index_file(tmp, header); + ASSERT_FALSE(indexed.data.empty()); + ASSERT_TRUE(f.indexer.merge(indexed.data.data(), indexed.data.size())); + f.set_header_host(f.workspace.path_pool.intern(header), f.workspace.path_pool.intern(src)); + f.save(); + } + + { + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=2 -c main.cpp")); + f.indexer.load(); + f.save(); + } + + // The header was deleted while its debt entry sat in the snapshot: a + // retry could never succeed, so the debt dies instead of keeping every + // later run partial forever. + ASSERT_TRUE(!llvm::sys::fs::remove(header)); + IndexerFixture f; + open_store(tmp, f.workspace); + f.workspace.cdb.add_command(tmp.root, src, llvm::StringRef("clang++ -DFOO=2 -c main.cpp")); + f.indexer.load(); + ASSERT_FALSE(f.indexer.pending_reason(f.workspace.path_pool.intern(header)).has_value()); +} + }; // TEST_SUITE(IndexerLoad) TEST_SUITE(IndexerRequeue) { diff --git a/tests/unit/support/cache_store_tests.cpp b/tests/unit/support/cache_store_tests.cpp index 60f9eb25b..630c47c50 100644 --- a/tests/unit/support/cache_store_tests.cpp +++ b/tests/unit/support/cache_store_tests.cpp @@ -175,6 +175,29 @@ TEST_CASE(VersionBumpDiscards) { ASSERT_FALSE(llvm::sys::fs::exists(tmp.path("root/cache/v1"))); } +TEST_CASE(LiveLayoutKept) { + TempDir tmp; + // A clice of another version is live inside v1 (our own pid stands in + // for its `tmp/{pid}` marker): the sweep must not delete the cache out + // from under it. + auto self = std::to_string(llvm::sys::Process::getProcessId()); + tmp.touch("root/cache/v1/tmp/" + self + "/0.pch"); + tmp.touch("root/cache/v1/pch/k1.pch", "live"); + + auto store = open_store(tmp, version + 1); + ASSERT_TRUE(llvm::sys::fs::exists(tmp.path("root/cache/v1/pch/k1.pch"))); +} + +TEST_CASE(CrashedLayoutDiscarded) { + TempDir tmp; + // A crashed old-version instance leaves its `tmp/{pid}` behind; a dead + // pid does not hold the layout alive. + tmp.touch(std::string("root/cache/v1/tmp/") + dead_pid + "/0.pch"); + + auto store = open_store(tmp, version + 1); + ASSERT_FALSE(llvm::sys::fs::exists(tmp.path("root/cache/v1"))); +} + TEST_CASE(LegacyLayoutDiscarded) { TempDir tmp; // Pre-versioning layout: blobs and metadata directly under cache/. @@ -654,6 +677,65 @@ TEST_CASE(InvalidateRemovesPair) { ASSERT_FALSE(fs::read(aux_path).has_value()); } +TEST_CASE(ReadOnlyOpenRequiresStore) { + TempDir tmp; + // The parent directory alone (config resolution creates it eagerly) + // must not pass for an existing store. + tmp.touch("root/cache/marker", ""); + auto store = CacheStore::open(tmp.path("root"), version, /*read_only=*/true); + ASSERT_FALSE(store.has_value()); + ASSERT_TRUE(store.error() == std::errc::no_such_file_or_directory); +} + +TEST_CASE(ReadOnlyOpenTouchesNothing) { + TempDir tmp; + { + auto store = open_store(tmp); + register_lru(store); + put(store, "pch", "k1", "blob"); + store.shutdown(); + } + + // A newer-version inspector must neither create its own directory nor + // sweep the older version a live server may still be using. + auto store = CacheStore::open(tmp.path("root"), version + 1, /*read_only=*/true); + ASSERT_FALSE(store.has_value()); + ASSERT_FALSE(llvm::sys::fs::exists(tmp.path("root/cache/v2"))); + + auto reader = CacheStore::open(tmp.path("root"), version, /*read_only=*/true); + ASSERT_TRUE(reader.has_value()); + register_lru(*reader); + ASSERT_TRUE(reader->lookup("pch", "k1").has_value()); + auto pid = std::to_string(llvm::sys::Process::getProcessId()); + ASSERT_FALSE(llvm::sys::fs::exists(tmp.path("root/cache/v1/tmp/" + pid))); +} + +TEST_CASE(ReadOnlyNeverWrites) { + TempDir tmp; + { + auto store = open_store(tmp); + register_lru(store); + put(store, "pch", "k1", "blob"); + store.shutdown(); + } + auto manifest_before = fs::read(tmp.path("root/cache/v1/manifest.json")); + ASSERT_TRUE(manifest_before.has_value()); + + auto reader = CacheStore::open(tmp.path("root"), version, /*read_only=*/true); + ASSERT_TRUE(reader.has_value()); + // A budget below the blob size must not evict at registration, an + // invalidate must not delete the live server's blob, and the atime + // bumps from lookups must not publish a manifest at shutdown — with + // no tmp dir it would even be staged in the working directory. + register_lru(*reader, 1); + ASSERT_TRUE(reader->lookup("pch", "k1").has_value()); + reader->invalidate("pch", "k1"); + reader->shutdown(); + + ASSERT_EQ(fs::read(tmp.path("root/cache/v1/pch/k1.pch")).value_or(""), "blob"); + ASSERT_EQ(fs::read(tmp.path("root/cache/v1/manifest.json")).value_or(""), *manifest_before); +} + }; // TEST_SUITE(CacheStore) } // namespace