feat: add CompileGraph for pull-based module dependency compilation

CMakeLists.txt

@@ -172,6 +172,7 @@ add_library(clice-core STATIC
     "${PROJECT_SOURCE_DIR}/src/server/stateless_worker.cpp"
     "${PROJECT_SOURCE_DIR}/src/server/stateful_worker.cpp"
     "${PROJECT_SOURCE_DIR}/src/server/worker_pool.cpp"
+    "${PROJECT_SOURCE_DIR}/src/server/compile_graph.cpp"
     "${PROJECT_SOURCE_DIR}/src/server/master_server.cpp"
     "${PROJECT_SOURCE_DIR}/src/server/config.cpp"
 )

src/server/compile_graph.cpp

@@ -0,0 +1,199 @@
+#include "server/compile_graph.h"
+
+#include <algorithm>
+
+#include "llvm/ADT/DenseSet.h"
+
+namespace clice {
+
+CompileGraph::CompileGraph(dispatch_fn dispatch, resolve_fn resolve) :
+    dispatch(std::move(dispatch)), resolve(std::move(resolve)) {}
+
+void CompileGraph::ensure_resolved(std::uint32_t path_id) {
+    auto& unit = units[path_id];
+    if(unit.resolved) {
+        return;
+    }
+
+    unit.path_id = path_id;
+    unit.resolved = true;
+    unit.dependencies = resolve(path_id);
+
+    // Copy deps locally — the loop below may insert into `units`,
+    // which can rehash the DenseMap and invalidate the `unit` reference.
+    auto deps = units[path_id].dependencies;
+
+    // Back-populate dependents.
+    for(auto dep_id: deps) {
+        auto& dep = units[dep_id];
+        dep.path_id = dep_id;
+        dep.dependents.push_back(path_id);
+    }
+}
+
+et::task<bool> CompileGraph::compile(std::uint32_t path_id) {
+    llvm::DenseSet<std::uint32_t> ancestors;
+    co_return co_await compile_impl(path_id, ancestors);
+}
+
+et::task<bool> CompileGraph::compile_impl(std::uint32_t path_id,
+                                          llvm::DenseSet<std::uint32_t> ancestors) {
+    ensure_resolved(path_id);
+
+    // Cycle detection: if this unit is already in the compile chain, bail out.
+    if(!ancestors.insert(path_id).second) {
+        co_return false;
+    }
+
+    // Re-lookup after ensure_resolved may have mutated the map.
+    auto it = units.find(path_id);
+
+    // Already clean.
+    if(!it->second.dirty) {
+        co_return true;
+    }
+
+    // Another task is already compiling this unit — wait for it.
+    if(it->second.compiling) {
+        auto& completion = *it->second.completion;
+        co_await completion.wait();
+        co_return !units.find(path_id)->second.dirty;
+    }
+
+    // Begin compilation.
+    it->second.compiling = true;
+    it->second.completion = std::make_unique<et::event>();
+
+    // Copy deps and token before co_await (DenseMap iterator safety).
+    auto deps = it->second.dependencies;
+    auto token = it->second.source->token();
+
+    // Compile all dependencies in parallel.
+    if(!deps.empty()) {
+        std::vector<et::task<bool, void, et::cancellation>> dep_tasks;
+        dep_tasks.reserve(deps.size());
+        for(auto dep_id: deps) {
+            dep_tasks.push_back(et::with_token(compile_impl(dep_id, ancestors), token));
+        }
+
+        auto results = co_await et::when_all(std::move(dep_tasks));
+
+        auto& u = units.find(path_id)->second;
+        if(results.is_cancelled()) {
+            u.compiling = false;
+            u.completion->set();
+            co_await et::cancel();
+        }
+
+        for(auto ok: *results) {
+            if(!ok) {
+                u.compiling = false;
+                u.completion->set();
+                co_return false;
+            }
+        }
+    }
+
+    // Dispatch the actual compilation, cancellable via the pre-captured token.
+    // Using the token captured before co_await ensures cancellation propagates
+    // correctly even if update() replaces the source during dependency compilation.
+    {
+        auto result = co_await et::with_token(dispatch(path_id), token);
+
+        auto& u = units.find(path_id)->second;
+        if(!result.has_value()) {
+            u.compiling = false;
+            u.completion->set();
+            co_await et::cancel();
+        }
+        if(!*result) {
+            u.compiling = false;
+            u.completion->set();
+            co_return false;
+        }
+    }
+
+    // Success.
+    auto& final_unit = units.find(path_id)->second;
+    final_unit.dirty = false;
+    final_unit.compiling = false;
+    final_unit.completion->set();
+    co_return true;
+}
+
+llvm::SmallVector<std::uint32_t> CompileGraph::update(std::uint32_t path_id) {
+    llvm::SmallVector<std::uint32_t> queue;
+    llvm::SmallVector<std::uint32_t> dirtied;
+    queue.push_back(path_id);
+
+    // Track visited nodes to avoid processing the same node twice.
+    llvm::DenseSet<std::uint32_t> visited;
+
+    while(!queue.empty()) {
+        auto current = queue.pop_back_val();
+
+        if(!visited.insert(current).second) {
+            continue;
+        }
+
+        auto it = units.find(current);
+        if(it == units.end()) {
+            continue;
+        }
+
+        auto& unit = it->second;
+
+        // Reset resolved so dependencies are re-scanned on next compile
+        // (the source file may have added/removed imports).
+        if(current == path_id) {
+            unit.resolved = false;
+            // Clear stale dependency edges — they'll be rebuilt by ensure_resolved.
+            for(auto dep_id: unit.dependencies) {
+                auto dep_it = units.find(dep_id);
+                if(dep_it != units.end()) {
+                    auto& dependents = dep_it->second.dependents;
+                    dependents.erase(std::remove(dependents.begin(), dependents.end(), path_id),
+                                     dependents.end());
+                }
+            }
+            unit.dependencies.clear();
+        }
+
+        // Cancel in-flight compilation if running.
+        if(unit.compiling) {
+            unit.source->cancel();
+            unit.source = std::make_unique<et::cancellation_source>();
+        }
+        unit.dirty = true;
+        dirtied.push_back(current);
+
+        // Always propagate to dependents.
+        for(auto dep_id: unit.dependents) {
+            queue.push_back(dep_id);
+        }
+    }
+
+    return dirtied;
+}
+
+void CompileGraph::cancel_all() {
+    for(auto& [_, unit]: units) {
+        unit.source->cancel();
+    }
+}
+
+bool CompileGraph::has_unit(std::uint32_t path_id) const {
+    return units.count(path_id);
+}
+
+bool CompileGraph::is_dirty(std::uint32_t path_id) const {
+    auto it = units.find(path_id);
+    return it != units.end() && it->second.dirty;
+}
+
+bool CompileGraph::is_compiling(std::uint32_t path_id) const {
+    auto it = units.find(path_id);
+    return it != units.end() && it->second.compiling;
+}
+
+}  // namespace clice

src/server/compile_graph.h

@@ -0,0 +1,72 @@
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+
+#include "eventide/async/async.h"
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace clice {
+
+namespace et = eventide;
+
+struct CompileUnit {
+    std::uint32_t path_id = 0;
+
+    /// Dependencies discovered lazily by resolve_fn.
+    llvm::SmallVector<std::uint32_t> dependencies;
+
+    /// Back-edges: units that depend on this unit.
+    llvm::SmallVector<std::uint32_t> dependents;
+
+    /// Whether resolve_fn has been called for this unit.
+    bool resolved = false;
+
+    bool dirty = true;
+    bool compiling = false;
+
+    std::unique_ptr<et::cancellation_source> source = std::make_unique<et::cancellation_source>();
+    std::unique_ptr<et::event> completion;
+};
+
+class CompileGraph {
+public:
+    /// Performs the actual compilation (e.g. produce PCM file).
+    using dispatch_fn = std::function<et::task<bool>(std::uint32_t path_id)>;
+
+    /// Returns the dependency path_ids for a given path_id (called lazily on first compile).
+    using resolve_fn = std::function<llvm::SmallVector<std::uint32_t>(std::uint32_t path_id)>;
+
+    CompileGraph(dispatch_fn dispatch, resolve_fn resolve);
+
+    /// Compile a unit and all its transitive dependencies.
+    et::task<bool> compile(std::uint32_t path_id);
+
+    /// Mark path_id and all transitive dependents as dirty,
+    /// cancelling any in-progress compilations.
+    /// Returns the set of all path_ids that were marked dirty.
+    llvm::SmallVector<std::uint32_t> update(std::uint32_t path_id);
+
+    void cancel_all();
+
+    bool has_unit(std::uint32_t path_id) const;
+    bool is_dirty(std::uint32_t path_id) const;
+    bool is_compiling(std::uint32_t path_id) const;
+
+private:
+    /// Get or create a unit, resolving its dependencies if needed.
+    void ensure_resolved(std::uint32_t path_id);
+
+    /// Internal compile with ancestor tracking for cycle detection.
+    et::task<bool> compile_impl(std::uint32_t path_id, llvm::DenseSet<std::uint32_t> ancestors);
+
+    dispatch_fn dispatch;
+    resolve_fn resolve;
+    llvm::DenseMap<std::uint32_t, CompileUnit> units;
+};
+
+}  // namespace clice