refactor: adopt LineMap, remove position wrapper layers

cmake/package.cmake

@@ -41,7 +41,7 @@ set(FLATBUFFERS_BUILD_FLATHASH OFF CACHE BOOL "" FORCE)
 FetchContent_Declare(
     kotatsu
     GIT_REPOSITORY https://github.com/clice-io/kotatsu
-    GIT_TAG b4bcd3c7e011812a345b4c520ccc77a6ed09174b
+    GIT_TAG 0b38494
 )
 
 set(KOTA_ENABLE_ZEST ON)

src/compile/compilation_unit.cpp

@@ -2,6 +2,8 @@
 #include "index/usr.h"
 #include "semantic/ast_utility.h"
 
+#include "kota/ipc/lsp/text.h"
+
 namespace clice {
 
 CompilationKind CompilationUnitRef::kind() {
@@ -125,6 +127,15 @@ auto CompilationUnitRef::interested_content() -> llvm::StringRef {
     return file_content(interested_file());
 }
 
+auto CompilationUnitRef::line_starts() -> std::span<const std::uint32_t> {
+    if(self->line_starts_cache.empty()) {
+        auto content = interested_content();
+        self->line_starts_cache =
+            kota::ipc::lsp::build_line_starts({content.data(), content.size()});
+    }
+    return self->line_starts_cache;
+}
+
 bool CompilationUnitRef::is_builtin_file(clang::FileID fid) {
     // No FileEntryRef => built-in/command line/scratch.
     if(!self->SM().getFileEntryRefForID(fid)) {

src/compile/compilation_unit.h

@@ -2,6 +2,7 @@
 
 #include <chrono>
 #include <cstdint>
+#include <span>
 #include <string>
 #include <utility>
 #include <vector>
@@ -138,6 +139,10 @@ class CompilationUnitRef {
     /// Get the content of interested file.
     auto interested_content() -> llvm::StringRef;
 
+    /// Get the byte offsets of each line start in the interested file.
+    /// Lazily computed and cached.
+    auto line_starts() -> std::span<const std::uint32_t>;
+
     /// Check if a file is a builtin file.
     bool is_builtin_file(clang::FileID fid);
 

src/compile/implement.h

@@ -85,6 +85,9 @@ struct CompilationUnitRef::Self {
     /// Cache for symbol id.
     llvm::DenseMap<const void*, std::uint64_t> symbol_hash_cache;
 
+    /// Cache for line starts of the interested file.
+    std::vector<std::uint32_t> line_starts_cache;
+
     llvm::BumpPtrAllocator path_storage;
 
     std::vector<Diagnostic> diagnostics;

src/feature/code_completion.cpp

@@ -255,11 +255,8 @@ class CodeCompletionCollector final : public clang::CodeCompleteConsumer {
         auto prefix = CompletionPrefix::from(content, offset);
         FuzzyMatcher matcher(prefix.spelling);
 
-        PositionMapper converter(content, encoding);
-        auto replace_range = protocol::Range{
-            .start = *converter.to_position(prefix.range.begin),
-            .end = *converter.to_position(prefix.range.end),
-        };
+        LineMap map(content, encoding);
+        auto replace_range = *map.to_range(prefix.range.begin, prefix.range.end);
 
         std::vector<protocol::CompletionItem> collected;
         collected.reserve(candidate_count);

src/feature/diagnostics.cpp

@@ -10,8 +10,6 @@ namespace clice::feature {
 
 namespace {
 
-namespace lsp = kota::ipc::lsp;
-
 auto to_uri(llvm::StringRef file) -> std::string {
     const auto file_view = std::string_view(file.data(), file.size());
 
@@ -49,13 +47,13 @@ void add_related(protocol::Diagnostic& diagnostic,
     }
 
     auto content = unit.file_content(raw.fid);
-    PositionMapper converter(content, encoding);
+    LineMap map(content, encoding);
 
     protocol::DiagnosticRelatedInformation related{
         .location =
             protocol::Location{
                                .uri = to_uri(unit.file_path(raw.fid)),
-                               .range = to_range(converter, raw.range),
+                               .range = *map.to_range(raw.range.begin, raw.range.end),
                                },
         .message = raw.message,
     };
@@ -70,6 +68,7 @@ void add_related(protocol::Diagnostic& diagnostic,
 
 auto diagnostics(CompilationUnitRef unit, PositionEncoding encoding)
     -> std::vector<protocol::Diagnostic> {
+    LineMap map(unit.interested_content(), unit.line_starts(), encoding);
     std::vector<protocol::Diagnostic> result;
     std::optional<protocol::Diagnostic> current;
 
@@ -80,8 +79,6 @@ auto diagnostics(CompilationUnitRef unit, PositionEncoding encoding)
         }
     };
 
-    PositionMapper main_converter(unit.interested_content(), encoding);
-
     for(const auto& raw: unit.diagnostics()) {
         auto level = raw.id.level;
 
@@ -136,7 +133,7 @@ auto diagnostics(CompilationUnitRef unit, PositionEncoding encoding)
         }
 
         if(raw.fid == unit.interested_file()) {
-            diagnostic.range = to_range(main_converter, raw.range);
+            diagnostic.range = *map.to_range(raw.range.begin, raw.range.end);
             current = std::move(diagnostic);
             continue;
         }
@@ -152,11 +149,9 @@ auto diagnostics(CompilationUnitRef unit, PositionEncoding encoding)
         }
 
         auto offset = unit.file_offset(include_location);
-        auto end_offset = offset + unit.token_spelling(include_location).size();
-        diagnostic.range = protocol::Range{
-            .start = *main_converter.to_position(offset),
-            .end = *main_converter.to_position(end_offset),
-        };
+        auto end_offset =
+            static_cast<std::uint32_t>(offset + unit.token_spelling(include_location).size());
+        diagnostic.range = *map.to_range(offset, end_offset);
 
         current = std::move(diagnostic);
     }

src/feature/document_links.cpp

@@ -18,9 +18,9 @@ auto document_links(CompilationUnitRef unit, PositionEncoding encoding)
     }
 
     auto content = unit.interested_content();
-    PositionMapper converter(content, encoding);
     auto& directives = directives_it->second;
     auto* lang_opts = &unit.lang_options();
+    LineMap map(content, unit.line_starts(), encoding);
 
     auto add_link = [&](clang::SourceLocation loc, llvm::StringRef target) {
         auto [fid, offset] = unit.decompose_location(loc);
@@ -29,7 +29,7 @@ auto document_links(CompilationUnitRef unit, PositionEncoding encoding)
         auto range = find_directive_argument(content, offset, lang_opts);
         if(!range)
             return;
-        protocol::DocumentLink link{.range = to_range(converter, *range)};
+        protocol::DocumentLink link{.range = *map.to_range(range->begin, range->end)};
         link.target = target.str();
         links.push_back(std::move(link));
     };

src/feature/document_symbols.cpp

@@ -178,13 +178,13 @@ void sort_symbols(std::vector<DocumentSymbol>& symbols) {
     }
 }
 
-auto to_protocol_symbol(const DocumentSymbol& symbol, const PositionMapper& converter)
+auto to_protocol_symbol(const DocumentSymbol& symbol, const LineMap& map)
     -> protocol::DocumentSymbol {
     protocol::DocumentSymbol result{
         .name = symbol.name,
         .kind = to_protocol_symbol_kind(symbol.kind),
-        .range = to_range(converter, symbol.range),
-        .selection_range = to_range(converter, symbol.selection_range),
+        .range = *map.to_range(symbol.range.begin, symbol.range.end),
+        .selection_range = *map.to_range(symbol.selection_range.begin, symbol.selection_range.end),
     };
 
     if(!symbol.detail.empty()) {
@@ -196,7 +196,7 @@ auto to_protocol_symbol(const DocumentSymbol& symbol, const PositionMapper& conv
         children.reserve(symbol.children.size());
         for(const auto& child: symbol.children) {
             children.push_back(
-                std::make_shared<protocol::DocumentSymbol>(to_protocol_symbol(child, converter)));
+                std::make_shared<protocol::DocumentSymbol>(to_protocol_symbol(child, map)));
         }
         result.children = std::move(children);
     }
@@ -215,13 +215,13 @@ auto document_symbols(CompilationUnitRef unit) -> std::vector<DocumentSymbol> {
 auto document_symbols(CompilationUnitRef unit, PositionEncoding encoding)
     -> std::vector<protocol::DocumentSymbol> {
     auto internal = document_symbols(unit);
+    LineMap map(unit.interested_content(), unit.line_starts(), encoding);
 
-    PositionMapper converter(unit.interested_content(), encoding);
     std::vector<protocol::DocumentSymbol> symbols;
     symbols.reserve(internal.size());
 
     for(const auto& symbol: internal) {
-        symbols.push_back(to_protocol_symbol(symbol, converter));
+        symbols.push_back(to_protocol_symbol(symbol, map));
     }
 
     return symbols;

src/feature/feature.h

@@ -15,18 +15,14 @@
 
 namespace clice::feature {
 
+namespace lsp = kota::ipc::lsp;
 namespace protocol = kota::ipc::protocol;
 
+using kota::ipc::lsp::LineMap;
 using kota::ipc::lsp::PositionEncoding;
-using kota::ipc::lsp::PositionMapper;
-using kota::ipc::lsp::parse_position_encoding;
 
-inline auto to_range(const PositionMapper& converter, LocalSourceRange range) -> protocol::Range {
-    return protocol::Range{
-        .start = *converter.to_position(range.begin),
-        .end = *converter.to_position(range.end),
-    };
-}
+// FIXME: feature code uses *map.to_range() without checking the optional.
+// Need a strategy for handling out-of-range offsets gracefully.
 
 struct CodeCompletionOptions {
     bool enable_keyword_snippet = false;

src/feature/folding_ranges.cpp

@@ -349,14 +349,13 @@ auto folding_ranges(CompilationUnitRef unit) -> std::vector<FoldingRange> {
 auto folding_ranges(CompilationUnitRef unit, PositionEncoding encoding)
     -> std::vector<protocol::FoldingRange> {
     auto collected = folding_ranges(unit);
-    PositionMapper converter(unit.interested_content(), encoding);
+    LineMap map(unit.interested_content(), unit.line_starts(), encoding);
 
     std::vector<protocol::FoldingRange> result;
     result.reserve(collected.size());
 
     for(const auto& item: collected) {
-        auto start = *converter.to_position(item.range.begin);
-        auto end = *converter.to_position(item.range.end);
+        auto [start, end] = *map.to_range(item.range.begin, item.range.end);
 
         protocol::FoldingRange range{
             .start_line = start.line,

src/feature/formatting.cpp

@@ -53,13 +53,15 @@ auto document_format(llvm::StringRef file,
         return edits;
     }
 
-    PositionMapper converter(content, encoding);
+    LineMap map(content, encoding);
 
     for(const auto& replacement: *replacements) {
-        protocol::TextEdit edit;
-        edit.range.start = *converter.to_position(replacement.getOffset());
-        edit.range.end = *converter.to_position(replacement.getOffset() + replacement.getLength());
-        edit.new_text = replacement.getReplacementText().str();
+        auto begin = replacement.getOffset();
+        auto end = begin + replacement.getLength();
+        protocol::TextEdit edit{
+            .range = *map.to_range(begin, end),
+            .new_text = replacement.getReplacementText().str(),
+        };
         edits.push_back(std::move(edit));
     }
 

src/feature/hover.cpp

@@ -1310,8 +1310,8 @@ auto to_protocol_hover(CompilationUnitRef unit,
     };
 
     if(info.symbol_range) {
-        PositionMapper converter(unit.interested_content(), encoding);
-        result.range = to_range(converter, *info.symbol_range);
+        LineMap map(unit.interested_content(), unit.line_starts(), encoding);
+        result.range = *map.to_range(info.symbol_range->begin, info.symbol_range->end);
     }
 
     return result;

src/feature/inlay_hints.cpp

@@ -928,14 +928,14 @@ auto inlay_hints(CompilationUnitRef unit,
                  const InlayHintsOptions& options,
                  PositionEncoding encoding) -> std::vector<protocol::InlayHint> {
     auto collected = inlay_hints(unit, target, options);
+    LineMap map(unit.interested_content(), unit.line_starts(), encoding);
 
-    PositionMapper converter(unit.interested_content(), encoding);
     std::vector<protocol::InlayHint> hints;
     hints.reserve(collected.size());
 
     for(const auto& hint: collected) {
         protocol::InlayHint out{
-            .position = *converter.to_position(hint.offset),
+            .position = *map.to_position(hint.offset),
             .label = hint.label,
         };
 

src/feature/semantic_tokens.cpp

@@ -479,21 +479,23 @@ class SemanticTokensCollector : public SemanticVisitor<SemanticTokensCollector>
 
 class SemanticTokenEncoder {
 public:
-    SemanticTokenEncoder(llvm::StringRef content,
+    SemanticTokenEncoder(CompilationUnitRef unit,
                          PositionEncoding encoding,
                          protocol::SemanticTokens& output) :
-        content(content), converter(content, encoding), output(output) {}
+        map(unit.interested_content(), unit.line_starts(), encoding), encoding(encoding),
+        output(output) {}
 
     void append(const SemanticToken& token) {
+        auto content = map.content();
         if(!token.range.valid() || token.range.end <= token.range.begin ||
            token.range.end > content.size()) {
             return;
         }
 
         auto begin = token.range.begin;
         auto end = token.range.end;
-        auto begin_position = *converter.to_position(begin);
-        auto end_position = *converter.to_position(end);
+        auto begin_position = *map.to_position(begin);
+        auto end_position = *map.to_position(end);
         auto begin_line = static_cast<std::uint32_t>(begin_position.line);
         auto begin_char = static_cast<std::uint32_t>(begin_position.character);
         auto end_line = static_cast<std::uint32_t>(end_position.line);
@@ -524,15 +526,15 @@ class SemanticTokenEncoder {
                     first_piece = false;
                 }
 
-                auto length = converter.measure(chunk.substr(chunk_offset, piece_size));
+                auto length = lsp::encoded_length(chunk.substr(chunk_offset, piece_size), encoding);
                 emit_relative(delta_line, delta_start, length, token.kind, token.modifiers);
 
                 chunk_offset += piece_size;
                 piece_size = 0;
             }
 
             if(piece_size > 0) {
-                auto length = converter.measure(chunk.substr(chunk_offset));
+                auto length = lsp::encoded_length(chunk.substr(chunk_offset), encoding);
                 emit_relative(1, 0, length, token.kind, token.modifiers);
             }
         }
@@ -559,8 +561,8 @@ class SemanticTokenEncoder {
     }
 
 private:
-    llvm::StringRef content;
-    PositionMapper converter;
+    lsp::LineMap map;
+    PositionEncoding encoding;
     protocol::SemanticTokens& output;
     std::uint32_t last_line = 0;
     std::uint32_t last_start_character = 0;
@@ -580,7 +582,7 @@ auto semantic_tokens(CompilationUnitRef unit, PositionEncoding encoding)
     protocol::SemanticTokens result;
     result.data.reserve(tokens.size() * 5);
 
-    SemanticTokenEncoder encoder(unit.interested_content(), encoding, result);
+    SemanticTokenEncoder encoder(unit, encoding, result);
     for(const auto& token: tokens) {
         encoder.append(token);
     }