Merge pull request #89 from sminez/issue-76

Fix: multiple tree-sitter highlighting bugs

Author: sminez

Cargo.lock

@@ -54,3 +54,5 @@ toml = "0.8.19"
 [dev-dependencies]
 simple_test_case = "1.2.0"
 criterion = "0.5"
+tree-sitter-python = "0.23.6"
+tree-sitter-rust = "0.23.2"

Cargo.toml

@@ -72,6 +72,7 @@ function = { fg = "#957FB8" }
 keyword = { fg = "#Bf616A" }
 module = { fg = "#2D4F67" }
 number = { fg = "#D27E99" }
+operator = { fg = "#E6C384" }
 punctuation = { fg = "#9CABCA" }
 string = { fg = "#61DCA5" }
 type = { fg = "#7E9CD8" }

data/config.toml

@@ -210,4 +210,3 @@
   "?."
   "?"
 ] @punctuation.delimiter
-

data/tree-sitter/queries/dart/highlights.scm

@@ -57,6 +57,9 @@
 
 (comment) @comment
 
+((comment) @comment.documentation
+  (#match? @comment.documentation "^/[*][*][^*].*[*]/$"))
+
 [
   "("
   ")"

data/tree-sitter/queries/proto/highlights.scm

@@ -1,3 +1,5 @@
+(shebang) @keyword.directive
+
 (function_item (identifier) @function)
 (function_signature_item (identifier) @function)
 (macro_invocation
@@ -6,6 +8,10 @@
 
 (identifier) @variable
 
+; Assume all-caps names are constants
+((identifier) @constant
+  (#match? @constant "^[A-Z][A-Z%d_]*$"))
+
 (const_item
   name: (identifier) @constant)
 (type_identifier) @type
@@ -18,10 +24,19 @@
   name: (identifier) @module)
 (self) @keyword.builtin
 
-"_" @character.special
+[
+  (line_comment)
+  (block_comment)
+  (outer_doc_comment_marker)
+  (inner_doc_comment_marker)
+] @comment @spell
+
+(line_comment
+  (doc_comment)) @comment.documentation
+
+(block_comment
+  (doc_comment)) @comment.documentation
 
-(line_comment) @comment
-(block_comment) @comment
 
 (boolean_literal) @boolean
 (integer_literal) @number
@@ -32,6 +47,14 @@
 (string_literal) @string
 (raw_string_literal) @string
 
+(use_wildcard
+  "*" @character.special)
+
+(remaining_field_pattern
+  ".." @character.special)
+
+"_" @character.special
+
 ; Keywords
 [
   "use"
@@ -111,30 +134,6 @@
 (closure_parameters
   "|" @punctuation.bracket)
 
-(type_arguments
-  [
-    "<"
-    ">"
-  ] @punctuation.bracket)
-
-(type_parameters
-  [
-    "<"
-    ">"
-  ] @punctuation.bracket)
-
-(bracketed_type
-  [
-    "<"
-    ">"
-  ] @punctuation.bracket)
-
-(for_lifetimes
-  [
-    "<"
-    ">"
-  ] @punctuation.bracket)
-
 [
   ","
   "."
@@ -185,17 +184,40 @@
   "||"
 ] @operator
 
-(use_wildcard
-  "*" @character.special)
+(type_arguments
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
 
-(remaining_field_pattern
-  ".." @character.special)
+(type_parameters
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(bracketed_type
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(for_lifetimes
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+
+(attribute_item
+  "#" @punctuation.special)
+
+(inner_attribute_item
+  [
+    "!"
+    "#"
+  ] @punctuation.special)
 
-; (attribute_item
-;   "#" @punctuation.special)
 
-; (inner_attribute_item
-;   [
-;     "!"
-;     "#"
-;   ] @punctuation.special)
+((identifier) @constant.builtin
+  (#any-of? @constant.builtin "Some" "None" "Ok" "Err"))

data/tree-sitter/queries/rust/highlights.scm

@@ -1,3 +1,17 @@
+(boolean_scalar) @boolean
+(null_scalar) @constant.builtin
+(double_quote_scalar) @string
+(single_quote_scalar) @string
+
+((block_scalar) @string
+  (#set! priority 99))
+
+(string_scalar) @string
+(escape_sequence) @string.escape
+(integer_scalar) @number
+(float_scalar) @number
+(comment) @comment
+
 [
   (anchor_name)
   (alias_name)
@@ -59,18 +73,3 @@
   "---"
   "..."
 ] @punctuation.special
-
-(boolean_scalar) @boolean
-(null_scalar) @constant.builtin
-(double_quote_scalar) @string
-(single_quote_scalar) @string
-
-((block_scalar) @string
-  (#set! priority 99))
-
-(string_scalar) @string
-(escape_sequence) @string.escape
-(integer_scalar) @number
-(float_scalar) @number
-(comment) @comment @spell
-

data/tree-sitter/queries/yaml/highlights.scm

@@ -0,0 +1,60 @@
+# Writing tree-sitter queries for ad
+
+### Overview
+
+`ad` currently supports a subset of the tree-sitter based syntax highlighting you may be familiar
+with from other text editors. The configuration options in your `config.toml` under the `tree_sitter`
+key will direct `ad` to search for scheme highlights files under a given directory for the languages
+that you configure. These query files can, for the most part, be copied from other editors but there
+are a couple of caveats that you should keep in mind.
+
+  1. Support for custom predicates (such as neovim's `#lua-match?` and `#contains?` is not provided.
+     Queries using unsupported predicates will be rejected, resulting in tree-sitter not being enabled
+     for the language in question.
+  2. ad follows the same precedence approach as neovim for overlapping queries, with the _last_ match
+     being used in place of previous matches. When writing your `highlights.scm` file you should place
+     more specific rules towards the end of the file and more general rules towards the start.
+  3. ad does not currently provide any support for indentation, injections, locals or folds. So the
+     resulting highlights you obtain may not exactly match what you are used to from another editor if
+     you are copying over existing queries.
+
+
+### Getting started
+
+ad provides some support for tree-sitter parsers that I have been able to test and verify as part of my
+local setup. For the most part they are adapted from the ones found in [nvim-treesitter](https://github.com/nvim-treesitter/nvim-treesitter/tree/master/queries)
+with the the modigications outlined below. If you are testing your own queries it is often better to start
+with a subset of the `highlights.scm` file from another editor and incrementally add in more queries
+while verifying that the results in ad look the way that you expect.
+
+Beyond that simple advice, the tree-sitter documentation on writing queries is the best place to start
+if you are looking to configure your own custom queries:
+  https://tree-sitter.github.io/tree-sitter/using-parsers/queries/index.html
+
+
+### Troubleshooting
+
+- **Where are my highlights that I configured?**
+  - First make sure that you don't have any typos in your config (we all do it) and that the highlighting
+    provided in the ad repo is working correctly.
+  - If you are sure that you have things set up correctly then try running `:view-logs` to see if there
+    are any warnings about the queries that you are using. You should see a log line asserting that the
+    tree-sitter parser has been initialised, and if not there should be an error message explaining why
+    setup failed.
+
+- **What is this warning about unsupported predicates?**
+  - A common feature of tree-sitter queries is the use of [predicates](https://tree-sitter.github.io/tree-sitter/using-parsers/queries/3-predicates-and-directives.html)
+    to conditionally select nodes in the tree based on their content as well as their type. Tree-sitter
+    supports providing and running custom predicates but it is up to the program running the query to
+    provide the implementation of how custom predicates are applied. `neovim` and other editors tend to
+    make extensive use of custom predicates that are not supported inside of ad. If a query you are
+    using contains any unsupported predicates it will be rejected with warning logs listing the predicates
+    that were found.
+  - Typically, you can replace `#lua-match?` with the tree-sitter built-in `#match?` predicate and have
+    it produce the expected result.
+  - You can also replace `#contains?` with `#match? .. ".*$contained_string.*"`
+
+- **Why are comments not rendering with the correct styling?**
+  - Comment queries from neovim frequently have a secondary `@spell` tag associated with them that ad
+    will end up using as the tag for the match (in place of, for example, `@comment`). If you remove
+    the `@spell` tag from your query you should see your comments receive the correct highlighting.

docs/tree-sitter-queries.md

@@ -414,7 +414,7 @@ impl GapBuffer {
     }
 
     /// Convert a byte index to a character index
-    pub fn byte_to_char(&self, byte_idx: usize) -> usize {
+    pub fn raw_byte_to_char(&self, byte_idx: usize) -> usize {
         self.chars_in_raw_range(0, byte_idx)
     }
 
@@ -739,7 +739,7 @@ impl GapBuffer {
     }
 
     #[inline]
-    fn byte_to_raw_byte(&self, byte: usize) -> usize {
+    pub fn byte_to_raw_byte(&self, byte: usize) -> usize {
         if byte > self.gap_start {
             byte + self.gap()
         } else {

src/buffer/internal.rs

@@ -410,6 +410,10 @@ impl Buffer {
         s
     }
 
+    pub(crate) fn pretty_print_ts_tree(&self) -> Option<String> {
+        self.ts_state.as_ref().map(|ts| ts.pretty_print_tree())
+    }
+
     pub(crate) fn string_lines(&self) -> Vec<String> {
         self.txt
             .iter_lines()

src/buffer/mod.rs

@@ -118,6 +118,7 @@ pub enum Action {
     ShellRun { cmd: String },
     ShellSend { cmd: String },
     ShowHelp,
+    TsShowTree,
     Undo,
     UpdateConfig { input: String },
     ViewLogs,
@@ -495,6 +496,13 @@ where
             .open_virtual("+logs", self.log_buffer.content(), false)
     }
 
+    pub(super) fn show_active_ts_tree(&mut self) {
+        match self.layout.active_buffer().pretty_print_ts_tree() {
+            Some(s) => self.layout.open_virtual("+ts-tree", s, false),
+            None => self.set_status_message("no tree-sitter tree for current buffer"),
+        }
+    }
+
     pub(super) fn show_help(&mut self) {
         self.layout.open_virtual("+help", gen_help_docs(), false)
     }

src/editor/actions.rs

@@ -138,6 +138,8 @@ fn parse_command(input: &str, active_buffer_id: usize, cwd: &Path) -> Result<Act
             input: input.to_string(),
         })),
 
+        "ts-show-tree" => Ok(Single(TsShowTree)),
+
         "view-logs" => Ok(Single(ViewLogs)),
 
         "w" | "write" => {

src/editor/commands.rs

@@ -492,6 +492,7 @@ where
             ShellReplace { cmd } => self.replace_dot_with_shell_cmd(&cmd),
             ShellRun { cmd } => self.run_shell_cmd(&cmd),
             ShowHelp => self.show_help(),
+            TsShowTree => self.show_active_ts_tree(),
             UpdateConfig { input } => self.update_config(&input),
             ViewLogs => self.view_logs(),
             Yank => self.set_clipboard(self.layout.active_buffer().dot_contents()),

src/editor/mod.rs

@@ -86,7 +86,7 @@ impl PositionEncoding {
             Self::Utf8 => {
                 let line_start = b.txt.line_to_char(pos.line as usize);
                 let byte_idx = b.txt.char_to_byte(line_start + pos.character as usize);
-                let col = b.txt.byte_to_char(byte_idx);
+                let col = b.txt.raw_byte_to_char(byte_idx);
 
                 (pos.line as usize, col)
             }