Semantic tokens highlighting

book/src/generated/typable-cmd.md

@@ -50,6 +50,7 @@
 | `:lsp-workspace-command` | Open workspace command picker |
 | `:lsp-restart` | Restarts the Language Server that is in use by the current doc |
 | `:tree-sitter-scopes` | Display tree sitter scopes, primarily for theming and development. |
+| `:semantic-tokens` | Display the semantic tokens, primarily for theming and development. |
 | `:debug-start`, `:dbg` | Start a debug session from a given template with given parameters. |
 | `:debug-remote`, `:dbg-tcp` | Connect to a debug adapter by TCP address and start a debugging session from a given template with given parameters. |
 | `:debug-eval` | Evaluate expression in current debug context. |

helix-lsp/src/client.rs

@@ -10,13 +10,13 @@ use lsp::PositionEncodingKind;
 use lsp_types as lsp;
 use serde::Deserialize;
 use serde_json::Value;
-use std::collections::HashMap;
 use std::future::Future;
 use std::process::Stdio;
 use std::sync::{
     atomic::{AtomicU64, Ordering},
-    Arc,
+    Arc, RwLock,
 };
+use std::{collections::HashMap, sync::RwLockReadGuard};
 use tokio::{
     io::{BufReader, BufWriter},
     process::{Child, Command},
@@ -38,6 +38,9 @@ pub struct Client {
     root_uri: Option<lsp::Url>,
     workspace_folders: Vec<lsp::WorkspaceFolder>,
     req_timeout: u64,
+
+    pub(crate) semantic_token_types_legend: RwLock<Vec<Arc<String>>>,
+    pub(crate) semantic_token_modifiers_legend: RwLock<Vec<Arc<String>>>,
 }
 
 impl Client {
@@ -110,6 +113,9 @@ impl Client {
             root_path,
             root_uri,
             workspace_folders,
+
+            semantic_token_types_legend: Default::default(),
+            semantic_token_modifiers_legend: Default::default(),
         };
 
         Ok((client, server_rx, initialize_notify))
@@ -161,6 +167,14 @@ impl Client {
             .unwrap_or_default()
     }
 
+    pub fn types_legend(&self) -> RwLockReadGuard<'_, Vec<Arc<String>>> {
+        self.semantic_token_types_legend.read().unwrap()
+    }
+
+    pub fn modifiers_legend(&self) -> RwLockReadGuard<'_, Vec<Arc<String>>> {
+        self.semantic_token_modifiers_legend.read().unwrap()
+    }
+
     pub fn config(&self) -> Option<&Value> {
         self.config.as_ref()
     }
@@ -315,6 +329,9 @@ impl Client {
                     execute_command: Some(lsp::DynamicRegistrationClientCapabilities {
                         dynamic_registration: Some(false),
                     }),
+                    semantic_tokens: Some(lsp::SemanticTokensWorkspaceClientCapabilities {
+                        refresh_support: Some(false),
+                    }),
                     ..Default::default()
                 }),
                 text_document: Some(lsp::TextDocumentClientCapabilities {
@@ -386,6 +403,22 @@ impl Client {
                     publish_diagnostics: Some(lsp::PublishDiagnosticsClientCapabilities {
                         ..Default::default()
                     }),
+                    semantic_tokens: Some(lsp::SemanticTokensClientCapabilities {
+                        dynamic_registration: Some(true),
+                        requests: lsp::SemanticTokensClientCapabilitiesRequests {
+                            range: Some(true),
+                            full: Some(lsp::SemanticTokensFullOptions::Bool(false)),
+                        },
+                        // Don't specify defaults for this, Helix will receive the server response and tell it it
+                        // support the same tokens.
+                        token_types: Vec::new(),
+                        token_modifiers: Vec::new(),
+                        overlapping_token_support: Some(true),
+                        multiline_token_support: Some(true),
+                        server_cancel_support: Some(true),
+                        augments_syntax_tokens: Some(true),
+                        ..Default::default()
+                    }),
                     ..Default::default()
                 }),
                 window: Some(lsp::WindowClientCapabilities {
@@ -702,6 +735,37 @@ impl Client {
         Some(self.call::<lsp::request::ResolveCompletionItem>(completion_item))
     }
 
+    pub fn text_document_semantic_tokens(
+        &self,
+        text_document: lsp::TextDocumentIdentifier,
+        range: lsp::Range,
+        work_done_token: Option<lsp::ProgressToken>,
+    ) -> Option<impl Future<Output = Result<Value>>> {
+        let capabilites = self.capabilities.get().unwrap();
+
+        let support_range = match capabilites.semantic_tokens_provider.as_ref()? {
+            lsp::SemanticTokensServerCapabilities::SemanticTokensOptions(opt) => opt.range?,
+            lsp::SemanticTokensServerCapabilities::SemanticTokensRegistrationOptions(opt) => {
+                opt.semantic_tokens_options.range?
+            }
+        };
+
+        if !support_range {
+            return None;
+        }
+
+        Some(self.call::<lsp::request::SemanticTokensRangeRequest>(
+            lsp::SemanticTokensRangeParams {
+                work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
+                partial_result_params: lsp::PartialResultParams {
+                    partial_result_token: None,
+                },
+                text_document,
+                range,
+            },
+        ))
+    }
+
     pub fn text_document_signature_help(
         &self,
         text_document: lsp::TextDocumentIdentifier,

helix-lsp/src/lib.rs

@@ -621,9 +621,46 @@ fn start_client(
             })
             .await;
 
-        if let Err(e) = value {
-            log::error!("failed to initialize language server: {}", e);
-            return;
+        let capabilities = match value {
+            Ok(value) => value,
+            Err(e) => {
+                log::error!("failed to initialize language server: {}", e);
+                return;
+            }
+        };
+
+        // Compute the legends for semantic tokens and put them in `Arc<String>`s, that way when
+        // documents use them, the cost is negligible: reasonable servers will send at most a few
+        // dozen strings but documents can reference each string hundreds of times.
+        let legends = match capabilities.semantic_tokens_provider.as_ref() {
+            Some(lsp::SemanticTokensServerCapabilities::SemanticTokensOptions(opt)) => {
+                Some(&opt.legend)
+            }
+            Some(lsp::SemanticTokensServerCapabilities::SemanticTokensRegistrationOptions(opt)) => {
+                Some(&opt.semantic_tokens_options.legend)
+            }
+            _ => None,
+        };
+
+        // We assume the names won't have spaces or invalid TOML in them else themes won't be able
+        // to name them. This has not been a problem yet, if it becomes one we will need to
+        // implement fixes here.
+        if let Some(legends) = legends {
+            let make_style_name = |v| Arc::new(format!("semantic.{v}"));
+
+            let mut types = _client.semantic_token_types_legend.write().unwrap();
+            types.clear();
+
+            for ty in &legends.token_types {
+                types.push(make_style_name(ty.as_str()));
+            }
+
+            let mut modifiers = _client.semantic_token_modifiers_legend.write().unwrap();
+            modifiers.clear();
+
+            for mo in &legends.token_modifiers {
+                modifiers.push(make_style_name(mo.as_str()));
+            }
         }
 
         // next up, notify<initialized>

helix-term/src/commands.rs

@@ -110,17 +110,7 @@ impl<'a> Context<'a> {
         T: for<'de> serde::Deserialize<'de> + Send + 'static,
         F: FnOnce(&mut Editor, &mut Compositor, T) + Send + 'static,
     {
-        let callback = Box::pin(async move {
-            let json = call.await?;
-            let response = serde_json::from_value(json)?;
-            let call: job::Callback = Callback::EditorCompositor(Box::new(
-                move |editor: &mut Editor, compositor: &mut Compositor| {
-                    callback(editor, compositor, response)
-                },
-            ));
-            Ok(call)
-        });
-        self.jobs.callback(callback);
+        self.jobs.callback(make_job_callback(call, callback));
     }
 
     /// Returns 1 if no explicit count was provided
@@ -130,6 +120,27 @@ impl<'a> Context<'a> {
     }
 }
 
+#[inline]
+fn make_job_callback<T, F>(
+    call: impl Future<Output = helix_lsp::Result<serde_json::Value>> + 'static + Send,
+    callback: F,
+) -> std::pin::Pin<Box<impl Future<Output = Result<Callback, anyhow::Error>>>>
+where
+    T: for<'de> serde::Deserialize<'de> + Send + 'static,
+    F: FnOnce(&mut Editor, &mut Compositor, T) + Send + 'static,
+{
+    Box::pin(async move {
+        let json = call.await?;
+        let response = serde_json::from_value(json)?;
+        let call: job::Callback = Callback::EditorCompositor(Box::new(
+            move |editor: &mut Editor, compositor: &mut Compositor| {
+                callback(editor, compositor, response)
+            },
+        ));
+        Ok(call)
+    })
+}
+
 use helix_view::{align_view, Align};
 
 /// A MappableCommand is either a static command like "jump_view_up" or a Typable command like

helix-term/src/commands/lsp.rs

@@ -26,6 +26,9 @@ use crate::{
     },
 };
 
+mod semantic_tokens;
+pub use semantic_tokens::*;
+
 use std::{
     borrow::Cow, cmp::Ordering, collections::BTreeMap, fmt::Write, path::PathBuf, sync::Arc,
 };