Implement textDocument/semanticTokens/full (#1194)

* Implement textDocument/semanticTokens/full

* Update documentation

* Call textDocument/semanticTokens/full on open, change and save (+ lint fixes)

* Remove default mappings

Author: martskins

autoload/LanguageClient.vim

@@ -907,6 +907,22 @@ function! LanguageClient#Notify(method, params) abort
                 \ }))
 endfunction
 
+function! LanguageClient#textDocument_semanticTokensFull(...) abort
+    if s:ShouldUseFloatWindow() && s:MoveIntoHoverPreview('__LCNHover__')
+        return
+    endif
+    let l:Callback = get(a:000, 1, v:null)
+    let l:params = {
+                \ 'filename': LSP#filename(),
+                \ 'text': LSP#text(),
+                \ 'line': LSP#line(),
+                \ 'character': LSP#character(),
+                \ 'handle': s:IsFalse(l:Callback),
+                \ }
+    call extend(l:params, get(a:000, 0, {}))
+    return LanguageClient#Call('textDocument/semanticTokens/full', l:params, l:Callback)
+endfunction
+
 function! LanguageClient#textDocument_hover(...) abort
     if s:ShouldUseFloatWindow() && s:MoveIntoHoverPreview('__LCNHover__')
         return
@@ -1692,20 +1708,6 @@ function! s:print_semantic_scopes(response) abort
     echo l:msg
 endfunction
 
-function! LanguageClient#showSemanticHighlightSymbols(...) abort
-    let l:params = get(a:000, 0, {})
-    let l:Callback = get(a:000, 1, v:null)
-
-    return LanguageClient#Call('languageClient/showSemanticHighlightSymbols', l:params, l:Callback)
-endfunction
-
-function! LanguageClient_showCursorSemanticHighlightSymbols(...) abort
-    let l:params = get(a:000, 0, {})
-    let l:Callback = get(a:000, 1, function('s:print_cursor_semantic_symbol'))
-
-    return LanguageClient#showSemanticHighlightSymbols(l:params, l:Callback)
-endfunction
-
 function! s:print_cursor_semantic_symbol(response) abort
     let l:symbols = a:response.result
     let l:lines = []

doc/LanguageClient.txt

@@ -516,104 +516,7 @@ Valid options: 1 | 0
 
 2.33 g:LanguageClient_semanticHighlightMaps   *g:LanguageClient_semanticHighlightMaps*
 
-String to list/map map. Defines the mapping of semantic highlighting "scopes" to
-highlight groups. This depends on the LSP server supporting the proposed
-semantic highlighting protocol, see:
-
-https://github.com/microsoft/language-server-protocol/issues/18
-https://github.com/microsoft/vscode-languageserver-node/issues/368
-
-
-Like |g:LanguageClient_serverCommands| this is a map where the keys are
-filetypes. However each submap has |regexp| keys and highlight group names
-as values (see |highlight-groups|).
->
-    let g:LanguageClient_semanticHighlightMaps = {
-        \ 'java': {
-        \   '^entity.name.function.java': 'Function',
-        \   '^entity.name.type.class.java': 'Type',
-        \   '^[^:]*entity.name.function.java': 'Function',
-        \   '^[^:]entity.name.type.class.java': 'Type'
-        \ }
-        \ }
-
-The |regexp| in the keys will be used to match semantic scopes. Then any symbols
-that have a semantic scope that matches the key will be highlighted with the
-associated highlight group value. Currently there is no defined order if a
-semantic scope can match multiple keys, so it is recommended to make the keys
-more specific to only match the desired scope(s).
-
-There are a fixed set of semantic scopes defined by the LSP server on startup.
-These can be viewed by calling |LanguageClient_showSemanticScopes| which will
-show all the semantic scopes and their currently mapped highlight group for
-the currently open buffer's filetype.
->
-     call LanguageClient_showSemanticScopes()
-
-     == Output from eclipse.jdt.ls ==
-
-     Highlight Group:
-      None
-     Semantic Scope:
-      invalid.deprecated.java
-       meta.class.java
-        source.java
-
-     Highlight Group:
-      None
-     Semantic Scope:
-      variable.other.autoboxing.java
-       meta.method.body.java
-        meta.method.java
-         meta.class.body.java
-          meta.class.java
-           source.java
-
-     ...
-
-Each semantic scope is a list of strings. They are printed with increasing
-indent to make it easier to read. For example the first scope is:
->
-     ['invalid.deprecated.java', 'meta.class.java', 'source.java']
-
-It is currently isn't mapped to any highlight group as indicated by the None.
-
-Often its more useful to find what semantic scope corresponds to a piece of
-text. This can be done by calling |LanguageClient_showCursorSemanticHighlightSymbols|
-while hovering over the text of interest.
->
-     call LanguageClient_showCursorSemanticHighlightSymbols()
-
-When matching the semantic scopes to keys in |LanguageClient_semanticHighlightMaps|,
-the scopes are concatentated using |LanguageClient_semanticScopeSeparator|
-which is set to the string |':'| by default. For the previous example the
-semantic scope would have this string form using the default separator:
->
-     invalid.deprecated.java:meta.class.java:source.java
-
-Here are a couple of example |regexp| keys that can/cannot match this scope:
->
-     'meta.class.java' =~ 'invalid.deprecated.java:meta.class.java:source.java'
-     '^meta.class.java' !~ 'invalid.deprecated.java:meta.class.java:source.java'
-     '^invalid.deprecated.java' =~ 'invalid.deprecated.java:meta.class.java:source.java'
-     'source.java$' =~ 'invalid.deprecated.java:meta.class.java:source.java'
-     'meta.class.java:source.java' =~ 'invalid.deprecated.java:meta.class.java:source.java'
-     'invalid.deprecated.java:.*:source.java' =~ 'invalid.deprecated.java:meta.class.java:source.java'
-
-Example configuration for eclipse.jdt.ls:
->
-     let g:LanguageClient_semanticHighlightMaps = {}
-     let g:LanguageClient_semanticHighlightMaps['java'] = {
-       \ '^storage.modifier.static.java:entity.name.function.java': 'JavaStaticMemberFunction',
-       \ '^meta.definition.variable.java:meta.class.body.java:meta.class.java': 'JavaMemberVariable',
-       \ '^entity.name.function.java': 'Function',
-       \ '^[^:]*entity.name.function.java': 'Function',
-       \ '^[^:]*entity.name.type.class.java': 'Type',
-       \ }
-
-     highlight! JavaStaticMemberFunction ctermfg=Green cterm=none guifg=Green gui=none
-     highlight! JavaMemberVariable ctermfg=White cterm=italic guifg=White gui=italic
-
+Deprecated, see g:LanguageClient_semanticTokenMappings.
 
 2.34 g:LanguageClient_applyCompletionAdditionalTextEdits *g:LanguageClient_applyCompletionAdditionalTextEdits*
 
@@ -760,6 +663,43 @@ Show completion item documentation in a floating window right next to pmenu.
 Default: 1
 Valid options: 1 | 0
 
+2.46 g:LanguageClient_semanticHighlightingEnabled *g:LanguageClient_semanticHighlightingEnabled*
+
+If enabled, the client will call the LSP semantic tokens method after opening,
+modifying or saving a buffer to try and apply semantic highlighting to the
+buffer.
+
+Default: 0
+Valid options: 1 | 0
+
+2.47 g:LanguageClient_semanticTokenMappings *g:LanguageClient_semanticTokenMappings*
+
+Maps LSP token types to vim highlight groups. Each item in this array is a
+dictionary with the fields name, modifiers and highlightGroup, where name is
+the name of the LSP token type, modifiers is a list of modifiers that need to
+be present in that token for this mapping to match and highlightGroup is the
+vim highlight group that should be applied to the token.
+
+Example:
+
+>
+let g:LanguageClient_semanticTokenMappings = [
+  { 'name': 'comment', 'modifiers': [], 'highlightGroup': 'Comment' },
+  { 'name': 'comment', 'modifiers': ['documentation'], 'highlightGroup': 'Question' },
+];
+<
+
+The first mapping in that example maps the tokens of type `comment` to the
+highlight group `Comment`, and the second matches the tokens of type `comment`
+that have the `documentation` modifier to the `Question` highlight group, this
+way documentation comments and regular comments are (potentially) displayed
+with a different colour.
+
+Default: []
+
+For a full list of the possible token types (name) see the LSP documentation:
+https://microsoft.github.io/language-server-protocol/specifications/specification-current/
+
 ==============================================================================
 3. Commands                                           *LanguageClientCommands*
 

src/config/mod.rs

@@ -1,7 +1,5 @@
 mod server_command;
 
-pub use server_command::*;
-
 use crate::{
     types::{
         CodeLensDisplay, DiagnosticsDisplay, DiagnosticsList, DocumentHighlightDisplay,
@@ -12,6 +10,7 @@ use crate::{
 use anyhow::{anyhow, Result};
 use lsp_types::{DiagnosticSeverity, MarkupKind, MessageType, TraceOption};
 use serde::Deserialize;
+pub use server_command::*;
 use std::collections::HashMap;
 use std::{path::PathBuf, str::FromStr, time::Duration};
 
@@ -32,6 +31,26 @@ impl LoggerConfig {
     }
 }
 
+#[derive(Debug, Clone, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct SemanticTokenMapping {
+    pub name: String,
+    #[serde(default)]
+    pub modifiers: Vec<String>,
+    pub highlight_group: String,
+}
+
+impl SemanticTokenMapping {
+    #[allow(dead_code)]
+    pub fn new(name: &str, modifiers: &[&str], highlight_group: &str) -> Self {
+        Self {
+            name: name.to_owned(),
+            modifiers: modifiers.iter().map(|i| i.to_string()).collect(),
+            highlight_group: highlight_group.to_owned(),
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct Config {
     pub auto_start: bool,
@@ -59,22 +78,32 @@ pub struct Config {
     pub selection_ui_auto_open: bool,
     pub use_virtual_text: UseVirtualText,
     pub echo_project_root: bool,
-    pub semantic_highlight_maps: HashMap<String, HashMap<String, String>>,
-    pub semantic_scope_separator: String,
     pub apply_completion_text_edits: bool,
     pub preferred_markup_kind: Option<Vec<MarkupKind>>,
     pub hide_virtual_texts_on_insert: bool,
     pub enable_extensions: Option<HashMap<String, bool>>,
     pub restart_on_crash: bool,
     pub max_restart_retries: u8,
+    /// semantic_token_mappings is a vec of SemanticTokenMappings, where a SemanticTokenMapping
+    /// defines the token type by it's name, the modifiers and the highlight group to be applied to
+    /// it.
+    ///
+    /// If no modifiers are configured for a type it will apply for all tokens of that type.
+    ///
+    /// For example:
+    ///
+    /// [
+    ///     { "name": "function", "modifiers": ["async"], "highlightGroup": "Function" }
+    ///     { "name": "type", "modifiers": [], "highlightGroup": "Type" }
+    /// ]
+    pub semantic_token_mappings: Vec<SemanticTokenMapping>,
+    pub semantic_highlighting_enabled: bool,
 }
 
 impl Default for Config {
     fn default() -> Self {
         Self {
             server_commands: HashMap::new(),
-            semantic_highlight_maps: HashMap::new(),
-            semantic_scope_separator: ":".into(),
             auto_start: true,
             selection_ui: SelectionUI::LocationList,
             selection_ui_auto_open: true,
@@ -105,6 +134,8 @@ impl Default for Config {
             is_nvim: false,
             restart_on_crash: true,
             max_restart_retries: 5,
+            semantic_token_mappings: vec![],
+            semantic_highlighting_enabled: false,
         }
     }
 }
@@ -135,15 +166,15 @@ struct DeserializableConfig {
     selection_ui_auto_open: u8,
     use_virtual_text: UseVirtualText,
     echo_project_root: u8,
-    semantic_highlight_maps: HashMap<String, HashMap<String, String>>,
-    semantic_scope_separator: String,
     apply_completion_text_edits: u8,
     preferred_markup_kind: Option<Vec<MarkupKind>>,
     hide_virtual_texts_on_insert: u8,
     enable_extensions: Option<HashMap<String, bool>>,
     code_lens_display: Option<CodeLensDisplay>,
     restart_on_crash: u8,
     max_restart_retries: u8,
+    semantic_token_mappings: Vec<SemanticTokenMapping>,
+    semantic_highlighting_enabled: u8,
 }
 
 impl Config {
@@ -172,8 +203,6 @@ impl Config {
             "selection_ui_auto_open": !!s:GetVar('LanguageClient_selectionUI_autoOpen', 1),
             "use_virtual_text": s:useVirtualText(),
             "echo_project_root": !!s:GetVar('LanguageClient_echoProjectRoot', 1),
-            "semantic_highlight_maps": s:GetVar('LanguageClient_semanticHighlightMaps', {}),
-            "semantic_scope_separator": s:GetVar('LanguageClient_semanticScopeSeparator', ':'),
             "apply_completion_text_edits": get(g:, 'LanguageClient_applyCompletionAdditionalTextEdits', 1),
             "preferred_markup_kind": get(g:, 'LanguageClient_preferredMarkupKind', v:null),
             "hide_virtual_texts_on_insert": s:GetVar('LanguageClient_hideVirtualTextsOnInsert', 0),
@@ -182,10 +211,11 @@ impl Config {
             "restart_on_crash": get(g:, 'LanguageClient_restartOnCrash', 1),
             "max_restart_retries": get(g:, 'LanguageClient_maxRestartRetries', 5),
             "server_stderr": get(g:, 'LanguageClient_serverStderr', v:null),
+            "semantic_token_mappings": get(g:, 'LanguageClient_semanticTokenMappings', []),
+            "semantic_highlighting_enabled": get(g:, 'LanguageClient_semanticHighlightingEnabled', 0),
         }"#;
 
         let res: DeserializableConfig = vim.eval(req.replace("\n", ""))?;
-
         let loaded_fzf = vim.eval::<_, i64>("get(g:, 'loaded_fzf')")? == 1;
         let selection_ui = match res.selection_ui {
             Some(s) => SelectionUI::from_str(&s)?,
@@ -238,14 +268,14 @@ impl Config {
             selection_ui_auto_open: res.selection_ui_auto_open == 1,
             use_virtual_text: res.use_virtual_text,
             echo_project_root: res.echo_project_root == 1,
-            semantic_highlight_maps: res.semantic_highlight_maps,
-            semantic_scope_separator: res.semantic_scope_separator,
             apply_completion_text_edits: res.apply_completion_text_edits == 1,
             preferred_markup_kind: res.preferred_markup_kind,
             hide_virtual_texts_on_insert: res.hide_virtual_texts_on_insert == 1,
             enable_extensions: res.enable_extensions,
             restart_on_crash: res.restart_on_crash == 1,
             max_restart_retries: res.max_restart_retries,
+            semantic_token_mappings: res.semantic_token_mappings,
+            semantic_highlighting_enabled: res.semantic_highlighting_enabled == 1,
         })
     }
 }