From 9b513cfda5ee751ca84e46701e77d55dbe951e57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20F=C3=B6rster?= Date: Sat, 18 Mar 2023 09:45:15 +0100 Subject: [PATCH 1/9] Add semantic tokens for undef and unused labels --- Cargo.lock | 23 ++++-- Cargo.toml | 3 +- src/features.rs | 1 + src/features/semantic_tokens.rs | 109 ++++++++++++++++++++++++++ src/features/semantic_tokens/label.rs | 73 +++++++++++++++++ src/server.rs | 30 ++++--- 6 files changed, 221 insertions(+), 18 deletions(-) create mode 100644 src/features/semantic_tokens.rs create mode 100644 src/features/semantic_tokens/label.rs diff --git a/Cargo.lock b/Cargo.lock index 59566d3b4..3423156d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -93,6 +93,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5dd14596c0e5b954530d0e6f1fd99b89c03e313aa2086e8da4303701a09e1cf" + [[package]] name = "block-buffer" version = "0.10.4" @@ -179,7 +185,7 @@ version = "3.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ - "bitflags", + "bitflags 1.3.2", "clap_lex 0.2.4", "indexmap", "textwrap", @@ -191,7 +197,7 @@ version = "4.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d7ae14b20b94cb02149ed21a86c423859cbe18dc7ed69845cace50e52b40a5" dependencies = [ - "bitflags", + "bitflags 1.3.2", "clap_derive", "clap_lex 0.3.2", "is-terminal", @@ -724,7 +730,7 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" dependencies = [ - "bitflags", + "bitflags 1.3.2", "inotify-sys", "libc", ] @@ -843,7 +849,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8367585489f01bc55dd27404dcf56b95e6da061a256a666ab23be9ba96a2e587" dependencies = [ - "bitflags", + "bitflags 1.3.2", "libc", ] @@ -931,7 +937,7 @@ version = "0.94.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b63735a13a1f9cd4f4835223d828ed9c2e35c8c5e61837774399f558b6a1237" dependencies = [ - "bitflags", + "bitflags 1.3.2", "serde", "serde_json", "serde_repr", @@ -998,7 +1004,7 @@ version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58ea850aa68a06e48fdb069c0ec44d0d64c8dbffa49bf3b6f7f0a901fdea1ba9" dependencies = [ - "bitflags", + "bitflags 1.3.2", "crossbeam-channel", "filetime", "fsevent-sys", @@ -1301,7 +1307,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1357,7 +1363,7 @@ version = "0.36.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd5c6ff11fecd55b40746d1995a02f2eb375bf8c00d192d521ee09f42bef37bc" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -1597,6 +1603,7 @@ version = "5.4.0" dependencies = [ "anyhow", "assert_unordered", + "bitflags 2.0.1", "chrono", "clap 4.1.8", "criterion", diff --git a/Cargo.toml b/Cargo.toml index 00d1cc4e2..bd9806f02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ doctest = false [dependencies] anyhow = "1.0.69" chrono = { version = "0.4.23", default-features = false, features = ["std"] } -clap = { version = "4.1.6", features = ["derive"] } +clap = { version = "4.1.8", features = ["derive"] } crossbeam-channel = "0.5.6" dashmap = "5.4.0" dirs = "4.0.0" @@ -69,6 +69,7 @@ thiserror = "1.0.38" threadpool = "1.8.1" titlecase = "2.2.1" unicode-normalization = "0.1.22" +bitflags = "2.0.1" [dependencies.salsa] git = "https://github.com/salsa-rs/salsa" diff --git a/src/features.rs b/src/features.rs index 0d82bcb4f..37e159da9 100644 --- a/src/features.rs +++ b/src/features.rs @@ -10,5 +10,6 @@ pub mod inlay_hint; pub mod link; pub mod reference; pub mod rename; +pub mod semantic_tokens; pub mod symbol; pub mod workspace_command; diff --git a/src/features/semantic_tokens.rs b/src/features/semantic_tokens.rs new file mode 100644 index 000000000..7c6203b5b --- /dev/null +++ b/src/features/semantic_tokens.rs @@ -0,0 +1,109 @@ +mod label; + +use bitflags::bitflags; +use lsp_types::{ + Position, Range, SemanticToken, SemanticTokenModifier, SemanticTokenType, SemanticTokens, + SemanticTokensLegend, Url, +}; +use rowan::TextRange; + +use crate::{ + db::Workspace, + util::{line_index::LineIndex, line_index_ext::LineIndexExt}, + Db, +}; + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] +#[repr(u32)] +pub enum TokenKind { + Label = 0, +} + +bitflags! { + #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] + pub struct TokenModifiers: u32 { + const NONE = 0; + const UNDEFINED = 1; + const UNUSED = 2; + } +} + +pub fn legend() -> SemanticTokensLegend { + SemanticTokensLegend { + token_types: vec![SemanticTokenType::new("label")], + token_modifiers: vec![ + SemanticTokenModifier::new("undefined"), + SemanticTokenModifier::new("unused"), + ], + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +pub struct Token { + pub range: TextRange, + pub kind: TokenKind, + pub modifiers: TokenModifiers, +} + +#[derive(Debug, Default)] +pub struct TokenBuilder { + tokens: Vec, +} + +impl TokenBuilder { + pub fn push(&mut self, token: Token) { + self.tokens.push(token); + } + + pub fn finish(mut self, line_index: &LineIndex) -> SemanticTokens { + let mut data = Vec::new(); + + self.tokens.sort_by_key(|token| token.range.start()); + + let mut last_pos = Position::new(0, 0); + for token in self.tokens { + let range = line_index.line_col_lsp_range(token.range); + let length = range.end.character - range.start.character; + let token_type = token.kind as u32; + let token_modifiers_bitset = token.modifiers.bits(); + + if range.start.line > last_pos.line { + let delta_line = range.start.line - last_pos.line; + let delta_start = range.start.character; + data.push(SemanticToken { + delta_line, + delta_start, + length, + token_type, + token_modifiers_bitset, + }); + } else { + let delta_line = 0; + let delta_start = last_pos.character - range.start.character; + data.push(SemanticToken { + delta_line, + delta_start, + length, + token_type, + token_modifiers_bitset, + }); + } + + last_pos = range.end; + } + + SemanticTokens { + result_id: None, + data, + } + } +} + +pub fn find_all(db: &dyn Db, uri: &Url, viewport: Range) -> Option { + let workspace = Workspace::get(db); + let document = workspace.lookup_uri(db, uri)?; + let viewport = document.line_index(db).offset_lsp_range(viewport); + let mut builder = TokenBuilder::default(); + label::find(db, document, viewport, &mut builder); + Some(builder.finish(document.line_index(db))) +} diff --git a/src/features/semantic_tokens/label.rs b/src/features/semantic_tokens/label.rs new file mode 100644 index 000000000..c542a05b5 --- /dev/null +++ b/src/features/semantic_tokens/label.rs @@ -0,0 +1,73 @@ +use rowan::TextRange; + +use crate::{ + db::{analysis::label, Document, Workspace}, + Db, +}; + +use super::{Token, TokenBuilder, TokenKind, TokenModifiers}; + +pub fn find( + db: &dyn Db, + document: Document, + viewport: TextRange, + builder: &mut TokenBuilder, +) -> Option<()> { + let labels = document.parse(db).as_tex()?.analyze(db).labels(db); + for label in labels + .iter() + .filter(|label| viewport.intersect(label.range(db)).is_some()) + { + let name = label.name(db).text(db); + let modifiers = match label.origin(db) { + label::Origin::Definition(_) => { + if !is_label_referenced(db, document, name) { + TokenModifiers::UNUSED + } else { + TokenModifiers::NONE + } + } + label::Origin::Reference(_) | label::Origin::ReferenceRange(_) => { + if !is_label_defined(db, document, name) { + TokenModifiers::UNDEFINED + } else { + TokenModifiers::NONE + } + } + }; + + let range = label.range(db); + builder.push(Token { + range, + kind: TokenKind::Label, + modifiers, + }); + } + + Some(()) +} + +fn is_label_defined(db: &dyn Db, child: Document, name: &str) -> bool { + Workspace::get(db) + .related(db, child) + .iter() + .filter_map(|document| document.parse(db).as_tex()) + .flat_map(|data| data.analyze(db).labels(db)) + .filter(|label| matches!(label.origin(db), label::Origin::Definition(_))) + .any(|label| label.name(db).text(db) == name) +} + +fn is_label_referenced(db: &dyn Db, child: Document, name: &str) -> bool { + Workspace::get(db) + .related(db, child) + .iter() + .filter_map(|document| document.parse(db).as_tex()) + .flat_map(|data| data.analyze(db).labels(db)) + .filter(|label| { + matches!( + label.origin(db), + label::Origin::Reference(_) | label::Origin::ReferenceRange(_) + ) + }) + .any(|label| label.name(db).text(db) == name) +} diff --git a/src/server.rs b/src/server.rs index bf0ef6426..ad2d6148a 100644 --- a/src/server.rs +++ b/src/server.rs @@ -28,7 +28,7 @@ use crate::{ build::{self, BuildParams, BuildResult, BuildStatus}, completion::{self, builder::CompletionItemData}, definition, folding, formatting, forward_search, highlight, hover, inlay_hint, link, - reference, rename, symbol, + reference, rename, semantic_tokens, symbol, workspace_command::{change_environment, clean, dep_graph}, }, normalize_uri, @@ -179,6 +179,14 @@ impl Server { ..Default::default() }), inlay_hint_provider: Some(OneOf::Left(true)), + semantic_tokens_provider: Some( + SemanticTokensServerCapabilities::SemanticTokensOptions(SemanticTokensOptions { + work_done_progress_options: Default::default(), + legend: semantic_tokens::legend(), + range: Some(true), + full: Some(SemanticTokensFullOptions::Bool(false)), + }), + ), ..ServerCapabilities::default() } } @@ -707,14 +715,6 @@ impl Server { Ok(()) } - fn semantic_tokens_range( - &self, - _id: RequestId, - _params: SemanticTokensRangeParams, - ) -> Result<()> { - Ok(()) - } - fn build(&mut self, id: RequestId, params: BuildParams) -> Result<()> { let mut uri = params.text_document.uri; normalize_uri(&mut uri); @@ -814,6 +814,18 @@ impl Server { Ok(()) } + fn semantic_tokens_range( + &mut self, + id: RequestId, + params: SemanticTokensRangeParams, + ) -> Result<()> { + self.run_with_db(id, move |db| { + semantic_tokens::find_all(db, ¶ms.text_document.uri, params.range) + }); + + Ok(()) + } + fn handle_file_event(&mut self, event: notify::Event) { let mut changed = false; From 0805a1287fe97cf83a82d56eb23e4409f83bc522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20F=C3=B6rster?= Date: Sat, 18 Mar 2023 11:16:15 +0100 Subject: [PATCH 2/9] Support full semantic tokens request --- src/server.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/server.rs b/src/server.rs index ad2d6148a..102064a90 100644 --- a/src/server.rs +++ b/src/server.rs @@ -13,7 +13,7 @@ use log::{error, info}; use lsp_server::{Connection, ErrorCode, Message, RequestId}; use lsp_types::{notification::*, request::*, *}; use once_cell::sync::Lazy; -use rowan::{ast::AstNode, TextSize}; +use rowan::{ast::AstNode, TextLen, TextRange, TextSize}; use rustc_hash::FxHashSet; use serde::{Deserialize, Serialize}; use serde_repr::{Deserialize_repr, Serialize_repr}; @@ -184,7 +184,7 @@ impl Server { work_done_progress_options: Default::default(), legend: semantic_tokens::legend(), range: Some(true), - full: Some(SemanticTokensFullOptions::Bool(false)), + full: Some(SemanticTokensFullOptions::Bool(true)), }), ), ..ServerCapabilities::default() @@ -814,6 +814,16 @@ impl Server { Ok(()) } + fn semantic_tokens_full(&mut self, id: RequestId, params: SemanticTokensParams) -> Result<()> { + self.run_with_db(id, move |db| { + let Some(document) = Workspace::get(db).lookup_uri(db, ¶ms.text_document.uri) else { return None }; + let range = document.line_index(db).line_col_lsp_range(TextRange::new(0.into(), document.text(db).text_len())); + semantic_tokens::find_all(db, ¶ms.text_document.uri, range) + }); + + Ok(()) + } + fn semantic_tokens_range( &mut self, id: RequestId, @@ -917,6 +927,9 @@ impl Server { .on::(|id, params| { self.semantic_tokens_range(id, params) })? + .on::(|id, params| { + self.semantic_tokens_full(id, params) + })? .on::(|id,params| { self.inlay_hints(id, params) })? From b6b2215406167f20f27aa3ee373671abc559186a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20F=C3=B6rster?= Date: Sat, 18 Mar 2023 11:19:01 +0100 Subject: [PATCH 3/9] Fix delta offset calculation --- src/features/semantic_tokens.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/features/semantic_tokens.rs b/src/features/semantic_tokens.rs index 7c6203b5b..6ca262110 100644 --- a/src/features/semantic_tokens.rs +++ b/src/features/semantic_tokens.rs @@ -79,7 +79,7 @@ impl TokenBuilder { }); } else { let delta_line = 0; - let delta_start = last_pos.character - range.start.character; + let delta_start = range.start.character - last_pos.character; data.push(SemanticToken { delta_line, delta_start, @@ -89,7 +89,7 @@ impl TokenBuilder { }); } - last_pos = range.end; + last_pos = range.start; } SemanticTokens { From 5d0d7d02009bbf6de6a33cf3b2959cae9dfc18c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20F=C3=B6rster?= Date: Sat, 18 Mar 2023 11:19:22 +0100 Subject: [PATCH 4/9] Mark TeX math delimiters as deprecated --- src/features/semantic_tokens.rs | 10 ++++++- .../semantic_tokens/math_delimiter.rs | 30 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 src/features/semantic_tokens/math_delimiter.rs diff --git a/src/features/semantic_tokens.rs b/src/features/semantic_tokens.rs index 6ca262110..f66074045 100644 --- a/src/features/semantic_tokens.rs +++ b/src/features/semantic_tokens.rs @@ -1,4 +1,5 @@ mod label; +mod math_delimiter; use bitflags::bitflags; use lsp_types::{ @@ -17,6 +18,7 @@ use crate::{ #[repr(u32)] pub enum TokenKind { Label = 0, + MathDelimiter = 1, } bitflags! { @@ -25,15 +27,20 @@ bitflags! { const NONE = 0; const UNDEFINED = 1; const UNUSED = 2; + const DEPRECATED = 4; } } pub fn legend() -> SemanticTokensLegend { SemanticTokensLegend { - token_types: vec![SemanticTokenType::new("label")], + token_types: vec![ + SemanticTokenType::new("label"), + SemanticTokenType::new("mathDelimiter"), + ], token_modifiers: vec![ SemanticTokenModifier::new("undefined"), SemanticTokenModifier::new("unused"), + SemanticTokenModifier::new("deprecated"), ], } } @@ -105,5 +112,6 @@ pub fn find_all(db: &dyn Db, uri: &Url, viewport: Range) -> Option Option<()> { + let root = document.parse(db).as_tex()?.root(db); + + for token in root + .covering_element(viewport) + .as_node()? + .descendants_with_tokens() + .filter_map(|elem| elem.into_token()) + .filter(|token| token.kind() == latex::DOLLAR) + { + builder.push(Token { + range: token.text_range(), + kind: TokenKind::MathDelimiter, + modifiers: TokenModifiers::DEPRECATED, + }); + } + + Some(()) +} From 88730dc9d3820e9aadde36a279b1a23256a2f59e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20F=C3=B6rster?= Date: Sat, 18 Mar 2023 11:34:36 +0100 Subject: [PATCH 5/9] Refactor full highlight request --- src/features/semantic_tokens.rs | 73 ++++++++++++------- src/features/semantic_tokens/label.rs | 20 ++--- .../semantic_tokens/math_delimiter.rs | 18 ++--- src/server.rs | 8 +- 4 files changed, 62 insertions(+), 57 deletions(-) diff --git a/src/features/semantic_tokens.rs b/src/features/semantic_tokens.rs index f66074045..875eb7108 100644 --- a/src/features/semantic_tokens.rs +++ b/src/features/semantic_tokens.rs @@ -6,24 +6,24 @@ use lsp_types::{ Position, Range, SemanticToken, SemanticTokenModifier, SemanticTokenType, SemanticTokens, SemanticTokensLegend, Url, }; -use rowan::TextRange; +use rowan::{TextLen, TextRange}; use crate::{ - db::Workspace, + db::{Document, Workspace}, util::{line_index::LineIndex, line_index_ext::LineIndexExt}, Db, }; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] #[repr(u32)] -pub enum TokenKind { +enum TokenKind { Label = 0, MathDelimiter = 1, } bitflags! { #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] - pub struct TokenModifiers: u32 { + struct TokenModifiers: u32 { const NONE = 0; const UNDEFINED = 1; const UNUSED = 2; @@ -31,29 +31,15 @@ bitflags! { } } -pub fn legend() -> SemanticTokensLegend { - SemanticTokensLegend { - token_types: vec![ - SemanticTokenType::new("label"), - SemanticTokenType::new("mathDelimiter"), - ], - token_modifiers: vec![ - SemanticTokenModifier::new("undefined"), - SemanticTokenModifier::new("unused"), - SemanticTokenModifier::new("deprecated"), - ], - } -} - #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] -pub struct Token { - pub range: TextRange, - pub kind: TokenKind, - pub modifiers: TokenModifiers, +struct Token { + range: TextRange, + kind: TokenKind, + modifiers: TokenModifiers, } #[derive(Debug, Default)] -pub struct TokenBuilder { +struct TokenBuilder { tokens: Vec, } @@ -106,12 +92,45 @@ impl TokenBuilder { } } -pub fn find_all(db: &dyn Db, uri: &Url, viewport: Range) -> Option { +#[derive(Clone, Copy)] +struct Context<'db> { + db: &'db dyn Db, + document: Document, + viewport: TextRange, +} + +pub fn legend() -> SemanticTokensLegend { + SemanticTokensLegend { + token_types: vec![ + SemanticTokenType::new("label"), + SemanticTokenType::new("mathDelimiter"), + ], + token_modifiers: vec![ + SemanticTokenModifier::new("undefined"), + SemanticTokenModifier::new("unused"), + SemanticTokenModifier::new("deprecated"), + ], + } +} + +pub fn find_all(db: &dyn Db, uri: &Url, viewport: Option) -> Option { let workspace = Workspace::get(db); let document = workspace.lookup_uri(db, uri)?; - let viewport = document.line_index(db).offset_lsp_range(viewport); + let viewport = viewport.map_or_else( + || TextRange::new(0.into(), document.text(db).text_len()), + |range| document.line_index(db).offset_lsp_range(range), + ); + + let context = Context { + db, + document, + viewport, + }; + let mut builder = TokenBuilder::default(); - label::find(db, document, viewport, &mut builder); - math_delimiter::find(db, document, viewport, &mut builder); + + label::find(context, &mut builder); + math_delimiter::find(context, &mut builder); + Some(builder.finish(document.line_index(db))) } diff --git a/src/features/semantic_tokens/label.rs b/src/features/semantic_tokens/label.rs index c542a05b5..4e7730972 100644 --- a/src/features/semantic_tokens/label.rs +++ b/src/features/semantic_tokens/label.rs @@ -1,34 +1,28 @@ -use rowan::TextRange; - use crate::{ db::{analysis::label, Document, Workspace}, Db, }; -use super::{Token, TokenBuilder, TokenKind, TokenModifiers}; +use super::{Context, Token, TokenBuilder, TokenKind, TokenModifiers}; -pub fn find( - db: &dyn Db, - document: Document, - viewport: TextRange, - builder: &mut TokenBuilder, -) -> Option<()> { - let labels = document.parse(db).as_tex()?.analyze(db).labels(db); +pub(super) fn find(context: Context, builder: &mut TokenBuilder) -> Option<()> { + let db = context.db; + let labels = context.document.parse(db).as_tex()?.analyze(db).labels(db); for label in labels .iter() - .filter(|label| viewport.intersect(label.range(db)).is_some()) + .filter(|label| context.viewport.intersect(label.range(db)).is_some()) { let name = label.name(db).text(db); let modifiers = match label.origin(db) { label::Origin::Definition(_) => { - if !is_label_referenced(db, document, name) { + if !is_label_referenced(db, context.document, name) { TokenModifiers::UNUSED } else { TokenModifiers::NONE } } label::Origin::Reference(_) | label::Origin::ReferenceRange(_) => { - if !is_label_defined(db, document, name) { + if !is_label_defined(db, context.document, name) { TokenModifiers::UNDEFINED } else { TokenModifiers::NONE diff --git a/src/features/semantic_tokens/math_delimiter.rs b/src/features/semantic_tokens/math_delimiter.rs index b1787aab8..aaba21bac 100644 --- a/src/features/semantic_tokens/math_delimiter.rs +++ b/src/features/semantic_tokens/math_delimiter.rs @@ -1,19 +1,13 @@ -use rowan::TextRange; +use crate::syntax::latex; -use crate::{db::Document, syntax::latex, Db}; +use super::{Context, Token, TokenBuilder, TokenKind, TokenModifiers}; -use super::{Token, TokenBuilder, TokenKind, TokenModifiers}; - -pub fn find( - db: &dyn Db, - document: Document, - viewport: TextRange, - builder: &mut TokenBuilder, -) -> Option<()> { - let root = document.parse(db).as_tex()?.root(db); +pub(super) fn find(context: Context, builder: &mut TokenBuilder) -> Option<()> { + let db = context.db; + let root = context.document.parse(db).as_tex()?.root(db); for token in root - .covering_element(viewport) + .covering_element(context.viewport) .as_node()? .descendants_with_tokens() .filter_map(|elem| elem.into_token()) diff --git a/src/server.rs b/src/server.rs index 102064a90..65b58f71f 100644 --- a/src/server.rs +++ b/src/server.rs @@ -13,7 +13,7 @@ use log::{error, info}; use lsp_server::{Connection, ErrorCode, Message, RequestId}; use lsp_types::{notification::*, request::*, *}; use once_cell::sync::Lazy; -use rowan::{ast::AstNode, TextLen, TextRange, TextSize}; +use rowan::{ast::AstNode, TextSize}; use rustc_hash::FxHashSet; use serde::{Deserialize, Serialize}; use serde_repr::{Deserialize_repr, Serialize_repr}; @@ -816,9 +816,7 @@ impl Server { fn semantic_tokens_full(&mut self, id: RequestId, params: SemanticTokensParams) -> Result<()> { self.run_with_db(id, move |db| { - let Some(document) = Workspace::get(db).lookup_uri(db, ¶ms.text_document.uri) else { return None }; - let range = document.line_index(db).line_col_lsp_range(TextRange::new(0.into(), document.text(db).text_len())); - semantic_tokens::find_all(db, ¶ms.text_document.uri, range) + semantic_tokens::find_all(db, ¶ms.text_document.uri, None) }); Ok(()) @@ -830,7 +828,7 @@ impl Server { params: SemanticTokensRangeParams, ) -> Result<()> { self.run_with_db(id, move |db| { - semantic_tokens::find_all(db, ¶ms.text_document.uri, params.range) + semantic_tokens::find_all(db, ¶ms.text_document.uri, Some(params.range)) }); Ok(()) From b78907ffecaef658cdcd82445288b6a4c9126bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20F=C3=B6rster?= Date: Sat, 18 Mar 2023 11:37:12 +0100 Subject: [PATCH 6/9] Only mark $$ as deprecated --- src/features/semantic_tokens/math_delimiter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/features/semantic_tokens/math_delimiter.rs b/src/features/semantic_tokens/math_delimiter.rs index aaba21bac..109822d0b 100644 --- a/src/features/semantic_tokens/math_delimiter.rs +++ b/src/features/semantic_tokens/math_delimiter.rs @@ -11,7 +11,7 @@ pub(super) fn find(context: Context, builder: &mut TokenBuilder) -> Option<()> { .as_node()? .descendants_with_tokens() .filter_map(|elem| elem.into_token()) - .filter(|token| token.kind() == latex::DOLLAR) + .filter(|token| token.kind() == latex::DOLLAR && token.text() == "$$") { builder.push(Token { range: token.text_range(), From e1c4a20a0319d474e4c75ce549fe0bb15f874b52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20F=C3=B6rster?= Date: Mon, 20 Mar 2023 16:35:15 +0100 Subject: [PATCH 7/9] Highlight undefined citations --- src/db/analysis.rs | 22 ++++++++++++ src/features/semantic_tokens.rs | 6 +++- src/features/semantic_tokens/citations.rs | 44 +++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 src/features/semantic_tokens/citations.rs diff --git a/src/db/analysis.rs b/src/db/analysis.rs index 1f1bd508a..4101b2f34 100644 --- a/src/db/analysis.rs +++ b/src/db/analysis.rs @@ -104,6 +104,12 @@ impl TheoremEnvironment { } } +#[derive(Debug, PartialEq, Eq, Clone, Hash)] +pub struct TexCitation { + pub key: String, + pub range: TextRange, +} + #[salsa::tracked] pub struct GraphicsPath { #[return_ref] @@ -130,6 +136,9 @@ pub struct TexAnalysis { #[return_ref] pub links: Vec, + #[return_ref] + pub citations: Vec, + #[return_ref] pub labels: Vec, @@ -162,6 +171,7 @@ impl TexAnalysis { impl TexAnalysis { pub(super) fn analyze(db: &dyn Db, root: &latex::SyntaxNode) -> Self { let mut links = Vec::new(); + let mut citations = Vec::new(); let mut labels = Vec::new(); let mut label_numbers = Vec::new(); let mut theorem_environments = Vec::new(); @@ -178,6 +188,17 @@ impl TexAnalysis { NodeOrToken::Node(node) => { TexLink::of_include(db, node.clone(), &mut links) .or_else(|| TexLink::of_import(db, node.clone(), &mut links)) + .or_else(|| { + let citation = latex::Citation::cast(node.clone())?; + for key in citation.key_list()?.keys() { + citations.push(TexCitation { + key: key.to_string(), + range: latex::small_range(&key), + }); + } + + Some(()) + }) .or_else(|| label::Name::of_definition(db, node.clone(), &mut labels)) .or_else(|| label::Name::of_reference(db, node.clone(), &mut labels)) .or_else(|| label::Name::of_reference_range(db, node.clone(), &mut labels)) @@ -210,6 +231,7 @@ impl TexAnalysis { Self::new( db, links, + citations, labels, label_numbers, theorem_environments, diff --git a/src/features/semantic_tokens.rs b/src/features/semantic_tokens.rs index 875eb7108..05c86e7b6 100644 --- a/src/features/semantic_tokens.rs +++ b/src/features/semantic_tokens.rs @@ -1,3 +1,4 @@ +mod citations; mod label; mod math_delimiter; @@ -18,7 +19,8 @@ use crate::{ #[repr(u32)] enum TokenKind { Label = 0, - MathDelimiter = 1, + Citation = 2, + MathDelimiter = 3, } bitflags! { @@ -103,6 +105,7 @@ pub fn legend() -> SemanticTokensLegend { SemanticTokensLegend { token_types: vec![ SemanticTokenType::new("label"), + SemanticTokenType::new("citation"), SemanticTokenType::new("mathDelimiter"), ], token_modifiers: vec![ @@ -130,6 +133,7 @@ pub fn find_all(db: &dyn Db, uri: &Url, viewport: Option) -> Option Option<()> { + let db = context.db; + let analysis = context.document.parse(db).as_tex()?.analyze(db); + for citation in analysis + .citations(db) + .iter() + .filter(|citation| context.viewport.intersect(citation.range).is_some()) + { + let modifiers = if !is_entry_defined(db, context.document, &citation.key) { + TokenModifiers::UNDEFINED + } else { + TokenModifiers::NONE + }; + + builder.push(Token { + range: citation.range, + kind: TokenKind::Citation, + modifiers, + }); + } + + Some(()) +} + +fn is_entry_defined(db: &dyn Db, child: Document, key: &str) -> bool { + Workspace::get(db) + .related(db, child) + .iter() + .filter_map(|document| document.parse(db).as_bib()) + .flat_map(|data| data.root(db).children()) + .filter_map(bibtex::Entry::cast) + .filter_map(|entry| entry.name_token()) + .any(|token| token.text() == key) +} From 0d5b0aacd08168001056b2e449948e544ccec9a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20F=C3=B6rster?= Date: Tue, 21 Mar 2023 20:53:42 +0100 Subject: [PATCH 8/9] Use different token types depending on labeled obj --- src/features/semantic_tokens.rs | 18 +++++-- src/features/semantic_tokens/label.rs | 68 +++++++++++++++++++-------- 2 files changed, 62 insertions(+), 24 deletions(-) diff --git a/src/features/semantic_tokens.rs b/src/features/semantic_tokens.rs index 05c86e7b6..eac0eb619 100644 --- a/src/features/semantic_tokens.rs +++ b/src/features/semantic_tokens.rs @@ -18,9 +18,14 @@ use crate::{ #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] #[repr(u32)] enum TokenKind { - Label = 0, - Citation = 2, - MathDelimiter = 3, + GenericLabel = 0, + SectionLabel, + FloatLabel, + TheoremLabel, + EquationLabel, + EnumItemLabel, + Citation, + MathDelimiter, } bitflags! { @@ -104,7 +109,12 @@ struct Context<'db> { pub fn legend() -> SemanticTokensLegend { SemanticTokensLegend { token_types: vec![ - SemanticTokenType::new("label"), + SemanticTokenType::new("genericLabel"), + SemanticTokenType::new("sectionLabel"), + SemanticTokenType::new("floatLabel"), + SemanticTokenType::new("theoremLabel"), + SemanticTokenType::new("equationLabel"), + SemanticTokenType::new("enumItemLabel"), SemanticTokenType::new("citation"), SemanticTokenType::new("mathDelimiter"), ], diff --git a/src/features/semantic_tokens/label.rs b/src/features/semantic_tokens/label.rs index 4e7730972..7e14d77f8 100644 --- a/src/features/semantic_tokens/label.rs +++ b/src/features/semantic_tokens/label.rs @@ -1,6 +1,6 @@ use crate::{ db::{analysis::label, Document, Workspace}, - Db, + util, Db, }; use super::{Context, Token, TokenBuilder, TokenKind, TokenModifiers}; @@ -11,29 +11,14 @@ pub(super) fn find(context: Context, builder: &mut TokenBuilder) -> Option<()> { for label in labels .iter() .filter(|label| context.viewport.intersect(label.range(db)).is_some()) + .copied() { - let name = label.name(db).text(db); - let modifiers = match label.origin(db) { - label::Origin::Definition(_) => { - if !is_label_referenced(db, context.document, name) { - TokenModifiers::UNUSED - } else { - TokenModifiers::NONE - } - } - label::Origin::Reference(_) | label::Origin::ReferenceRange(_) => { - if !is_label_defined(db, context.document, name) { - TokenModifiers::UNDEFINED - } else { - TokenModifiers::NONE - } - } - }; - + let kind = token_type(context, label); + let modifiers = token_modifiers(context, label); let range = label.range(db); builder.push(Token { range, - kind: TokenKind::Label, + kind, modifiers, }); } @@ -41,6 +26,49 @@ pub(super) fn find(context: Context, builder: &mut TokenBuilder) -> Option<()> { Some(()) } +fn token_type(context: Context, label: label::Name) -> TokenKind { + let db = context.db; + let definition = match label.origin(db) { + label::Origin::Definition(_) => Some((context.document, label)), + label::Origin::Reference(_) | label::Origin::ReferenceRange(_) => { + util::label::find_label_definition(db, context.document, label.name(db)) + } + }; + + match definition + .and_then(|(doc, label)| util::label::render(db, doc, label)) + .map(|label| label.object) + { + Some(util::label::LabeledObject::Section { .. }) => TokenKind::SectionLabel, + Some(util::label::LabeledObject::Float { .. }) => TokenKind::FloatLabel, + Some(util::label::LabeledObject::EnumItem { .. }) => TokenKind::EnumItemLabel, + Some(util::label::LabeledObject::Equation { .. }) => TokenKind::EquationLabel, + Some(util::label::LabeledObject::Theorem { .. }) => TokenKind::TheoremLabel, + None => TokenKind::GenericLabel, + } +} + +fn token_modifiers(context: Context, label: label::Name) -> TokenModifiers { + let db = context.db; + let name = label.name(db).text(db); + match label.origin(db) { + label::Origin::Definition(_) => { + if !is_label_referenced(db, context.document, name) { + TokenModifiers::UNUSED + } else { + TokenModifiers::NONE + } + } + label::Origin::Reference(_) | label::Origin::ReferenceRange(_) => { + if !is_label_defined(db, context.document, name) { + TokenModifiers::UNDEFINED + } else { + TokenModifiers::NONE + } + } + } +} + fn is_label_defined(db: &dyn Db, child: Document, name: &str) -> bool { Workspace::get(db) .related(db, child) From ef71c69db0e3df6e455cf473e06cbfe27e1f5076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20F=C3=B6rster?= Date: Fri, 24 Mar 2023 21:55:28 +0100 Subject: [PATCH 9/9] Classify citation tokens depending of entry type --- src/features/semantic_tokens.rs | 55 ++++++++++++++--------- src/features/semantic_tokens/citations.rs | 52 +++++++++++++-------- 2 files changed, 68 insertions(+), 39 deletions(-) diff --git a/src/features/semantic_tokens.rs b/src/features/semantic_tokens.rs index eac0eb619..9cab23420 100644 --- a/src/features/semantic_tokens.rs +++ b/src/features/semantic_tokens.rs @@ -24,7 +24,14 @@ enum TokenKind { TheoremLabel, EquationLabel, EnumItemLabel, - Citation, + + GenericCitation, + ArticleCitation, + BookCitation, + CollectionCitation, + PartCitation, + ThesisCitation, + MathDelimiter, } @@ -38,6 +45,31 @@ bitflags! { } } +pub fn legend() -> SemanticTokensLegend { + SemanticTokensLegend { + token_types: vec![ + SemanticTokenType::new("genericLabel"), + SemanticTokenType::new("sectionLabel"), + SemanticTokenType::new("floatLabel"), + SemanticTokenType::new("theoremLabel"), + SemanticTokenType::new("equationLabel"), + SemanticTokenType::new("enumItemLabel"), + SemanticTokenType::new("genericCitation"), + SemanticTokenType::new("articleCitation"), + SemanticTokenType::new("bookCitation"), + SemanticTokenType::new("collectionCitation"), + SemanticTokenType::new("partCitation"), + SemanticTokenType::new("thesisCitation"), + SemanticTokenType::new("mathDelimiter"), + ], + token_modifiers: vec![ + SemanticTokenModifier::new("undefined"), + SemanticTokenModifier::new("unused"), + SemanticTokenModifier::new("deprecated"), + ], + } +} + #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] struct Token { range: TextRange, @@ -52,6 +84,7 @@ struct TokenBuilder { impl TokenBuilder { pub fn push(&mut self, token: Token) { + log::info!("Adding sem token {token:#?}"); self.tokens.push(token); } @@ -106,26 +139,6 @@ struct Context<'db> { viewport: TextRange, } -pub fn legend() -> SemanticTokensLegend { - SemanticTokensLegend { - token_types: vec![ - SemanticTokenType::new("genericLabel"), - SemanticTokenType::new("sectionLabel"), - SemanticTokenType::new("floatLabel"), - SemanticTokenType::new("theoremLabel"), - SemanticTokenType::new("equationLabel"), - SemanticTokenType::new("enumItemLabel"), - SemanticTokenType::new("citation"), - SemanticTokenType::new("mathDelimiter"), - ], - token_modifiers: vec![ - SemanticTokenModifier::new("undefined"), - SemanticTokenModifier::new("unused"), - SemanticTokenModifier::new("deprecated"), - ], - } -} - pub fn find_all(db: &dyn Db, uri: &Url, viewport: Option) -> Option { let workspace = Workspace::get(db); let document = workspace.lookup_uri(db, uri)?; diff --git a/src/features/semantic_tokens/citations.rs b/src/features/semantic_tokens/citations.rs index c2a6c4248..b1c71e448 100644 --- a/src/features/semantic_tokens/citations.rs +++ b/src/features/semantic_tokens/citations.rs @@ -1,9 +1,9 @@ use rowan::ast::AstNode; use crate::{ - db::{Document, Workspace}, - syntax::bibtex::{self, HasName}, - Db, + db::Workspace, + syntax::bibtex::{self, HasName, HasType}, + util::lang_data::{BibtexEntryTypeCategory, LANGUAGE_DATA}, }; use super::{Context, Token, TokenBuilder, TokenKind, TokenModifiers}; @@ -16,29 +16,45 @@ pub(super) fn find(context: Context, builder: &mut TokenBuilder) -> Option<()> { .iter() .filter(|citation| context.viewport.intersect(citation.range).is_some()) { - let modifiers = if !is_entry_defined(db, context.document, &citation.key) { - TokenModifiers::UNDEFINED - } else { + let entry = Workspace::get(db) + .related(db, context.document) + .iter() + .filter_map(|document| document.parse(db).as_bib()) + .flat_map(|data| data.root(db).children()) + .filter_map(bibtex::Entry::cast) + .find(|entry| { + entry + .name_token() + .map_or(false, |name| name.text() == &citation.key) + }); + + let modifiers = if entry.is_some() { TokenModifiers::NONE + } else { + TokenModifiers::UNDEFINED + }; + + let kind = match entry + .and_then(|entry| entry.type_token()) + .and_then(|token| LANGUAGE_DATA.find_entry_type(&token.text()[1..])) + .map(|doc| doc.category) + { + Some(BibtexEntryTypeCategory::String) => unreachable!(), + Some(BibtexEntryTypeCategory::Misc) => TokenKind::GenericCitation, + Some(BibtexEntryTypeCategory::Article) => TokenKind::ArticleCitation, + Some(BibtexEntryTypeCategory::Book) => TokenKind::BookCitation, + Some(BibtexEntryTypeCategory::Part) => TokenKind::PartCitation, + Some(BibtexEntryTypeCategory::Thesis) => TokenKind::ThesisCitation, + Some(BibtexEntryTypeCategory::Collection) => TokenKind::CollectionCitation, + None => TokenKind::GenericCitation, }; builder.push(Token { range: citation.range, - kind: TokenKind::Citation, + kind, modifiers, }); } Some(()) } - -fn is_entry_defined(db: &dyn Db, child: Document, key: &str) -> bool { - Workspace::get(db) - .related(db, child) - .iter() - .filter_map(|document| document.parse(db).as_bib()) - .flat_map(|data| data.root(db).children()) - .filter_map(bibtex::Entry::cast) - .filter_map(|entry| entry.name_token()) - .any(|token| token.text() == key) -}