Skip to content

Commit 04ed07d

Browse files
siddhant-deepsourcesiddhant-deepsource
authored
Update rust lexer (#548)
1 parent d1f9876 commit 04ed07d

File tree

2 files changed

+28
-35
lines changed

2 files changed

+28
-35
lines changed

Diff for: ‎lexers/r/rust.go

+25-31
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ import (
99
var Rust = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "Rust",
12-
Aliases: []string{"rust"},
12+
Aliases: []string{"rust", "rs"},
1313
Filenames: []string{"*.rs", "*.rs.in"},
14-
MimeTypes: []string{"text/rust"},
14+
MimeTypes: []string{"text/rust", "text/x-rust"},
1515
EnsureNL: true,
1616
},
1717
rustRules,
@@ -35,20 +35,22 @@ func rustRules() Rules {
3535
{`r#*"(?:\\.|[^\\;])*"#*`, LiteralString, nil},
3636
{`"(?:\\.|[^\\"])*"`, LiteralString, nil},
3737
{`\$([a-zA-Z_]\w*|\(,?|\),?|,?)`, CommentPreproc, nil},
38-
{Words(``, `\b`, `as`, `async`, `await`, `const`, `crate`, `else`, `extern`, `for`, `if`, `impl`, `in`, `loop`, `match`, `move`, `mut`, `pub`, `ref`, `return`, `static`, `super`, `trait`, `unsafe`, `use`, `where`, `while`), Keyword, nil},
39-
{Words(``, `\b`, `abstract`, `become`, `box`, `do`, `final`, `macro`, `override`, `priv`, `try`, `typeof`, `unsized`, `virtual`, `yield`), KeywordReserved, nil},
38+
{Words(``, `\b`, `as`, `async`, `await`, `box`, `const`, `crate`, `dyn`, `else`, `extern`, `for`, `if`, `impl`, `in`, `loop`, `match`, `move`, `mut`, `pub`, `ref`, `return`, `static`, `super`, `trait`, `unsafe`, `use`, `where`, `while`), Keyword, nil},
39+
{Words(``, `\b`, `abstract`, `become`, `do`, `final`, `macro`, `override`, `priv`, `typeof`, `try`, `unsized`, `virtual`, `yield`), KeywordReserved, nil},
4040
{`(true|false)\b`, KeywordConstant, nil},
41+
{`self\b`, NameBuiltinPseudo, nil},
4142
{`mod\b`, Keyword, Push("modname")},
4243
{`let\b`, KeywordDeclaration, nil},
4344
{`fn\b`, Keyword, Push("funcname")},
4445
{`(struct|enum|type|union)\b`, Keyword, Push("typename")},
4546
{`(default)(\s+)(type|fn)\b`, ByGroups(Keyword, Text, Keyword), nil},
46-
{Words(``, `\b`, `u8`, `u16`, `u32`, `u64`, `u128`, `i8`, `i16`, `i32`, `i64`, `i128`, `usize`, `isize`, `f32`, `f64`, `str`, `bool`), KeywordType, nil},
47-
{`self\b`, NameBuiltinPseudo, nil},
48-
{Words(``, `\b`, `Copy`, `Send`, `Sized`, `Sync`, `Drop`, `Fn`, `FnMut`, `FnOnce`, `Box`, `ToOwned`, `Clone`, `PartialEq`, `PartialOrd`, `Eq`, `Ord`, `AsRef`, `AsMut`, `Into`, `From`, `Default`, `Iterator`, `Extend`, `IntoIterator`, `DoubleEndedIterator`, `ExactSizeIterator`, `Option`, `Some`, `None`, `Result`, `Ok`, `Err`, `SliceConcatExt`, `String`, `ToString`, `Vec`), NameBuiltin, nil},
47+
{Words(``, `\b`, `u8`, `u16`, `u32`, `u64`, `u128`, `i8`, `i16`, `i32`, `i64`, `i128`, `usize`, `isize`, `f32`, `f64`, `char`, `str`, `bool`), KeywordType, nil},
48+
{`[sS]elf\b`, NameBuiltinPseudo, nil},
49+
{Words(``, `\b`, `Copy`, `Send`, `Sized`, `Sync`, `Unpin`, `Drop`, `Fn`, `FnMut`, `FnOnce`, `drop`, `Box`, `ToOwned`, `Clone`, `PartialEq`, `PartialOrd`, `Eq`, `Ord`, `AsRef`, `AsMut`, `Into`, `From`, `Default`, `Iterator`, `Extend`, `IntoIterator`, `DoubleEndedIterator`, `ExactSizeIterator`, `Option`, `Some`, `None`, `Result`, `Ok`, `Err`, `String`, `ToString`, `Vec`), NameBuiltin, nil},
50+
{Words(``, `!`, `asm`, `assert`, `assert_eq`, `assert_ne`, `cfg`, `column`, `compile_error`, `concat`, `concat_idents`, `dbg`, `debug_assert`, `debug_assert_eq`, `debug_assert_ne`, `env`, `eprint`, `eprintln`, `file`, `format`, `format_args`, `format_args_nl`, `global_asm`, `include`, `include_bytes`, `include_str`, `is_aarch64_feature_detected`, `is_arm_feature_detected`, `is_mips64_feature_detected`, `is_mips_feature_detected`, `is_powerpc64_feature_detected`, `is_powerpc_feature_detected`, `is_x86_feature_detected`, `line`, `llvm_asm`, `log_syntax`, `macro_rules`, `matches`, `module_path`, `option_env`, `panic`, `print`, `println`, `stringify`, `thread_local`, `todo`, `trace_macros`, `unimplemented`, `unreachable`, `vec`, `write`, `writeln`), NameFunctionMagic, nil},
4951
{`::\b`, Text, nil},
5052
{`(?::|->)`, Text, Push("typename")},
51-
{`(break|continue)(\s*)(\'[A-Za-z_]\w*)?`, ByGroups(Keyword, TextWhitespace, NameLabel), nil},
53+
{`(break|continue)(\b\s*)(\'[A-Za-z_]\w*)?`, ByGroups(Keyword, TextWhitespace, NameLabel), nil},
5254
{`'(\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0|\\u\{[0-9a-fA-F]{1,6}\}|.)'`, LiteralStringChar, nil},
5355
{`b'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\0|\\u\{[0-9a-fA-F]{1,6}\}|.)'`, LiteralStringChar, nil},
5456
{`0b[01_]+`, LiteralNumberBin, Push("number_lit")},
@@ -57,15 +59,15 @@ func rustRules() Rules {
5759
{`[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|\.[0-9_]*(?!\.)|[eE][+\-]?[0-9_]+)`, LiteralNumberFloat, Push("number_lit")},
5860
{`[0-9][0-9_]*`, LiteralNumberInteger, Push("number_lit")},
5961
{`b"`, LiteralString, Push("bytestring")},
60-
{`b?r(#*)".*?"\1`, LiteralString, nil},
61-
{`'static`, NameBuiltin, nil},
62-
{`'[a-zA-Z_]\w*`, NameAttribute, nil},
62+
{`(?s)b?r(#*)".*?"\1`, LiteralString, nil},
63+
{`'`, Operator, Push("lifetime")},
64+
{`\.\.=?`, Operator, nil},
6365
{`[{}()\[\],.;]`, Punctuation, nil},
6466
{`[+\-*/%&|<>^!~@=:?]`, Operator, nil},
6567
{`(r#)?[a-zA-Z_]\w*`, Name, nil},
68+
{`r#[a-zA-Z_]\w*`, Name, nil},
6669
{`#!?\[`, CommentPreproc, Push("attribute[")},
67-
{`([A-Za-z_]\w*)(!)(\s*)([A-Za-z_]\w*)?(\s*)(\{)`, ByGroups(CommentPreproc, Punctuation, TextWhitespace, Name, TextWhitespace, Punctuation), Push("macro{")},
68-
{`([A-Za-z_]\w*)(!)(\s*)([A-Za-z_]\w*)?(\()`, ByGroups(CommentPreproc, Punctuation, TextWhitespace, Name, Punctuation), Push("macro(")},
70+
{`#`, Text, nil},
6971
},
7072
"comment": {
7173
{`[^*/]+`, CommentMultiline, nil},
@@ -92,11 +94,17 @@ func rustRules() Rules {
9294
"typename": {
9395
{`\s+`, Text, nil},
9496
{`&`, KeywordPseudo, nil},
95-
{Words(``, `\b`, `Copy`, `Send`, `Sized`, `Sync`, `Drop`, `Fn`, `FnMut`, `FnOnce`, `Box`, `ToOwned`, `Clone`, `PartialEq`, `PartialOrd`, `Eq`, `Ord`, `AsRef`, `AsMut`, `Into`, `From`, `Default`, `Iterator`, `Extend`, `IntoIterator`, `DoubleEndedIterator`, `ExactSizeIterator`, `Option`, `Some`, `None`, `Result`, `Ok`, `Err`, `SliceConcatExt`, `String`, `ToString`, `Vec`), NameBuiltin, nil},
96-
{Words(``, `\b`, `u8`, `u16`, `u32`, `u64`, `i8`, `i16`, `i32`, `i64`, `usize`, `isize`, `f32`, `f64`, `str`, `bool`), KeywordType, nil},
97+
{`'`, Operator, Push("lifetime")},
98+
{Words(``, `\b`, `Copy`, `Send`, `Sized`, `Sync`, `Unpin`, `Drop`, `Fn`, `FnMut`, `FnOnce`, `drop`, `Box`, `ToOwned`, `Clone`, `PartialEq`, `PartialOrd`, `Eq`, `Ord`, `AsRef`, `AsMut`, `Into`, `From`, `Default`, `Iterator`, `Extend`, `IntoIterator`, `DoubleEndedIterator`, `ExactSizeIterator`, `Option`, `Some`, `None`, `Result`, `Ok`, `Err`, `String`, `ToString`, `Vec`), NameBuiltin, nil},
99+
{Words(``, `\b`, `u8`, `u16`, `u32`, `u64`, `u128`, `i8`, `i16`, `i32`, `i64`, `i128`, `usize`, `isize`, `f32`, `f64`, `char`, `str`, `bool`), KeywordType, nil},
97100
{`[a-zA-Z_]\w*`, NameClass, Pop(1)},
98101
Default(Pop(1)),
99102
},
103+
"lifetime": {
104+
{`(static|_)`, NameBuiltin, nil},
105+
{`[a-zA-Z_]+\w*`, NameAttribute, nil},
106+
Default(Pop(1)),
107+
},
100108
"number_lit": {
101109
{`[ui](8|16|32|64|size)`, Keyword, Pop(1)},
102110
{`f(32|64)`, Keyword, Pop(1)},
@@ -112,28 +120,14 @@ func rustRules() Rules {
112120
{`\\x[89a-fA-F][0-9a-fA-F]`, LiteralStringEscape, nil},
113121
Include("string"),
114122
},
115-
"macro{": {
116-
{`\{`, Operator, Push()},
117-
{`\}`, Operator, Pop(1)},
118-
},
119-
"macro(": {
120-
{`\(`, Operator, Push()},
121-
{`\)`, Operator, Pop(1)},
122-
},
123123
"attribute_common": {
124124
{`"`, LiteralString, Push("string")},
125125
{`\[`, CommentPreproc, Push("attribute[")},
126-
{`\(`, CommentPreproc, Push("attribute(")},
127126
},
128127
"attribute[": {
129128
Include("attribute_common"),
130-
{`\];?`, CommentPreproc, Pop(1)},
131-
{`[^"\]]+`, CommentPreproc, nil},
132-
},
133-
"attribute(": {
134-
Include("attribute_common"),
135-
{`\);?`, CommentPreproc, Pop(1)},
136-
{`[^")]+`, CommentPreproc, nil},
129+
{`\]`, CommentPreproc, Pop(1)},
130+
{`[^"\]\[]+`, CommentPreproc, nil},
137131
},
138132
}
139133
}

Diff for: ‎lexers/testdata/rust.expected

+3-4
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,7 @@
5858
{"type":"TextWhitespace","value":"\n\n "},
5959
{"type":"CommentSingle","value":"// Some comment\n"},
6060
{"type":"TextWhitespace","value":" "},
61-
{"type":"Name","value":"println"},
62-
{"type":"Operator","value":"!"},
61+
{"type":"NameFunctionMagic","value":"println!"},
6362
{"type":"Punctuation","value":"("},
6463
{"type":"LiteralString","value":"\"rect1 is {:?}\""},
6564
{"type":"Punctuation","value":","},
@@ -80,8 +79,8 @@
8079
{"type":"TextWhitespace","value":" "},
8180
{"type":"Name","value":"i"},
8281
{"type":"Text","value":": "},
83-
{"type":"NameClass","value":"i128"},
84-
{"type":"TextWhitespace","value":" "},
82+
{"type":"KeywordType","value":"i128"},
83+
{"type":"Text","value":" "},
8584
{"type":"Operator","value":"="},
8685
{"type":"TextWhitespace","value":" "},
8786
{"type":"LiteralNumberInteger","value":"1117"},

0 commit comments

Comments
 (0)