Skip to content

Commit 881d54f

Browse files
Steven Pennyalecthomas
Steven Penny
authored and
alecthomas
committed
Improve number literals for several languages
Using these examples: ~~~ 0x21 1_000 1e3 ~~~ Several languages failed that support these syntax. Here are ones I found: Lexer | 0x21 | 1_000 | 1e3 -----------|------|-------|----- C# | fail | fail | fail Go | | fail | JavaScript | | fail | fail PHP | | fail | Python | | fail | Ruby | | | fail I fixed these issues, and added tests.
1 parent 74f5402 commit 881d54f

18 files changed

+145
-7
lines changed

Diff for: ‎lexers/c/csharp.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ var CSharp = internal.Register(MustNewLexer(
2929
{`\$@?"(""|[^"])*"`, LiteralString, nil},
3030
{`"(\\\\|\\"|[^"\n])*["\n]`, LiteralString, nil},
3131
{`'\\.'|'[^\\]'`, LiteralStringChar, nil},
32-
{`[0-9](\.[0-9]*)?([eE][+-][0-9]+)?[flFLdD]?|0[xX][0-9a-fA-F]+[Ll]?`, LiteralNumber, nil},
32+
{`0[xX][0-9a-fA-F]+[Ll]?|[0-9_](\.[0-9]*)?([eE][+-]?[0-9]+)?[flFLdD]?`, LiteralNumber, nil},
3333
{`#[ \t]*(if|endif|else|elif|define|undef|line|error|warning|region|endregion|pragma)\b.*?\n`, CommentPreproc, nil},
3434
{`\b(extern)(\s+)(alias)\b`, ByGroups(Keyword, Text, Keyword), nil},
3535
{`(abstract|as|async|await|base|break|by|case|catch|checked|const|continue|default|delegate|do|else|enum|event|explicit|extern|false|finally|fixed|for|foreach|goto|if|implicit|in|interface|internal|is|let|lock|new|null|on|operator|out|override|params|private|protected|public|readonly|ref|return|sealed|sizeof|stackalloc|static|switch|this|throw|true|try|typeof|unchecked|unsafe|virtual|void|while|get|set|new|partial|yield|add|remove|value|alias|ascending|descending|from|group|into|orderby|select|thenby|where|join|equals)\b`, Keyword, nil},

Diff for: ‎lexers/circular/php.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ var phpCommonRules = Rules{
4747
{`\d+e[+-]?[0-9]+`, LiteralNumberFloat, nil},
4848
{`0[0-7]+`, LiteralNumberOct, nil},
4949
{`0x[a-f0-9]+`, LiteralNumberHex, nil},
50-
{`\d+`, LiteralNumberInteger, nil},
50+
{`[\d_]+`, LiteralNumberInteger, nil},
5151
{`0b[01]+`, LiteralNumberBin, nil},
5252
{`'([^'\\]*(?:\\.[^'\\]*)*)'`, LiteralStringSingle, nil},
5353
{"`([^`\\\\]*(?:\\\\.[^`\\\\]*)*)`", LiteralStringBacktick, nil},

Diff for: ‎lexers/g/go.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ var Go = internal.Register(MustNewLexer(
3838
{`\.\d+([eE][+\-]?\d+)?`, LiteralNumberFloat, nil},
3939
{`0[0-7]+`, LiteralNumberOct, nil},
4040
{`0[xX][0-9a-fA-F]+`, LiteralNumberHex, nil},
41-
{`(0|[1-9][0-9]*)`, LiteralNumberInteger, nil},
41+
{`(0|[1-9][0-9_]*)`, LiteralNumberInteger, nil},
4242
{`'(\\['"\\abfnrtv]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|[^\\])'`, LiteralStringChar, nil},
4343
{"(`)([^`]*)(`)", ByGroups(LiteralString, Using(TypeRemappingLexer(GoTextTemplate, TypeMapping{{Other, LiteralString, nil}})), LiteralString), nil},
4444
{`"(\\\\|\\"|[^"])*"`, LiteralString, nil},

Diff for: ‎lexers/j/javascript.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,11 @@ var JavascriptRules = Rules{
2626
{`\A#! ?/.*?\n`, CommentHashbang, nil},
2727
{`^(?=\s|/|<!--)`, Text, Push("slashstartsregex")},
2828
Include("commentsandwhitespace"),
29-
{`(\.\d+|[0-9]+\.[0-9]*)([eE][-+]?[0-9]+)?`, LiteralNumberFloat, nil},
29+
{`\d+(\.\d*|[eE][+\-]?\d+)`, LiteralNumberFloat, nil},
3030
{`0[bB][01]+`, LiteralNumberBin, nil},
3131
{`0[oO][0-7]+`, LiteralNumberOct, nil},
3232
{`0[xX][0-9a-fA-F]+`, LiteralNumberHex, nil},
33-
{`[0-9]+`, LiteralNumberInteger, nil},
33+
{`[0-9_]+`, LiteralNumberInteger, nil},
3434
{`\.\.\.|=>`, Punctuation, nil},
3535
{`\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?`, Operator, Push("slashstartsregex")},
3636
{`[{(\[;,]`, Punctuation, Push("slashstartsregex")},

Diff for: ‎lexers/p/python.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ var Python = internal.Register(MustNewLexer(
6767
{`0[bB][01]+`, LiteralNumberBin, nil},
6868
{`0[xX][a-fA-F0-9]+`, LiteralNumberHex, nil},
6969
{`\d+L`, LiteralNumberIntegerLong, nil},
70-
{`\d+j?`, LiteralNumberInteger, nil},
70+
{`[\d_]+j?`, LiteralNumberInteger, nil},
7171
},
7272
"backtick": {
7373
{"`.*?`", LiteralStringBacktick, nil},

Diff for: ‎lexers/r/ruby.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ var Ruby = internal.Register(MustNewLexer(
3939
{`(0_?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?`, ByGroups(LiteralNumberOct, Text, Operator), nil},
4040
{`(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*)(\s*)([/?])?`, ByGroups(LiteralNumberHex, Text, Operator), nil},
4141
{`(0b[01]+(?:_[01]+)*)(\s*)([/?])?`, ByGroups(LiteralNumberBin, Text, Operator), nil},
42-
{`([\d]+(?:_\d+)*)(\s*)([/?])?`, ByGroups(LiteralNumberInteger, Text, Operator), nil},
42+
{`([\d]+(?:[_e]\d+)*)(\s*)([/?])?`, ByGroups(LiteralNumberInteger, Text, Operator), nil},
4343
{`@@[a-zA-Z_]\w*`, NameVariableClass, nil},
4444
{`@[a-zA-Z_]\w*`, NameVariableInstance, nil},
4545
{`\$\w+`, NameVariableGlobal, nil},

Diff for: ‎lexers/testdata/csharp.actual

+3
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,7 @@ foreach (DriveInfo drive in drives)
1010
}
1111
}
1212

13+
int n = 0x21 + 1_000;
14+
double n2 = 1e3;
15+
1316
// Comment as last line should be recognised properly

Diff for: ‎lexers/testdata/csharp.expected

+22
Original file line numberDiff line numberDiff line change
@@ -70,5 +70,27 @@
7070
{"type":"Text","value":"\n"},
7171
{"type":"Punctuation","value":"}"},
7272
{"type":"Text","value":"\n\n"},
73+
{"type":"KeywordType","value":"int"},
74+
{"type":"Text","value":" "},
75+
{"type":"Name","value":"n"},
76+
{"type":"Text","value":" "},
77+
{"type":"Punctuation","value":"="},
78+
{"type":"Text","value":" "},
79+
{"type":"LiteralNumber","value":"0x21"},
80+
{"type":"Text","value":" "},
81+
{"type":"Punctuation","value":"+"},
82+
{"type":"Text","value":" "},
83+
{"type":"LiteralNumber","value":"1_000"},
84+
{"type":"Punctuation","value":";"},
85+
{"type":"Text","value":"\n"},
86+
{"type":"KeywordType","value":"double"},
87+
{"type":"Text","value":" "},
88+
{"type":"Name","value":"n2"},
89+
{"type":"Text","value":" "},
90+
{"type":"Punctuation","value":"="},
91+
{"type":"Text","value":" "},
92+
{"type":"LiteralNumber","value":"1e3"},
93+
{"type":"Punctuation","value":";"},
94+
{"type":"Text","value":"\n\n"},
7395
{"type":"CommentSingle","value":"// Comment as last line should be recognised properly\n"}
7496
]

Diff for: ‎lexers/testdata/go.actual

+3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ func main() {
66
fmt.Println("Hello World!")
77
}
88

9+
var n int = 0x21 + 1_000
10+
var n2 float64 = 1e3
11+
912
func hello(a int) {
1013
fmt.Println("Hello World!").Hello()
1114

Diff for: ‎lexers/testdata/go.expected

+24
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,30 @@
2323
{"type":"Text","value":"\n"},
2424
{"type":"Punctuation","value":"}"},
2525
{"type":"Text","value":"\n\n"},
26+
{"type":"KeywordDeclaration","value":"var"},
27+
{"type":"Text","value":" "},
28+
{"type":"NameOther","value":"n"},
29+
{"type":"Text","value":" "},
30+
{"type":"KeywordType","value":"int"},
31+
{"type":"Text","value":" "},
32+
{"type":"Punctuation","value":"="},
33+
{"type":"Text","value":" "},
34+
{"type":"LiteralNumberHex","value":"0x21"},
35+
{"type":"Text","value":" "},
36+
{"type":"Operator","value":"+"},
37+
{"type":"Text","value":" "},
38+
{"type":"LiteralNumberInteger","value":"1_000"},
39+
{"type":"Text","value":"\n"},
40+
{"type":"KeywordDeclaration","value":"var"},
41+
{"type":"Text","value":" "},
42+
{"type":"NameOther","value":"n2"},
43+
{"type":"Text","value":" "},
44+
{"type":"KeywordType","value":"float64"},
45+
{"type":"Text","value":" "},
46+
{"type":"Punctuation","value":"="},
47+
{"type":"Text","value":" "},
48+
{"type":"LiteralNumberFloat","value":"1e3"},
49+
{"type":"Text","value":"\n\n"},
2650
{"type":"KeywordDeclaration","value":"func"},
2751
{"type":"Text","value":" "},
2852
{"type":"NameFunction","value":"hello"},

Diff for: ‎lexers/testdata/javascript.actual

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
11
sum = parseInt(num1) + parseInt(num2) // "+" means "add"
22
alert("Sum = " + sum) // "+" means combine into a string
33
const filePath = String.raw`C:\Development\profile\aboutme.html`;
4+
5+
let n = 0x21 + 1_000;
6+
let n2 = 1e3;

Diff for: ‎lexers/testdata/javascript.expected

+22
Original file line numberDiff line numberDiff line change
@@ -37,5 +37,27 @@
3737
{"type":"NameOther","value":"raw"},
3838
{"type":"LiteralStringBacktick","value":"`C:\\Development\\profile\\aboutme.html`"},
3939
{"type":"Punctuation","value":";"},
40+
{"type":"Text","value":"\n\n"},
41+
{"type":"KeywordDeclaration","value":"let"},
42+
{"type":"Text","value":" "},
43+
{"type":"NameOther","value":"n"},
44+
{"type":"Text","value":" "},
45+
{"type":"Operator","value":"="},
46+
{"type":"Text","value":" "},
47+
{"type":"LiteralNumberHex","value":"0x21"},
48+
{"type":"Text","value":" "},
49+
{"type":"Operator","value":"+"},
50+
{"type":"Text","value":" "},
51+
{"type":"LiteralNumberInteger","value":"1_000"},
52+
{"type":"Punctuation","value":";"},
53+
{"type":"Text","value":"\n"},
54+
{"type":"KeywordDeclaration","value":"let"},
55+
{"type":"Text","value":" "},
56+
{"type":"NameOther","value":"n2"},
57+
{"type":"Text","value":" "},
58+
{"type":"Operator","value":"="},
59+
{"type":"Text","value":" "},
60+
{"type":"LiteralNumberFloat","value":"1e3"},
61+
{"type":"Punctuation","value":";"},
4062
{"type":"Text","value":"\n"}
4163
]

Diff for: ‎lexers/testdata/php.actual

+3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
$docs = $modx->getIterator('modResource', ["parent" => 84]);
22

3+
$n = 0x21 + 1_000;
4+
$n2 = 1e3;
5+
36
foreach($docs as $doc){
47
$q=$doc->content;
58
$doc->set("content", preg_replace("/Some value/i", "Replacement", $q));

Diff for: ‎lexers/testdata/php.expected

+18
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,24 @@
1818
{"type":"LiteralNumberInteger","value":"84"},
1919
{"type":"Punctuation","value":"]);"},
2020
{"type":"Text","value":"\n\n"},
21+
{"type":"NameVariable","value":"$n"},
22+
{"type":"Text","value":" "},
23+
{"type":"Operator","value":"="},
24+
{"type":"Text","value":" "},
25+
{"type":"LiteralNumberHex","value":"0x21"},
26+
{"type":"Text","value":" "},
27+
{"type":"Operator","value":"+"},
28+
{"type":"Text","value":" "},
29+
{"type":"LiteralNumberInteger","value":"1_000"},
30+
{"type":"Punctuation","value":";"},
31+
{"type":"Text","value":"\n"},
32+
{"type":"NameVariable","value":"$n2"},
33+
{"type":"Text","value":" "},
34+
{"type":"Operator","value":"="},
35+
{"type":"Text","value":" "},
36+
{"type":"LiteralNumberFloat","value":"1e3"},
37+
{"type":"Punctuation","value":";"},
38+
{"type":"Text","value":"\n\n"},
2139
{"type":"Keyword","value":"foreach"},
2240
{"type":"Punctuation","value":"("},
2341
{"type":"NameVariable","value":"$docs"},

Diff for: ‎lexers/testdata/python.actual

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
n = 0x21 + 1_000
2+
n2 = 1e3

Diff for: ‎lexers/testdata/python.expected

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[
2+
{"type":"Name","value":"n"},
3+
{"type":"Text","value":" "},
4+
{"type":"Operator","value":"="},
5+
{"type":"Text","value":" "},
6+
{"type":"LiteralNumberHex","value":"0x21"},
7+
{"type":"Text","value":" "},
8+
{"type":"Operator","value":"+"},
9+
{"type":"Text","value":" "},
10+
{"type":"LiteralNumberInteger","value":"1_000"},
11+
{"type":"Text","value":"\n"},
12+
{"type":"Name","value":"n2"},
13+
{"type":"Text","value":" "},
14+
{"type":"Operator","value":"="},
15+
{"type":"Text","value":" "},
16+
{"type":"LiteralNumberFloat","value":"1e3"},
17+
{"type":"Text","value":"\n"}
18+
]

Diff for: ‎lexers/testdata/ruby.actual

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
n = 0x21 + 1_000
2+
n2 = 1e3

Diff for: ‎lexers/testdata/ruby.expected

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[
2+
{"type":"Name","value":"n"},
3+
{"type":"Text","value":" "},
4+
{"type":"Operator","value":"="},
5+
{"type":"Text","value":" "},
6+
{"type":"LiteralNumberHex","value":"0x21"},
7+
{"type":"Text","value":" "},
8+
{"type":"Operator","value":"+"},
9+
{"type":"Text","value":" "},
10+
{"type":"LiteralNumberInteger","value":"1_000"},
11+
{"type":"Text","value":"\n"},
12+
{"type":"Name","value":"n2"},
13+
{"type":"Text","value":" "},
14+
{"type":"Operator","value":"="},
15+
{"type":"Text","value":" "},
16+
{"type":"LiteralNumberInteger","value":"1e3"},
17+
{"type":"Text","value":"\n"}
18+
]

0 commit comments

Comments
 (0)