Skip to content

Commit 59126c5

Browse files
moorereasonmoorereason
authored
Add NewLazyLexer to defer rules definitions and reduce init costs (#449)
Add NewLazyLexer and MustNewLazyLexer which accept a function that returns the rules for the lexer. This allows us to defer the rules definitions until they're needed. Lexers in a, g, s, and x packages have been updated to use the new lazy lexer.
1 parent 5da8316 commit 59126c5

File tree

188 files changed

+1934
-1132
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

188 files changed

+1934
-1132
lines changed

Diff for: ‎lexers/a/abap.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,19 @@ import (
66
)
77

88
// ABAP lexer.
9-
var Abap = internal.Register(MustNewLexer(
9+
var Abap = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "ABAP",
1212
Aliases: []string{"abap"},
1313
Filenames: []string{"*.abap", "*.ABAP"},
1414
MimeTypes: []string{"text/x-abap"},
1515
CaseInsensitive: true,
1616
},
17-
Rules{
17+
abapRules,
18+
))
19+
20+
func abapRules() Rules {
21+
return Rules{
1822
"common": {
1923
{`\s+`, Text, nil},
2024
{`^\*.*$`, CommentSingle, nil},
@@ -52,5 +56,5 @@ var Abap = internal.Register(MustNewLexer(
5256
{`[/;:()\[\],.]`, Punctuation, nil},
5357
{`(!)(\w+)`, ByGroups(Operator, Name), nil},
5458
},
55-
},
56-
))
59+
}
60+
}

Diff for: ‎lexers/a/abnf.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,18 @@ import (
66
)
77

88
// Abnf lexer.
9-
var Abnf = internal.Register(MustNewLexer(
9+
var Abnf = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "ABNF",
1212
Aliases: []string{"abnf"},
1313
Filenames: []string{"*.abnf"},
1414
MimeTypes: []string{"text/x-abnf"},
1515
},
16-
Rules{
16+
abnfRules,
17+
))
18+
19+
func abnfRules() Rules {
20+
return Rules{
1721
"root": {
1822
{`;.*$`, CommentSingle, nil},
1923
{`(%[si])?"[^"]*"`, Literal, nil},
@@ -34,5 +38,5 @@ var Abnf = internal.Register(MustNewLexer(
3438
{`\s+`, Text, nil},
3539
{`.`, Text, nil},
3640
},
37-
},
38-
))
41+
}
42+
}

Diff for: ‎lexers/a/actionscript.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
)
77

88
// Actionscript lexer.
9-
var Actionscript = internal.Register(MustNewLexer(
9+
var Actionscript = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "ActionScript",
1212
Aliases: []string{"as", "actionscript"},
@@ -15,7 +15,11 @@ var Actionscript = internal.Register(MustNewLexer(
1515
NotMultiline: true,
1616
DotAll: true,
1717
},
18-
Rules{
18+
actionscriptRules,
19+
))
20+
21+
func actionscriptRules() Rules {
22+
return Rules{
1923
"root": {
2024
{`\s+`, Text, nil},
2125
{`//.*?\n`, CommentSingle, nil},
@@ -35,5 +39,5 @@ var Actionscript = internal.Register(MustNewLexer(
3539
{`"(\\\\|\\"|[^"])*"`, LiteralStringDouble, nil},
3640
{`'(\\\\|\\'|[^'])*'`, LiteralStringSingle, nil},
3741
},
38-
},
39-
))
42+
}
43+
}

Diff for: ‎lexers/a/actionscript3.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,19 @@ import (
66
)
77

88
// Actionscript 3 lexer.
9-
var Actionscript3 = internal.Register(MustNewLexer(
9+
var Actionscript3 = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "ActionScript 3",
1212
Aliases: []string{"as3", "actionscript3"},
1313
Filenames: []string{"*.as"},
1414
MimeTypes: []string{"application/x-actionscript3", "text/x-actionscript3", "text/actionscript3"},
1515
DotAll: true,
1616
},
17-
Rules{
17+
actionscript3Rules,
18+
))
19+
20+
func actionscript3Rules() Rules {
21+
return Rules{
1822
"root": {
1923
{`\s+`, Text, nil},
2024
{`(function\s+)([$a-zA-Z_]\w*)(\s*)(\()`, ByGroups(KeywordDeclaration, NameFunction, Text, Operator), Push("funcparams")},
@@ -52,5 +56,5 @@ var Actionscript3 = internal.Register(MustNewLexer(
5256
{`,`, Operator, Pop(1)},
5357
Default(Pop(1)),
5458
},
55-
},
56-
))
59+
}
60+
}

Diff for: ‎lexers/a/ada.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,19 @@ import (
66
)
77

88
// Ada lexer.
9-
var Ada = internal.Register(MustNewLexer(
9+
var Ada = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "Ada",
1212
Aliases: []string{"ada", "ada95", "ada2005"},
1313
Filenames: []string{"*.adb", "*.ads", "*.ada"},
1414
MimeTypes: []string{"text/x-ada"},
1515
CaseInsensitive: true,
1616
},
17-
Rules{
17+
adaRules,
18+
))
19+
20+
func adaRules() Rules {
21+
return Rules{
1822
"root": {
1923
{`[^\S\n]+`, Text, nil},
2024
{`--.*?\n`, CommentSingle, nil},
@@ -110,5 +114,5 @@ var Ada = internal.Register(MustNewLexer(
110114
{`\)`, Punctuation, Pop(1)},
111115
Include("root"),
112116
},
113-
},
114-
))
117+
}
118+
}

Diff for: ‎lexers/a/angular2.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,18 @@ import (
66
)
77

88
// Angular2 lexer.
9-
var Angular2 = internal.Register(MustNewLexer(
9+
var Angular2 = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "Angular2",
1212
Aliases: []string{"ng2"},
1313
Filenames: []string{},
1414
MimeTypes: []string{},
1515
},
16-
Rules{
16+
angular2Rules,
17+
))
18+
19+
func angular2Rules() Rules {
20+
return Rules{
1721
"root": {
1822
{`[^{([*#]+`, Other, nil},
1923
{`(\{\{)(\s*)`, ByGroups(CommentPreproc, Text), Push("ngExpression")},
@@ -38,5 +42,5 @@ var Angular2 = internal.Register(MustNewLexer(
3842
{`'.*?'`, LiteralString, Pop(1)},
3943
{`[^\s>]+`, LiteralString, Pop(1)},
4044
},
41-
},
42-
))
45+
}
46+
}

Diff for: ‎lexers/a/antlr.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,18 @@ import (
66
)
77

88
// ANTLR lexer.
9-
var ANTLR = internal.Register(MustNewLexer(
9+
var ANTLR = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "ANTLR",
1212
Aliases: []string{"antlr"},
1313
Filenames: []string{},
1414
MimeTypes: []string{},
1515
},
16-
Rules{
16+
antlrRules,
17+
))
18+
19+
func antlrRules() Rules {
20+
return Rules{
1721
"whitespace": {
1822
{`\s+`, TextWhitespace, nil},
1923
},
@@ -97,5 +101,5 @@ var ANTLR = internal.Register(MustNewLexer(
97101
{`(\$[a-zA-Z]+)(\.?)(text|value)?`, ByGroups(NameVariable, Punctuation, NameProperty), nil},
98102
{`(\\\\|\\\]|\\\[|[^\[\]])+`, Other, nil},
99103
},
100-
},
101-
))
104+
}
105+
}

Diff for: ‎lexers/a/apache.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,19 @@ import (
66
)
77

88
// Apacheconf lexer.
9-
var Apacheconf = internal.Register(MustNewLexer(
9+
var Apacheconf = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "ApacheConf",
1212
Aliases: []string{"apacheconf", "aconf", "apache"},
1313
Filenames: []string{".htaccess", "apache.conf", "apache2.conf"},
1414
MimeTypes: []string{"text/x-apacheconf"},
1515
CaseInsensitive: true,
1616
},
17-
Rules{
17+
apacheconfRules,
18+
))
19+
20+
func apacheconfRules() Rules {
21+
return Rules{
1822
"root": {
1923
{`\s+`, Text, nil},
2024
{`(#.*?)$`, Comment, nil},
@@ -34,5 +38,5 @@ var Apacheconf = internal.Register(MustNewLexer(
3438
{`"([^"\\]*(?:\\.[^"\\]*)*)"`, LiteralStringDouble, nil},
3539
{`[^\s"\\]+`, Text, nil},
3640
},
37-
},
38-
))
41+
}
42+
}

Diff for: ‎lexers/a/apl.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,18 @@ import (
66
)
77

88
// Apl lexer.
9-
var Apl = internal.Register(MustNewLexer(
9+
var Apl = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "APL",
1212
Aliases: []string{"apl"},
1313
Filenames: []string{"*.apl"},
1414
MimeTypes: []string{},
1515
},
16-
Rules{
16+
aplRules,
17+
))
18+
19+
func aplRules() Rules {
20+
return Rules{
1721
"root": {
1822
{`\s+`, Text, nil},
1923
{`[⍝#].*$`, CommentSingle, nil},
@@ -32,5 +36,5 @@ var Apl = internal.Register(MustNewLexer(
3236
{`[⍺⍵⍶⍹∇:]`, NameBuiltinPseudo, nil},
3337
{`[{}]`, KeywordType, nil},
3438
},
35-
},
36-
))
39+
}
40+
}

Diff for: ‎lexers/a/applescript.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,19 @@ import (
66
)
77

88
// Applescript lexer.
9-
var Applescript = internal.Register(MustNewLexer(
9+
var Applescript = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "AppleScript",
1212
Aliases: []string{"applescript"},
1313
Filenames: []string{"*.applescript"},
1414
MimeTypes: []string{},
1515
DotAll: true,
1616
},
17-
Rules{
17+
applescriptRules,
18+
))
19+
20+
func applescriptRules() Rules {
21+
return Rules{
1822
"root": {
1923
{`\s+`, Text, nil},
2024
{`¬\n`, LiteralStringEscape, nil},
@@ -51,5 +55,5 @@ var Applescript = internal.Register(MustNewLexer(
5155
{`[^*(]+`, CommentMultiline, nil},
5256
{`[*(]`, CommentMultiline, nil},
5357
},
54-
},
55-
))
58+
}
59+
}

Diff for: ‎lexers/a/arduino.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,19 @@ import (
66
)
77

88
// Arduino lexer.
9-
var Arduino = internal.Register(MustNewLexer(
9+
var Arduino = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "Arduino",
1212
Aliases: []string{"arduino"},
1313
Filenames: []string{"*.ino"},
1414
MimeTypes: []string{"text/x-arduino"},
1515
EnsureNL: true,
1616
},
17-
Rules{
17+
arduinoRules,
18+
))
19+
20+
func arduinoRules() Rules {
21+
return Rules{
1822
"statements": {
1923
{Words(``, `\b`, `catch`, `const_cast`, `delete`, `dynamic_cast`, `explicit`, `export`, `friend`, `mutable`, `namespace`, `new`, `operator`, `private`, `protected`, `public`, `reinterpret_cast`, `restrict`, `static_cast`, `template`, `this`, `throw`, `throws`, `try`, `typeid`, `typename`, `using`, `virtual`, `constexpr`, `nullptr`, `decltype`, `thread_local`, `alignas`, `alignof`, `static_assert`, `noexcept`, `override`, `final`), Keyword, nil},
2024
{`char(16_t|32_t)\b`, KeywordType, nil},
@@ -106,5 +110,5 @@ var Arduino = internal.Register(MustNewLexer(
106110
{`^\s*#endif.*?(?<!\\)\n`, CommentPreproc, Pop(1)},
107111
{`.*?\n`, Comment, nil},
108112
},
109-
},
110-
))
113+
}
114+
}

Diff for: ‎lexers/a/awk.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,18 @@ import (
66
)
77

88
// Awk lexer.
9-
var Awk = internal.Register(MustNewLexer(
9+
var Awk = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "Awk",
1212
Aliases: []string{"awk", "gawk", "mawk", "nawk"},
1313
Filenames: []string{"*.awk"},
1414
MimeTypes: []string{"application/x-awk"},
1515
},
16-
Rules{
16+
awkRules,
17+
))
18+
19+
func awkRules() Rules {
20+
return Rules{
1721
"commentsandwhitespace": {
1822
{`\s+`, Text, nil},
1923
{`#.*$`, CommentSingle, nil},
@@ -44,5 +48,5 @@ var Awk = internal.Register(MustNewLexer(
4448
{`"(\\\\|\\"|[^"])*"`, LiteralStringDouble, nil},
4549
{`'(\\\\|\\'|[^'])*'`, LiteralStringSingle, nil},
4650
},
47-
},
48-
))
51+
}
52+
}

Diff for: ‎lexers/b/ballerina.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,19 @@ import (
66
)
77

88
// Ballerina lexer.
9-
var Ballerina = internal.Register(MustNewLexer(
9+
var Ballerina = internal.Register(MustNewLazyLexer(
1010
&Config{
1111
Name: "Ballerina",
1212
Aliases: []string{"ballerina"},
1313
Filenames: []string{"*.bal"},
1414
MimeTypes: []string{"text/x-ballerina"},
1515
DotAll: true,
1616
},
17-
Rules{
17+
ballerinaRules,
18+
))
19+
20+
func ballerinaRules() Rules {
21+
return Rules{
1822
"root": {
1923
{`[^\S\n]+`, Text, nil},
2024
{`//.*?\n`, CommentSingle, nil},
@@ -42,5 +46,5 @@ var Ballerina = internal.Register(MustNewLexer(
4246
"import": {
4347
{`[\w.]+`, NameNamespace, Pop(1)},
4448
},
45-
},
46-
))
49+
}
50+
}

0 commit comments

Comments
 (0)