Skip to content

Commit 5da8316

Browse files
alecthomasalecthomas
committed
Fix a few bugs including sub-lexers adding additional newlines when
EnsureNL is true.
1 parent 2a1e1a1 commit 5da8316

File tree

7 files changed

+56
-34
lines changed

7 files changed

+56
-34
lines changed

Diff for: ‎formatters/tty_indexed.go

+27-5
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,20 @@ var c = chroma.MustParseColour
1717

1818
var ttyTables = map[int]*ttyTable{
1919
8: {
20+
foreground: map[chroma.Colour]string{
21+
c("#000000"): "\033[30m", c("#7f0000"): "\033[31m", c("#007f00"): "\033[32m", c("#7f7fe0"): "\033[33m",
22+
c("#00007f"): "\033[34m", c("#7f007f"): "\033[35m", c("#007f7f"): "\033[36m", c("#e5e5e5"): "\033[37m",
23+
c("#555555"): "\033[1m\033[30m", c("#ff0000"): "\033[1m\033[31m", c("#00ff00"): "\033[1m\033[32m", c("#ffff00"): "\033[1m\033[33m",
24+
c("#0000ff"): "\033[1m\033[34m", c("#ff00ff"): "\033[1m\033[35m", c("#00ffff"): "\033[1m\033[36m", c("#ffffff"): "\033[1m\033[37m",
25+
},
26+
background: map[chroma.Colour]string{
27+
c("#000000"): "\033[40m", c("#7f0000"): "\033[41m", c("#007f00"): "\033[42m", c("#7f7fe0"): "\033[43m",
28+
c("#00007f"): "\033[44m", c("#7f007f"): "\033[45m", c("#007f7f"): "\033[46m", c("#e5e5e5"): "\033[47m",
29+
c("#555555"): "\033[1m\033[40m", c("#ff0000"): "\033[1m\033[41m", c("#00ff00"): "\033[1m\033[42m", c("#ffff00"): "\033[1m\033[43m",
30+
c("#0000ff"): "\033[1m\033[44m", c("#ff00ff"): "\033[1m\033[45m", c("#00ffff"): "\033[1m\033[46m", c("#ffffff"): "\033[1m\033[47m",
31+
},
32+
},
33+
16: {
2034
foreground: map[chroma.Colour]string{
2135
c("#000000"): "\033[30m", c("#7f0000"): "\033[31m", c("#007f00"): "\033[32m", c("#7f7fe0"): "\033[33m",
2236
c("#00007f"): "\033[34m", c("#7f007f"): "\033[35m", c("#007f7f"): "\033[36m", c("#e5e5e5"): "\033[37m",
@@ -227,15 +241,11 @@ type indexedTTYFormatter struct {
227241
func (c *indexedTTYFormatter) Format(w io.Writer, style *chroma.Style, it chroma.Iterator) (err error) {
228242
theme := styleToEscapeSequence(c.table, style)
229243
for token := it(); token != chroma.EOF; token = it() {
230-
// TODO: Cache token lookups?
231244
clr, ok := theme[token.Type]
232245
if !ok {
233246
clr, ok = theme[token.Type.SubCategory()]
234247
if !ok {
235248
clr = theme[token.Type.Category()]
236-
// if !ok {
237-
// clr = theme[chroma.InheritStyle]
238-
// }
239249
}
240250
}
241251
if clr != "" {
@@ -249,10 +259,22 @@ func (c *indexedTTYFormatter) Format(w io.Writer, style *chroma.Style, it chroma
249259
return nil
250260
}
251261

262+
// TTY is an 8-colour terminal formatter.
263+
//
264+
// The Lab colour space is used to map RGB values to the most appropriate index colour.
265+
var TTY = Register("terminal", &indexedTTYFormatter{ttyTables[8]})
266+
252267
// TTY8 is an 8-colour terminal formatter.
253268
//
254269
// The Lab colour space is used to map RGB values to the most appropriate index colour.
255-
var TTY8 = Register("terminal", &indexedTTYFormatter{ttyTables[8]})
270+
var TTY8 = Register("terminal8", &indexedTTYFormatter{ttyTables[8]})
271+
272+
// TTY16 is a 16-colour terminal formatter.
273+
//
274+
// It uses \033[3xm for normal colours and \033[90Xm for bright colours.
275+
//
276+
// The Lab colour space is used to map RGB values to the most appropriate index colour.
277+
var TTY16 = Register("terminal16", &indexedTTYFormatter{ttyTables[16]})
256278

257279
// TTY256 is a 256-colour terminal formatter.
258280
//

Diff for: ‎iterator.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import "strings"
44

55
// An Iterator across tokens.
66
//
7-
// nil will be returned at the end of the Token stream.
7+
// EOF will be returned at the end of the Token stream.
88
//
99
// If an error occurs within an Iterator, it may propagate this in a panic. Formatters should recover.
1010
type Iterator func() Token

Diff for: ‎lexers/testdata/arduino.expected

+1-2
Original file line numberDiff line numberDiff line change
@@ -541,6 +541,5 @@
541541
{"type":"Text","value":" \n "},
542542
{"type":"Punctuation","value":"}"},
543543
{"type":"Text","value":"\n"},
544-
{"type":"Punctuation","value":"}"},
545-
{"type":"Text","value":"\n"}
544+
{"type":"Punctuation","value":"}"}
546545
]

Diff for: ‎lexers/testdata/promql.expected

-6
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
{"type":"LiteralString","value":"alertmanager"},
1616
{"type":"Punctuation","value":"\"}"},
1717
{"type":"TextWhitespace","value":"\n\n"},
18-
1918
{"type":"CommentSingle","value":"# Aggregation operators"},
2019
{"type":"TextWhitespace","value":"\n"},
2120
{"type":"Keyword","value":"sum"},
@@ -47,7 +46,6 @@
4746
{"type":"TextWhitespace","value":" "},
4847
{"type":"LiteralNumberInteger","value":"1024"},
4948
{"type":"TextWhitespace","value":"\n\n"},
50-
5149
{"type":"CommentSingle","value":"# Metric with multiple lables and whitespaces"},
5250
{"type":"TextWhitespace","value":"\n"},
5351
{"type":"NameVariable","value":"go_gc_duration_seconds"},
@@ -67,7 +65,6 @@
6765
{"type":"TextWhitespace","value":" "},
6866
{"type":"Punctuation","value":"}"},
6967
{"type":"TextWhitespace","value":"\n\n"},
70-
7168
{"type":"CommentSingle","value":"# Expression and comment"},
7269
{"type":"TextWhitespace","value":"\n"},
7370
{"type":"NameVariable","value":"go_gc_duration_seconds"},
@@ -80,7 +77,6 @@
8077
{"type":"TextWhitespace","value":" "},
8178
{"type":"CommentSingle","value":"# single comment"},
8279
{"type":"TextWhitespace","value":"\n\n"},
83-
8480
{"type":"CommentSingle","value":"# Delta function"},
8581
{"type":"TextWhitespace","value":"\n"},
8682
{"type":"KeywordReserved","value":"delta"},
@@ -96,7 +92,6 @@
9692
{"type":"Punctuation","value":"]"},
9793
{"type":"Operator","value":")"},
9894
{"type":"TextWhitespace","value":"\n\n"},
99-
10095
{"type":"CommentSingle","value":"# Sum with arguments"},
10196
{"type":"TextWhitespace","value":"\n"},
10297
{"type":"Keyword","value":"sum"},
@@ -114,7 +109,6 @@
114109
{"type":"NameVariable","value":"instance_memory_usage_bytes"},
115110
{"type":"Operator","value":")"},
116111
{"type":"TextWhitespace","value":"\n\n"},
117-
118112
{"type":"CommentSingle","value":"# Multi-line with offset"},
119113
{"type":"TextWhitespace","value":"\n"},
120114
{"type":"KeywordReserved","value":"label_replace"},

Diff for: ‎lexers/testdata/stylus.expected

+17-17
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
[
2-
{"type":"NameVariable", "value":"$white"},
3-
{"type":"Text", "value":" "},
4-
{"type":"Operator", "value":"?="},
5-
{"type":"Text", "value":" "},
6-
{"type":"LiteralNumberHex", "value":"#fff"},
7-
{"type":"Text", "value":"\n"},
2+
{"type":"NameVariable","value":"$white"},
3+
{"type":"Text","value":" "},
4+
{"type":"Operator","value":"?="},
5+
{"type":"Text","value":" "},
6+
{"type":"LiteralNumberHex","value":"#fff"},
7+
{"type":"Text","value":"\n"},
88
{"type":"NameTag","value":"body"},
9-
{"type":"Text", "value":" "},
10-
{"type":"Punctuation", "value":"{"},
11-
{"type":"Text", "value":"\n "},
12-
{"type":"NameProperty", "value":"color"},
13-
{"type":"Punctuation", "value":":"},
14-
{"type":"Text", "value":" "},
15-
{"type":"NameVariable", "value":"$white"},
16-
{"type":"Punctuation", "value":";"},
17-
{"type":"Text", "value":"\n"},
18-
{"type":"Punctuation", "value":"}"}
19-
]
9+
{"type":"Text","value":" "},
10+
{"type":"Punctuation","value":"{"},
11+
{"type":"Text","value":"\n "},
12+
{"type":"NameProperty","value":"color"},
13+
{"type":"Punctuation","value":":"},
14+
{"type":"Text","value":" "},
15+
{"type":"NameVariable","value":"$white"},
16+
{"type":"Punctuation","value":";"},
17+
{"type":"Text","value":"\n"},
18+
{"type":"Punctuation","value":"}"}
19+
]

Diff for: ‎lexers/testdata/tsx.expected

+1-2
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,5 @@
6060
{"type":"LiteralStringSingle","value":"'root'"},
6161
{"type":"Punctuation","value":"),"},
6262
{"type":"Text","value":"\n"},
63-
{"type":"Punctuation","value":")"},
64-
{"type":"Text","value":"\n"}
63+
{"type":"Punctuation","value":")"}
6564
]

Diff for: ‎regexp.go

+9-1
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,7 @@ type LexerState struct {
264264
MutatorContext map[interface{}]interface{}
265265
iteratorStack []Iterator
266266
options *TokeniseOptions
267+
newlineAdded bool
267268
}
268269

269270
// Set mutator context.
@@ -278,7 +279,11 @@ func (l *LexerState) Get(key interface{}) interface{} {
278279

279280
// Iterator returns the next Token from the lexer.
280281
func (l *LexerState) Iterator() Token { // nolint: gocognit
281-
for l.Pos < len(l.Text) && len(l.Stack) > 0 {
282+
end := len(l.Text)
283+
if l.newlineAdded {
284+
end--
285+
}
286+
for l.Pos < end && len(l.Stack) > 0 {
282287
// Exhaust the iterator stack, if any.
283288
for len(l.iteratorStack) > 0 {
284289
n := len(l.iteratorStack) - 1
@@ -432,10 +437,13 @@ func (r *RegexLexer) Tokenise(options *TokeniseOptions, text string) (Iterator,
432437
if options.EnsureLF {
433438
text = ensureLF(text)
434439
}
440+
newlineAdded := false
435441
if !options.Nested && r.config.EnsureNL && !strings.HasSuffix(text, "\n") {
436442
text += "\n"
443+
newlineAdded = true
437444
}
438445
state := &LexerState{
446+
newlineAdded: newlineAdded,
439447
options: options,
440448
Lexer: r,
441449
Text: []rune(text),

0 commit comments

Comments
 (0)