Skip to content

Commit 7dbcddc

Browse files
kenshawalecthomas
kenshaw
authored and
alecthomas
committed
Change Text token to TextWhitespace token in SQL lexers
For consistency with other lexers, convert the Text token on whitespace to TextWhitespace.
1 parent 898761d commit 7dbcddc

File tree

4 files changed

+73
-73
lines changed

4 files changed

+73
-73
lines changed

Diff for: ‎lexers/m/mysql.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,19 @@ var MySQL = internal.Register(MustNewLexer(
1717
},
1818
Rules{
1919
"root": {
20-
{`\s+`, Text, nil},
20+
{`\s+`, TextWhitespace, nil},
2121
{`(#|--\s+).*\n?`, CommentSingle, nil},
2222
{`/\*`, CommentMultiline, Push("multiline-comments")},
2323
{`[0-9]+`, LiteralNumberInteger, nil},
2424
{`[0-9]*\.[0-9]+(e[+-][0-9]+)`, LiteralNumberFloat, nil},
2525
{`((?:_[a-z0-9]+)?)(')`, ByGroups(LiteralStringAffix, LiteralStringSingle), Push("string")},
2626
{`((?:_[a-z0-9]+)?)(")`, ByGroups(LiteralStringAffix, LiteralStringDouble), Push("double-string")},
2727
{"[+*/<>=~!@#%^&|`?-]", Operator, nil},
28-
{`\b(tinyint|smallint|mediumint|int|integer|bigint|date|datetime|time|bit|bool|tinytext|mediumtext|longtext|text|tinyblob|mediumblob|longblob|blob|float|double|double\s+precision|real|numeric|dec|decimal|timestamp|year|char|varchar|varbinary|varcharacter|enum|set)(\b\s*)(\()?`, ByGroups(KeywordType, Text, Punctuation), nil},
28+
{`\b(tinyint|smallint|mediumint|int|integer|bigint|date|datetime|time|bit|bool|tinytext|mediumtext|longtext|text|tinyblob|mediumblob|longblob|blob|float|double|double\s+precision|real|numeric|dec|decimal|timestamp|year|char|varchar|varbinary|varcharacter|enum|set)(\b\s*)(\()?`, ByGroups(KeywordType, TextWhitespace, Punctuation), nil},
2929
{`\b(add|all|alter|analyze|and|as|asc|asensitive|before|between|bigint|binary|blob|both|by|call|cascade|case|change|char|character|check|collate|column|condition|constraint|continue|convert|create|cross|current_date|current_time|current_timestamp|current_user|cursor|database|databases|day_hour|day_microsecond|day_minute|day_second|dec|decimal|declare|default|delayed|delete|desc|describe|deterministic|distinct|distinctrow|div|double|drop|dual|each|else|elseif|enclosed|escaped|exists|exit|explain|fetch|flush|float|float4|float8|for|force|foreign|from|fulltext|grant|group|having|high_priority|hour_microsecond|hour_minute|hour_second|identified|if|ignore|in|index|infile|inner|inout|insensitive|insert|int|int1|int2|int3|int4|int8|integer|interval|into|is|iterate|join|key|keys|kill|leading|leave|left|like|limit|lines|load|localtime|localtimestamp|lock|long|loop|low_priority|match|minute_microsecond|minute_second|mod|modifies|natural|no_write_to_binlog|not|numeric|on|optimize|option|optionally|or|order|out|outer|outfile|precision|primary|privileges|procedure|purge|raid0|read|reads|real|references|regexp|release|rename|repeat|replace|require|restrict|return|revoke|right|rlike|schema|schemas|second_microsecond|select|sensitive|separator|set|show|smallint|soname|spatial|specific|sql|sql_big_result|sql_calc_found_rows|sql_small_result|sqlexception|sqlstate|sqlwarning|ssl|starting|straight_join|table|terminated|then|to|trailing|trigger|undo|union|unique|unlock|unsigned|update|usage|use|user|using|utc_date|utc_time|utc_timestamp|values|varying|when|where|while|with|write|x509|xor|year_month|zerofill)\b`, Keyword, nil},
3030
{`\b(auto_increment|engine|charset|tables)\b`, KeywordPseudo, nil},
3131
{`(true|false|null)`, NameConstant, nil},
32-
{`([a-z_]\w*)(\s*)(\()`, ByGroups(NameFunction, Text, Punctuation), nil},
32+
{`([a-z_]\w*)(\s*)(\()`, ByGroups(NameFunction, TextWhitespace, Punctuation), nil},
3333
{`[a-z_]\w*`, Name, nil},
3434
{`@[a-z0-9]*[._]*[a-z0-9]*`, NameVariable, nil},
3535
{`[;:()\[\],.]`, Punctuation, nil},

Diff for: ‎lexers/p/plsql.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ var PLpgSQL = internal.Register(MustNewLexer(
2121
{`:=`, Operator, nil},
2222
{`\<\<[a-z]\w*\>\>`, NameLabel, nil},
2323
{`\#[a-z]\w*\b`, KeywordPseudo, nil},
24-
{`\s+`, Text, nil},
24+
{`\s+`, TextWhitespace, nil},
2525
{`--.*\n?`, CommentSingle, nil},
2626
{`/\*`, CommentMultiline, Push("multiline-comments")},
2727
{`(bigint|bigserial|bit|bit\s+varying|bool|boolean|box|bytea|char|character|character\s+varying|cidr|circle|date|decimal|double\s+precision|float4|float8|inet|int|int2|int4|int8|integer|interval|json|jsonb|line|lseg|macaddr|money|numeric|path|pg_lsn|point|polygon|real|serial|serial2|serial4|serial8|smallint|smallserial|text|time|timestamp|timestamptz|timetz|tsquery|tsvector|txid_snapshot|uuid|varbit|varchar|with\s+time\s+zone|without\s+time\s+zone|xml|anyarray|anyelement|anyenum|anynonarray|anyrange|cstring|fdw_handler|internal|language_handler|opaque|record|void)\b`, NameBuiltin, nil},

Diff for: ‎lexers/s/sql.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ var SQL = internal.Register(MustNewLexer(
1717
},
1818
Rules{
1919
"root": {
20-
{`\s+`, Text, nil},
20+
{`\s+`, TextWhitespace, nil},
2121
{`--.*\n?`, CommentSingle, nil},
2222
{`/\*`, CommentMultiline, Push("multiline-comments")},
2323
{`'`, LiteralStringSingle, Push("string")},

Diff for: ‎lexers/testdata/markdown.expected

+68-68
Original file line numberDiff line numberDiff line change
@@ -218,123 +218,123 @@
218218
{"type":"Keyword","value":"AS"},
219219
{"type":"Text","value":" "},
220220
{"type":"LiteralStringHeredoc","value":"$emp_stamp$"},
221-
{"type":"Text","value":"\n"},
221+
{"type":"TextWhitespace","value":"\n"},
222222
{"type":"Keyword","value":"BEGIN"},
223-
{"type":"Text","value":"\n "},
223+
{"type":"TextWhitespace","value":"\n "},
224224
{"type":"CommentSingle","value":"-- Check that empname and salary are given\n"},
225-
{"type":"Text","value":" "},
225+
{"type":"TextWhitespace","value":" "},
226226
{"type":"Keyword","value":"IF"},
227-
{"type":"Text","value":" "},
227+
{"type":"TextWhitespace","value":" "},
228228
{"type":"Name","value":"NEW"},
229229
{"type":"LiteralNumberFloat","value":"."},
230230
{"type":"Name","value":"empname"},
231-
{"type":"Text","value":" "},
231+
{"type":"TextWhitespace","value":" "},
232232
{"type":"Keyword","value":"IS"},
233-
{"type":"Text","value":" "},
233+
{"type":"TextWhitespace","value":" "},
234234
{"type":"Keyword","value":"NULL"},
235-
{"type":"Text","value":" "},
235+
{"type":"TextWhitespace","value":" "},
236236
{"type":"Keyword","value":"THEN"},
237-
{"type":"Text","value":"\n "},
237+
{"type":"TextWhitespace","value":"\n "},
238238
{"type":"Keyword","value":"RAISE"},
239-
{"type":"Text","value":" "},
239+
{"type":"TextWhitespace","value":" "},
240240
{"type":"Keyword","value":"EXCEPTION"},
241-
{"type":"Text","value":" "},
241+
{"type":"TextWhitespace","value":" "},
242242
{"type":"LiteralStringSingle","value":"'empname cannot be null'"},
243243
{"type":"Punctuation","value":";"},
244-
{"type":"Text","value":"\n "},
244+
{"type":"TextWhitespace","value":"\n "},
245245
{"type":"Keyword","value":"END"},
246-
{"type":"Text","value":" "},
246+
{"type":"TextWhitespace","value":" "},
247247
{"type":"Keyword","value":"IF"},
248248
{"type":"Punctuation","value":";"},
249-
{"type":"Text","value":"\n "},
249+
{"type":"TextWhitespace","value":"\n "},
250250
{"type":"Keyword","value":"IF"},
251-
{"type":"Text","value":" "},
251+
{"type":"TextWhitespace","value":" "},
252252
{"type":"Name","value":"NEW"},
253253
{"type":"LiteralNumberFloat","value":"."},
254254
{"type":"Name","value":"salary"},
255-
{"type":"Text","value":" "},
255+
{"type":"TextWhitespace","value":" "},
256256
{"type":"Keyword","value":"IS"},
257-
{"type":"Text","value":" "},
257+
{"type":"TextWhitespace","value":" "},
258258
{"type":"Keyword","value":"NULL"},
259-
{"type":"Text","value":" "},
259+
{"type":"TextWhitespace","value":" "},
260260
{"type":"Keyword","value":"THEN"},
261-
{"type":"Text","value":"\n "},
261+
{"type":"TextWhitespace","value":"\n "},
262262
{"type":"Keyword","value":"RAISE"},
263-
{"type":"Text","value":" "},
263+
{"type":"TextWhitespace","value":" "},
264264
{"type":"Keyword","value":"EXCEPTION"},
265-
{"type":"Text","value":" "},
265+
{"type":"TextWhitespace","value":" "},
266266
{"type":"LiteralStringSingle","value":"'% cannot have null salary'"},
267267
{"type":"Punctuation","value":","},
268-
{"type":"Text","value":" "},
268+
{"type":"TextWhitespace","value":" "},
269269
{"type":"Name","value":"NEW"},
270270
{"type":"LiteralNumberFloat","value":"."},
271271
{"type":"Name","value":"empname"},
272272
{"type":"Punctuation","value":";"},
273-
{"type":"Text","value":"\n "},
273+
{"type":"TextWhitespace","value":"\n "},
274274
{"type":"Keyword","value":"END"},
275-
{"type":"Text","value":" "},
275+
{"type":"TextWhitespace","value":" "},
276276
{"type":"Keyword","value":"IF"},
277277
{"type":"Punctuation","value":";"},
278-
{"type":"Text","value":"\n\n "},
278+
{"type":"TextWhitespace","value":"\n\n "},
279279
{"type":"CommentSingle","value":"-- Who works for us when she must pay for it?\n"},
280-
{"type":"Text","value":" "},
280+
{"type":"TextWhitespace","value":" "},
281281
{"type":"Keyword","value":"IF"},
282-
{"type":"Text","value":" "},
282+
{"type":"TextWhitespace","value":" "},
283283
{"type":"Name","value":"NEW"},
284284
{"type":"LiteralNumberFloat","value":"."},
285285
{"type":"Name","value":"salary"},
286-
{"type":"Text","value":" "},
286+
{"type":"TextWhitespace","value":" "},
287287
{"type":"Operator","value":"\u003c"},
288-
{"type":"Text","value":" "},
288+
{"type":"TextWhitespace","value":" "},
289289
{"type":"LiteralNumberFloat","value":"0"},
290-
{"type":"Text","value":" "},
290+
{"type":"TextWhitespace","value":" "},
291291
{"type":"Keyword","value":"THEN"},
292-
{"type":"Text","value":"\n "},
292+
{"type":"TextWhitespace","value":"\n "},
293293
{"type":"Keyword","value":"RAISE"},
294-
{"type":"Text","value":" "},
294+
{"type":"TextWhitespace","value":" "},
295295
{"type":"Keyword","value":"EXCEPTION"},
296-
{"type":"Text","value":" "},
296+
{"type":"TextWhitespace","value":" "},
297297
{"type":"LiteralStringSingle","value":"'% cannot have a negative salary'"},
298298
{"type":"Punctuation","value":","},
299-
{"type":"Text","value":" "},
299+
{"type":"TextWhitespace","value":" "},
300300
{"type":"Name","value":"NEW"},
301301
{"type":"LiteralNumberFloat","value":"."},
302302
{"type":"Name","value":"empname"},
303303
{"type":"Punctuation","value":";"},
304-
{"type":"Text","value":"\n "},
304+
{"type":"TextWhitespace","value":"\n "},
305305
{"type":"Keyword","value":"END"},
306-
{"type":"Text","value":" "},
306+
{"type":"TextWhitespace","value":" "},
307307
{"type":"Keyword","value":"IF"},
308308
{"type":"Punctuation","value":";"},
309-
{"type":"Text","value":"\n\n "},
309+
{"type":"TextWhitespace","value":"\n\n "},
310310
{"type":"CommentSingle","value":"-- Remember who changed the payroll when\n"},
311-
{"type":"Text","value":" "},
311+
{"type":"TextWhitespace","value":" "},
312312
{"type":"Name","value":"NEW"},
313313
{"type":"LiteralNumberFloat","value":"."},
314314
{"type":"Name","value":"last_date"},
315-
{"type":"Text","value":" "},
315+
{"type":"TextWhitespace","value":" "},
316316
{"type":"Operator","value":":="},
317-
{"type":"Text","value":" "},
317+
{"type":"TextWhitespace","value":" "},
318318
{"type":"Keyword","value":"current_timestamp"},
319319
{"type":"Punctuation","value":";"},
320-
{"type":"Text","value":"\n "},
320+
{"type":"TextWhitespace","value":"\n "},
321321
{"type":"Name","value":"NEW"},
322322
{"type":"LiteralNumberFloat","value":"."},
323323
{"type":"Name","value":"last_user"},
324-
{"type":"Text","value":" "},
324+
{"type":"TextWhitespace","value":" "},
325325
{"type":"Operator","value":":="},
326-
{"type":"Text","value":" "},
326+
{"type":"TextWhitespace","value":" "},
327327
{"type":"Keyword","value":"current_user"},
328328
{"type":"Punctuation","value":";"},
329-
{"type":"Text","value":"\n "},
329+
{"type":"TextWhitespace","value":"\n "},
330330
{"type":"Keyword","value":"RETURN"},
331-
{"type":"Text","value":" "},
331+
{"type":"TextWhitespace","value":" "},
332332
{"type":"Name","value":"NEW"},
333333
{"type":"Punctuation","value":";"},
334-
{"type":"Text","value":"\n"},
334+
{"type":"TextWhitespace","value":"\n"},
335335
{"type":"Keyword","value":"END"},
336336
{"type":"Punctuation","value":";"},
337-
{"type":"Text","value":"\n"},
337+
{"type":"TextWhitespace","value":"\n"},
338338
{"type":"LiteralStringHeredoc","value":"$emp_stamp$"},
339339
{"type":"Text","value":" "},
340340
{"type":"Keyword","value":"LANGUAGE"},
@@ -380,54 +380,54 @@
380380
{"type":"LiteralStringSingle","value":"plpgsql"},
381381
{"type":"Text","value":" "},
382382
{"type":"LiteralStringHeredoc","value":"$$"},
383-
{"type":"Text","value":"\n"},
383+
{"type":"TextWhitespace","value":"\n"},
384384
{"type":"Keyword","value":"declare"},
385-
{"type":"Text","value":" "},
385+
{"type":"TextWhitespace","value":" "},
386386
{"type":"Name","value":"r"},
387-
{"type":"Text","value":" "},
387+
{"type":"TextWhitespace","value":" "},
388388
{"type":"NameBuiltin","value":"record"},
389389
{"type":"Punctuation","value":";"},
390-
{"type":"Text","value":"\n"},
390+
{"type":"TextWhitespace","value":"\n"},
391391
{"type":"Keyword","value":"begin"},
392-
{"type":"Text","value":"\n "},
392+
{"type":"TextWhitespace","value":"\n "},
393393
{"type":"Keyword","value":"for"},
394-
{"type":"Text","value":" "},
394+
{"type":"TextWhitespace","value":" "},
395395
{"type":"Name","value":"r"},
396-
{"type":"Text","value":" "},
396+
{"type":"TextWhitespace","value":" "},
397397
{"type":"Keyword","value":"in"},
398-
{"type":"Text","value":" "},
398+
{"type":"TextWhitespace","value":" "},
399399
{"type":"Keyword","value":"select"},
400-
{"type":"Text","value":" "},
400+
{"type":"TextWhitespace","value":" "},
401401
{"type":"Operator","value":"*"},
402-
{"type":"Text","value":" "},
402+
{"type":"TextWhitespace","value":" "},
403403
{"type":"Keyword","value":"from"},
404-
{"type":"Text","value":" "},
404+
{"type":"TextWhitespace","value":" "},
405405
{"type":"Name","value":"books"},
406-
{"type":"Text","value":"\n"},
406+
{"type":"TextWhitespace","value":"\n"},
407407
{"type":"Keyword","value":"loop"},
408-
{"type":"Text","value":"\n "},
408+
{"type":"TextWhitespace","value":"\n "},
409409
{"type":"Keyword","value":"execute"},
410-
{"type":"Text","value":" "},
410+
{"type":"TextWhitespace","value":" "},
411411
{"type":"LiteralStringSingle","value":"'select '''"},
412-
{"type":"Text","value":" "},
412+
{"type":"TextWhitespace","value":" "},
413413
{"type":"Operator","value":"||"},
414-
{"type":"Text","value":" "},
414+
{"type":"TextWhitespace","value":" "},
415415
{"type":"Name","value":"r"},
416416
{"type":"LiteralNumberFloat","value":"."},
417417
{"type":"Name","value":"title"},
418-
{"type":"Text","value":" "},
418+
{"type":"TextWhitespace","value":" "},
419419
{"type":"Operator","value":"||"},
420-
{"type":"Text","value":" "},
420+
{"type":"TextWhitespace","value":" "},
421421
{"type":"LiteralStringSingle","value":"''''"},
422422
{"type":"Punctuation","value":";"},
423-
{"type":"Text","value":"\n"},
423+
{"type":"TextWhitespace","value":"\n"},
424424
{"type":"Keyword","value":"end"},
425-
{"type":"Text","value":" "},
425+
{"type":"TextWhitespace","value":" "},
426426
{"type":"Keyword","value":"loop"},
427427
{"type":"Punctuation","value":";"},
428-
{"type":"Text","value":"\n"},
428+
{"type":"TextWhitespace","value":"\n"},
429429
{"type":"Keyword","value":"end"},
430-
{"type":"Text","value":"\n"},
430+
{"type":"TextWhitespace","value":"\n"},
431431
{"type":"LiteralStringHeredoc","value":"$$"},
432432
{"type":"Punctuation","value":";"},
433433
{"type":"Text","value":"\n\n"},

0 commit comments

Comments
 (0)