Skip to content
This repository has been archived by the owner on Nov 5, 2021. It is now read-only.

Added support for Perl quote-like operators to fix #1101 #50

Merged
merged 5 commits into from
Oct 17, 2018
Merged

Added support for Perl quote-like operators to fix #1101 #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6feaf43
Added tests for quoted strings and regexes using alternative delimiters
goober99 Oct 10, 2018
80e9270
Separated quote-like operators from other keywords
goober99 Oct 10, 2018
40fdb5d
Added support for Perl quote-like operators to fix #1101
goober99 Oct 10, 2018
3928208
Fixed failing tests due to regexp modifiers being handled differently
goober99 Oct 10, 2018
8694e28
Modified original regex rule to only catch instances of bare /regex/
goober99 Oct 10, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 46 additions & 2 deletions src/perl/perl.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,14 +310,23 @@ testTokenization('perl', [
[
{
line: 'm/abc/',
tokens: [{ startIndex: 0, type: 'regexp.perl' }],
tokens: [
{ startIndex: 0, type: 'regexp.delim.perl' },
{ startIndex: 2, type: 'regexp.perl' },
{ startIndex: 5, type: 'regexp.delim.perl' },
],
},
],

[
{
line: 'm/[abc]+/e',
tokens: [{ startIndex: 0, type: 'regexp.perl' }],
tokens: [
{ startIndex: 0, type: 'regexp.delim.perl' },
{ startIndex: 2, type: 'regexp.perl' },
{ startIndex: 8, type: 'regexp.delim.perl' },
{ startIndex: 9, type: 'regexp.modifier.perl' },
],
},
],

Expand Down Expand Up @@ -429,4 +438,39 @@ testTokenization('perl', [
],
},
],

// Quoted constructs
[
{
line: "m!can't!",
tokens: [
{ startIndex: 0, type: 'regexp.delim.perl' },
{ startIndex: 2, type: 'regexp.perl' },
{ startIndex: 7, type: 'regexp.delim.perl' },
],
},
],

[
{
line: 'q XfooX',
tokens: [
{ startIndex: 0, type: 'string.delim.perl' },
{ startIndex: 3, type: 'string.perl' },
{ startIndex: 6, type: 'string.delim.perl' },
],
},
],

[
{
line: 'qq(test $foo)',
tokens: [
{ startIndex: 0, type: 'string.delim.perl' },
{ startIndex: 3, type: 'string.perl' },
{ startIndex: 8, type: 'variable.perl' },
{ startIndex: 12, type: 'string.delim.perl' },
],
},
],
]);
88 changes: 77 additions & 11 deletions src/perl/perl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,23 +48,18 @@ export const language = <ILanguage>{
'__DATA__',
'else',
'lock',
'qw',
'__END__',
'elsif',
'lt',
'qx',
'__FILE__',
'eq',
'm',
's',
'__LINE__',
'exp',
'ne',
'sub',
'__PACKAGE__',
'for',
'no',
'tr',
'and',
'foreach',
'or',
Expand All @@ -75,16 +70,12 @@ export const language = <ILanguage>{
'until',
'continue',
'gt',
'q',
'while',
'CORE',
'if',
'qq',
'xor',
'do',
'le',
'qr',
'y',

'__DIE__',
'__WARN__',
Expand Down Expand Up @@ -468,6 +459,7 @@ export const language = <ILanguage>{

// operators
symbols: /[:+\-\^*$&%@=<>!?|\/~\.]/,
quoteLikeOps: ['qr', 'm', 's', 'q', 'qq', 'qx', 'qw', 'tr', 'y'],

escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,

Expand All @@ -477,12 +469,13 @@ export const language = <ILanguage>{
{ include: '@whitespace' },

[
/[a-zA-Z\-_][\w\-_]+/,
/[a-zA-Z\-_][\w\-_]*/,
{
cases: {
'@keywords': 'keyword',
'@builtinFunctions': 'type.identifier',
'@builtinFileHandlers': 'variable.predefined',
'@quoteLikeOps': { token: '@rematch', next: 'quotedConstructs' },
'@default': '',
},
},
Expand Down Expand Up @@ -512,7 +505,7 @@ export const language = <ILanguage>{

// RegExp
[
/[goseximacplud]*[\/](?:(?:\[(?:\\]|[^\]])+\])|(?:\\\/|[^\]\/]))*[\/]\w*\s*(?=[).,;]|$)/,
/[\/](?:(?:\[(?:\\]|[^\]])+\])|(?:\\\/|[^\]\/]))*[\/]\w*\s*(?=[).,;]|$)/,
'regexp',
],

Expand Down Expand Up @@ -555,6 +548,79 @@ export const language = <ILanguage>{
[/./, 'string'],
],

// Quoted constructs
// Percent strings in Ruby are similar to quote-like operators in Perl.
// This is adapted from pstrings in ../ruby/ruby.ts.
quotedConstructs: [
[/(q|qw|tr|y)\s*\(/, { token: 'string.delim', switchTo: '@qstring.(.)' }],
[/(q|qw|tr|y)\s*\[/, { token: 'string.delim', switchTo: '@qstring.[.]' }],
[/(q|qw|tr|y)\s*\{/, { token: 'string.delim', switchTo: '@qstring.{.}' }],
[/(q|qw|tr|y)\s*</, { token: 'string.delim', switchTo: '@qstring.<.>' }],
[/(q|qw|tr|y)#/, { token: 'string.delim', switchTo: '@qstring.#.#' }],
[/(q|qw|tr|y)\s*([^A-Za-z0-9#\s])/, { token: 'string.delim', switchTo: '@qstring.$2.$2' }],
[/(q|qw|tr|y)\s+(\w)/, { token: 'string.delim', switchTo: '@qstring.$2.$2' }],

[/(qr|m|s)\s*\(/, { token: 'regexp.delim', switchTo: '@qregexp.(.)' }],
[/(qr|m|s)\s*\[/, { token: 'regexp.delim', switchTo: '@qregexp.[.]' }],
[/(qr|m|s)\s*\{/, { token: 'regexp.delim', switchTo: '@qregexp.{.}' }],
[/(qr|m|s)\s*</, { token: 'regexp.delim', switchTo: '@qregexp.<.>' }],
[/(qr|m|s)#/, { token: 'regexp.delim', switchTo: '@qregexp.#.#' }],
[/(qr|m|s)\s*([^A-Za-z0-9_#\s])/, { token: 'regexp.delim', switchTo: '@qregexp.$2.$2' }],
[/(qr|m|s)\s+(\w)/, { token: 'regexp.delim', switchTo: '@qregexp.$2.$2' }],

[/(qq|qx)\s*\(/, { token: 'string.delim', switchTo: '@qqstring.(.)' }],
[/(qq|qx)\s*\[/, { token: 'string.delim', switchTo: '@qqstring.[.]' }],
[/(qq|qx)\s*\{/, { token: 'string.delim', switchTo: '@qqstring.{.}' }],
[/(qq|qx)\s*</, { token: 'string.delim', switchTo: '@qqstring.<.>' }],
[/(qq|qx)#/, { token: 'string.delim', switchTo: '@qqstring.#.#' }],
[/(qq|qx)\s*([^A-Za-z0-9#\s])/, { token: 'string.delim', switchTo: '@qqstring.$2.$2' }],
[/(qq|qx)\s+(\w)/, { token: 'string.delim', switchTo: '@qqstring.$2.$2' }],
],

// Non-expanded quoted string
// qstring<open>.<close>
// open = open delimiter
// close = close delimiter
qstring: [
[/\\./, 'string.escape'],
[/./, {
cases: {
'$#==$S3': { token: 'string.delim', next: '@pop' },
'$#==$S2': { token: 'string.delim', next: '@push' }, // nested delimiters
'@default': 'string'
}
}],
],

// Quoted regexp
// qregexp.<open>.<close>
// open = open delimiter
// close = close delimiter
qregexp: [
{ include: '@variables' },
[/\\./, 'regexp.escape'],
[/./, {
cases: {
'$#==$S3': { token: 'regexp.delim', next: '@regexpModifiers' },
'$#==$S2': { token: 'regexp.delim', next: '@push' }, // nested delimiters
'@default': 'regexp'
}
}],
],

regexpModifiers: [
[/[msixpodualngcer]+/, { token: 'regexp.modifier', next: '@popall' }],
],

// Expanded quoted string
// qqstring.<open>.<close>
// open = open delimiter
// close = close delimiter
qqstring: [
{ include: '@variables' },
{ include: '@qstring' },
],

heredoc: [
[
/<<\s*['"`]?([\w\-]+)['"`]?/,
Expand Down