Skip to content

Commit

Permalink
[Lexer] Fix wrong interaction backtick + multi-dollar interpolation
Browse files Browse the repository at this point in the history 
^KT-71073 Fixed
^KT-68971 Fixed
  • Loading branch information
@serras @qodana-bot
serras authored and qodana-bot committed Sep 5, 2024
1 parent 0a833b0 commit 2608d3a
Show file tree
Hide file tree
Showing 7 changed files with 665 additions and 289 deletions.
6 changes: 4 additions & 2 deletions compiler/psi/src/org/jetbrains/kotlin/lexer/Kotlin.flex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ THREE_OR_MORE_QUO = ({THREE_QUO}\"*)

REGULAR_STRING_PART=[^\\\"\n\$]+
SHORT_TEMPLATE_ENTRY={INTERPOLATION}{IDENTIFIER}
LONELY_DOLLAR=\$
LONELY_DOLLAR=\$+
LONG_TEMPLATE_ENTRY_START={INTERPOLATION}\{
LONELY_BACKTICK=`

Expand Down Expand Up @@ -181,9 +181,10 @@ LONELY_BACKTICK=`
int rest = yylength() - interpolationPrefix;
if (interpolationPrefix == requiredInterpolationPrefix) {
pushState(SHORT_TEMPLATE_ENTRY);
yypushback(yylength() - interpolationPrefix);
yypushback(rest);
return KtTokens.SHORT_TEMPLATE_ENTRY_START;
} else if (interpolationPrefix < requiredInterpolationPrefix) {
yypushback(rest);
return KtTokens.REGULAR_STRING_PART;
} else {
yypushback(requiredInterpolationPrefix + rest);
Expand All @@ -204,6 +205,7 @@ LONELY_BACKTICK=`
pushState(LONG_TEMPLATE_ENTRY);
return KtTokens.LONG_TEMPLATE_ENTRY_START;
} else if (interpolationPrefix < requiredInterpolationPrefix) {
yypushback(1);
return KtTokens.REGULAR_STRING_PART;
} else {
yypushback(requiredInterpolationPrefix + 1);
Expand Down
244 changes: 123 additions & 121 deletions compiler/psi/src/org/jetbrains/kotlin/lexer/_JetLexer.java
View file

Large diffs are not rendered by default.

166 changes: 83 additions & 83 deletions compiler/testData/diagnostics/tests/DisabledMultiDollarInterpolation.fir.txt
View file

Large diffs are not rendered by default.

166 changes: 83 additions & 83 deletions compiler/testData/diagnostics/tests/EnabledMultiDollarInterpolation.fir.txt
View file

Large diffs are not rendered by default.

36 changes: 36 additions & 0 deletions compiler/testData/psi/stringTemplates/MultiDollarBacktick.kt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
fun parserBug() {
// No closing backtick — not an identifier
"""
${"${"$`identifier"}"} "}"}
""".trimIndent()

// Two backticks — everything inside is an identifier
"""
${"${"$`identifier"}"} `"}"}
""".trimIndent()

// Escaped dollar, not an identifier
"""
${"${"\$`identifier"}"} `"}"}
""".trimIndent()

// Innermost string should not grab too much
"""
${"${$$$"$$`identifier"}"} `"}"}
""".trimIndent()

// Three dollars is the escape sequence, everything inside the backticks is an identifier
"""
${"${$$$"$$$`identifier"}"} `"}"}
""".trimIndent()

// Without the closing backtick the innermost string stops where expected
"""
${"${$$$"$$`identifier"}"} "}"}
""".trimIndent()

// on simple strings
$$$"$$`identifier"
"${$$$"$$`identifier"}
"${$$$"$$$`identifier`"}
}
331 changes: 331 additions & 0 deletions compiler/testData/psi/stringTemplates/MultiDollarBacktick.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,331 @@
KtFile: MultiDollarBacktick.kt
PACKAGE_DIRECTIVE
<empty list>
IMPORT_LIST
<empty list>
FUN
PsiElement(fun)('fun')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('parserBug')
VALUE_PARAMETER_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
PsiWhiteSpace(' ')
BLOCK
PsiElement(LBRACE)('{')
PsiWhiteSpace('\n ')
PsiComment(EOL_COMMENT)('// No closing backtick — not an identifier')
PsiWhiteSpace('\n ')
DOT_QUALIFIED_EXPRESSION
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"""')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('\n')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('$')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('`identifier')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('"')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('"')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('\n')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
PsiElement(CLOSING_QUOTE)('"""')
PsiElement(DOT)('.')
CALL_EXPRESSION
REFERENCE_EXPRESSION
PsiElement(IDENTIFIER)('trimIndent')
VALUE_ARGUMENT_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
PsiWhiteSpace('\n\n ')
PsiComment(EOL_COMMENT)('// Two backticks — everything inside is an identifier')
PsiWhiteSpace('\n ')
DOT_QUALIFIED_EXPRESSION
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"""')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('\n')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"')
SHORT_STRING_TEMPLATE_ENTRY
PsiElement(SHORT_TEMPLATE_ENTRY_START)('$')
REFERENCE_EXPRESSION
PsiElement(IDENTIFIER)('`identifier"}"} `')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('\n')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
PsiElement(CLOSING_QUOTE)('"""')
PsiElement(DOT)('.')
CALL_EXPRESSION
REFERENCE_EXPRESSION
PsiElement(IDENTIFIER)('trimIndent')
VALUE_ARGUMENT_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
PsiWhiteSpace('\n\n ')
PsiComment(EOL_COMMENT)('// Escaped dollar, not an identifier')
PsiWhiteSpace('\n ')
DOT_QUALIFIED_EXPRESSION
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"""')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('\n')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"')
ESCAPE_STRING_TEMPLATE_ENTRY
PsiElement(ESCAPE_SEQUENCE)('\$')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('`identifier')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' `')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('"')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('"')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('\n')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
PsiElement(CLOSING_QUOTE)('"""')
PsiElement(DOT)('.')
CALL_EXPRESSION
REFERENCE_EXPRESSION
PsiElement(IDENTIFIER)('trimIndent')
VALUE_ARGUMENT_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
PsiWhiteSpace('\n\n ')
PsiComment(EOL_COMMENT)('// Innermost string should not grab too much')
PsiWhiteSpace('\n ')
DOT_QUALIFIED_EXPRESSION
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"""')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('\n')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(INTERPOLATION_PREFIX)('$$$')
PsiElement(OPEN_QUOTE)('"')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('$$')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('`identifier')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' `')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('"')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('"')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('\n')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
PsiElement(CLOSING_QUOTE)('"""')
PsiElement(DOT)('.')
CALL_EXPRESSION
REFERENCE_EXPRESSION
PsiElement(IDENTIFIER)('trimIndent')
VALUE_ARGUMENT_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
PsiWhiteSpace('\n\n ')
PsiComment(EOL_COMMENT)('// Three dollars is the escape sequence, everything inside the backticks is an identifier')
PsiWhiteSpace('\n ')
DOT_QUALIFIED_EXPRESSION
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"""')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('\n')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(INTERPOLATION_PREFIX)('$$$')
PsiElement(OPEN_QUOTE)('"')
SHORT_STRING_TEMPLATE_ENTRY
PsiElement(SHORT_TEMPLATE_ENTRY_START)('$$$')
REFERENCE_EXPRESSION
PsiElement(IDENTIFIER)('`identifier"}"} `')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('\n')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
PsiElement(CLOSING_QUOTE)('"""')
PsiElement(DOT)('.')
CALL_EXPRESSION
REFERENCE_EXPRESSION
PsiElement(IDENTIFIER)('trimIndent')
VALUE_ARGUMENT_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
PsiWhiteSpace('\n\n ')
PsiComment(EOL_COMMENT)('// Without the closing backtick the innermost string stops where expected')
PsiWhiteSpace('\n ')
DOT_QUALIFIED_EXPRESSION
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"""')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('\n')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(INTERPOLATION_PREFIX)('$$$')
PsiElement(OPEN_QUOTE)('"')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('$$')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('`identifier')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('"')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('"')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('}')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('\n')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)(' ')
PsiElement(CLOSING_QUOTE)('"""')
PsiElement(DOT)('.')
CALL_EXPRESSION
REFERENCE_EXPRESSION
PsiElement(IDENTIFIER)('trimIndent')
VALUE_ARGUMENT_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
PsiWhiteSpace('\n\n ')
PsiComment(EOL_COMMENT)('// on simple strings')
PsiWhiteSpace('\n ')
STRING_TEMPLATE
PsiElement(INTERPOLATION_PREFIX)('$$$')
PsiElement(OPEN_QUOTE)('"')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('$$')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('`identifier')
PsiElement(CLOSING_QUOTE)('"')
PsiWhiteSpace('\n ')
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(INTERPOLATION_PREFIX)('$$$')
PsiElement(OPEN_QUOTE)('"')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('$$')
LITERAL_STRING_TEMPLATE_ENTRY
PsiElement(REGULAR_STRING_PART)('`identifier')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
PsiErrorElement:Expecting '"'
<empty list>
PsiWhiteSpace('\n ')
STRING_TEMPLATE
PsiElement(OPEN_QUOTE)('"')
LONG_STRING_TEMPLATE_ENTRY
PsiElement(LONG_TEMPLATE_ENTRY_START)('${')
STRING_TEMPLATE
PsiElement(INTERPOLATION_PREFIX)('$$$')
PsiElement(OPEN_QUOTE)('"')
SHORT_STRING_TEMPLATE_ENTRY
PsiElement(SHORT_TEMPLATE_ENTRY_START)('$$$')
REFERENCE_EXPRESSION
PsiElement(IDENTIFIER)('`identifier`')
PsiElement(CLOSING_QUOTE)('"')
PsiElement(LONG_TEMPLATE_ENTRY_END)('}')
PsiErrorElement:Expecting '"'
<empty list>
PsiWhiteSpace('\n')
PsiElement(RBRACE)('}')
Loading

0 comments on commit 2608d3a

Please sign in to comment.