Skip to content

Commit

Permalink
[JLINE-730] Support for comments in DefaultParser (#731)
Browse files Browse the repository at this point in the history
  • Loading branch information
@snuyanzin
snuyanzin authored Nov 8, 2021
1 parent f89e28a commit 1315fc0
Show file tree
Hide file tree
Showing 2 changed files with 297 additions and 9 deletions.
146 changes: 137 additions & 9 deletions reader/src/main/java/org/jline/reader/impl/DefaultParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,27 @@ public enum Bracket {
ANGLE // <>
}

public static class BlockCommentDelims {
private final String start;
private final String end;
public BlockCommentDelims(String start, String end) {
if (start == null || end == null
|| start.isEmpty() || end.isEmpty() || start.equals(end)) {
throw new IllegalArgumentException("Bad block comment delimiter!");
}
this.start = start;
this.end = end;
}

public String getStart() {
return start;
}

public String getEnd() {
return end;
}
}

private char[] quoteChars = {'\'', '"'};

private char[] escapeChars = {'\\'};
Expand All @@ -39,6 +60,10 @@ public enum Bracket {

private char[] closingBrackets = null;

private String[] lineCommentDelims = null;

private BlockCommentDelims blockCommentDelims = null;

private String regexVariable = "[a-zA-Z_]+[a-zA-Z0-9_-]*((\\.|\\['|\\[\"|\\[)[a-zA-Z0-9_-]*(|']|\"]|]))?";
private String regexCommand = "[:]?[a-zA-Z]+[a-zA-Z0-9_-]*";
private int commandGroup = 4;
Expand All @@ -47,6 +72,16 @@ public enum Bracket {
// Chainable setters
//

public DefaultParser lineCommentDelims(final String[] lineCommentDelims) {
this.lineCommentDelims = lineCommentDelims;
return this;
}

public DefaultParser blockCommentDelims(final BlockCommentDelims blockCommentDelims) {
this.blockCommentDelims = blockCommentDelims;
return this;
}

public DefaultParser quoteChars(final char[] chars) {
this.quoteChars = chars;
return this;
Expand Down Expand Up @@ -107,6 +142,22 @@ public char[] getEscapeChars() {
return this.escapeChars;
}

public void setLineCommentDelims(String[] lineCommentDelims) {
this.lineCommentDelims = lineCommentDelims;
}

public String[] getLineCommentDelims() {
return this.lineCommentDelims;
}

public void setBlockCommentDelims(BlockCommentDelims blockCommentDelims) {
this.blockCommentDelims = blockCommentDelims;
}

public BlockCommentDelims getBlockCommentDelims() {
return blockCommentDelims;
}

public void setEofOnUnclosedQuote(boolean eofOnUnclosedQuote) {
this.eofOnUnclosedQuote = eofOnUnclosedQuote;
}
Expand Down Expand Up @@ -225,6 +276,11 @@ public ParsedLine parse(final String line, final int cursor, ParseContext contex
int rawWordStart = 0;
BracketChecker bracketChecker = new BracketChecker(cursor);
boolean quotedWord = false;
boolean lineCommented = false;
boolean blockCommented = false;
boolean blockCommentInRightOrder = true;
final String blockCommentEnd = blockCommentDelims == null ? null : blockCommentDelims.end;
final String blockCommentStart = blockCommentDelims == null ? null : blockCommentDelims.start;

for (int i = 0; (line != null) && (i < line.length()); i++) {
// once we reach the cursor, set the
Expand All @@ -237,7 +293,7 @@ public ParsedLine parse(final String line, final int cursor, ParseContext contex
rawWordCursor = i - rawWordStart;
}

if (quoteStart < 0 && isQuoteChar(line, i)) {
if (quoteStart < 0 && isQuoteChar(line, i) && !lineCommented && !blockCommented) {
// Start a quote block
quoteStart = i;
if (current.length()==0) {
Expand All @@ -258,17 +314,40 @@ public ParsedLine parse(final String line, final int cursor, ParseContext contex
quoteStart = -1;
quotedWord = false;
} else if (quoteStart < 0 && isDelimiter(line, i)) {
// Delimiter
if (current.length() > 0) {
words.add(current.toString());
current.setLength(0); // reset the arg
if (rawWordCursor >= 0 && rawWordLength < 0) {
rawWordLength = i - rawWordStart;
if (lineCommented) {
if (isCommentDelim(line, i, System.lineSeparator())) {
lineCommented = false;
}
} else if (blockCommented) {
if (isCommentDelim(line, i, blockCommentEnd)) {
blockCommented = false;
}
} else {
// Delimiter
rawWordLength = handleDelimiterAndGetRawWordLength(current, words, rawWordStart, rawWordCursor, rawWordLength, i);
rawWordStart = i + 1;
}
rawWordStart = i + 1;
} else {
if (!isEscapeChar(line, i)) {
if (quoteStart < 0 && !blockCommented && (lineCommented || isLineCommentStarted(line, i))) {
lineCommented = true;
} else if (quoteStart < 0 && !lineCommented
&& (blockCommented || isCommentDelim(line, i, blockCommentStart))) {
if (blockCommented) {
if (blockCommentEnd != null && isCommentDelim(line, i, blockCommentEnd)) {
blockCommented = false;
i += blockCommentEnd.length() - 1;
}
} else {
blockCommented = true;
rawWordLength = handleDelimiterAndGetRawWordLength(current, words, rawWordStart, rawWordCursor, rawWordLength, i);
i += blockCommentStart == null ? 0 : blockCommentStart.length() - 1;
rawWordStart = i + 1;
}
} else if (quoteStart < 0 && !lineCommented
&& isCommentDelim(line, i, blockCommentEnd)) {
current.append(line.charAt(i));
blockCommentInRightOrder = false;
} else if (!isEscapeChar(line, i)) {
current.append(line.charAt(i));
if (quoteStart < 0) {
bracketChecker.check(line, i);
Expand Down Expand Up @@ -301,6 +380,14 @@ public ParsedLine parse(final String line, final int cursor, ParseContext contex
throw new EOFError(-1, -1, "Missing closing quote", line.charAt(quoteStart) == '\''
? "quote" : "dquote");
}
if (blockCommented) {
throw new EOFError(-1, -1, "Missing closing block comment delimiter",
"add: " + blockCommentEnd);
}
if (!blockCommentInRightOrder) {
throw new EOFError(-1, -1, "Missing opening block comment delimiter",
"missing: " + blockCommentStart);
}
if (bracketChecker.isClosingBracketMissing() || bracketChecker.isOpeningBracketMissing()) {
String message = null;
String missing = null;
Expand Down Expand Up @@ -333,6 +420,17 @@ public boolean isDelimiter(final CharSequence buffer, final int pos) {
return !isQuoted(buffer, pos) && !isEscaped(buffer, pos) && isDelimiterChar(buffer, pos);
}

private int handleDelimiterAndGetRawWordLength(StringBuilder current, List<String> words, int rawWordStart, int rawWordCursor, int rawWordLength, int pos) {
if (current.length() > 0) {
words.add(current.toString());
current.setLength(0); // reset the arg
if (rawWordCursor >= 0 && rawWordLength < 0) {
return pos - rawWordStart;
}
}
return rawWordLength;
}

public boolean isQuoted(final CharSequence buffer, final int pos) {
return false;
}
Expand All @@ -351,6 +449,36 @@ public boolean isQuoteChar(final CharSequence buffer, final int pos) {
return false;
}

private boolean isCommentDelim(final CharSequence buffer, final int pos, final String pattern) {
if (pos < 0) {
return false;
}

if (pattern != null) {
final int length = pattern.length();
if (length <= buffer.length() - pos) {
for (int i = 0; i < length; i++) {
if (pattern.charAt(i) != buffer.charAt(pos + i)) {
return false;
}
}
return true;
}
}
return false;
}

public boolean isLineCommentStarted(final CharSequence buffer, final int pos) {
if (lineCommentDelims != null) {
for (String comment: lineCommentDelims) {
if (isCommentDelim(buffer, pos, comment)) {
return true;
}
}
}
return false;
}

@Override
public boolean isEscapeChar(char ch) {
if (escapeChars != null) {
Expand Down
160 changes: 160 additions & 0 deletions reader/src/test/java/org/jline/reader/completer/DefaultParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
package org.jline.reader.completer;

import java.util.Arrays;
import java.util.Collections;

import org.jline.reader.EOFError;
import org.jline.reader.ParsedLine;
import org.jline.reader.impl.DefaultParser;
import org.jline.reader.impl.ReaderTestSupport;
Expand Down Expand Up @@ -87,4 +89,162 @@ public void testEscapedQuotes() {
delimited = parser.parse("'1 '2\\' 3", 0);
assertEquals(Arrays.asList("1 2'", "3"), delimited.words());
}

@Test(expected = IllegalArgumentException.class)
public void testNullStartBlockCommentDelim() {
parser.setBlockCommentDelims(new DefaultParser.BlockCommentDelims(null, "*/"));
}

@Test(expected = IllegalArgumentException.class)
public void testNullEndBlockCommentsDelim() {
parser.setBlockCommentDelims(new DefaultParser.BlockCommentDelims("/*", null));
}

@Test(expected = IllegalArgumentException.class)
public void testEqualBlockCommentsDelims() {
parser.setBlockCommentDelims(new DefaultParser.BlockCommentDelims("/*", "/*"));
}

@Test(expected = IllegalArgumentException.class)
public void testEmptyStartBlockCommentsDelims() {
parser.setBlockCommentDelims(new DefaultParser.BlockCommentDelims("", "*/"));
}

@Test(expected = IllegalArgumentException.class)
public void testEmptyEndBlockCommentsDelims() {
parser.setBlockCommentDelims(new DefaultParser.BlockCommentDelims("/*", ""));
}

@Test
public void testBashComments() {
parser.setLineCommentDelims(new String[] {"#"});
delimited = parser.parse("1 2 # 3", 0);
assertEquals(Arrays.asList("1", "2"), delimited.words());

delimited = parser.parse("#\\'1 '2' 3", 0);
assertEquals(Collections.emptyList(), delimited.words());

delimited = parser.parse("'#'\\'1 '2' 3", 0);
assertEquals(Arrays.asList("#'1", "2", "3"), delimited.words());

delimited = parser.parse("#1 " + System.lineSeparator() + " '2' 3", 0);
assertEquals(Arrays.asList("2", "3"), delimited.words());
}

@Test
public void testJavaComments() {
parser.setLineCommentDelims(new String[] {"//"});
parser.setBlockCommentDelims(new DefaultParser.BlockCommentDelims("/*", "*/"));

delimited = parser.parse("1 2 # 3", 0);
assertEquals(Arrays.asList("1", "2", "#", "3"), delimited.words());

delimited = parser.parse("1 2 // 3", 0);
assertEquals(Arrays.asList("1", "2"), delimited.words());

delimited = parser.parse("/*\\'1 \n '2' \n3*/", 0);
assertEquals(Collections.emptyList(), delimited.words());

delimited = parser.parse("'//'\\'1 /*'2'\n */3", 0);
assertEquals(Arrays.asList("//'1", "3"), delimited.words());


delimited = parser.parse("hello/*comment*/world", 0);
assertEquals(Arrays.asList("hello", "world"), delimited.words());
}

@Test
public void testSqlComments() {
// The test check sql line comment --
// and sql block comments /* */
parser.setLineCommentDelims(new String[]{"--"});
parser.setBlockCommentDelims(new DefaultParser.BlockCommentDelims("/*", "*/"));

delimited = parser.parse("/*/g */", 0);
assertEquals(Collections.emptyList(), delimited.words());

delimited = parser.parse("/**/g", 0);
assertEquals(Arrays.asList("g"), delimited.words());

delimited = parser.parse("select '--';", 0);
assertEquals(Arrays.asList("select", "--;"), delimited.words());
delimited = parser.parse("select --; '--';", 0);
assertEquals(Arrays.asList("select"), delimited.words());

delimited = parser.parse("select 1/* 789*/ ; '--';", 0);
assertEquals(Arrays.asList("select", "1", ";", "--;"), delimited.words());
delimited = parser.parse("select 1/* 789 \n */ ; '--';", 0);
assertEquals(Arrays.asList("select", "1", ";", "--;"), delimited.words());

delimited = parser.parse("select 1/* 789 \n * / ; '--';*/", 0);
assertEquals(Arrays.asList("select", "1"), delimited.words());
delimited = parser.parse("select '1';--comment", 0);
assertEquals(Arrays.asList("select", "1;"), delimited.words());

delimited = parser.parse("select '1';-----comment", 0);
assertEquals(Arrays.asList("select", "1;"), delimited.words());

delimited = parser.parse("select '1';--comment\n", 0);
assertEquals(Arrays.asList("select", "1;"), delimited.words());

delimited = parser.parse("select '1';--comment\n\n", 0);
assertEquals(Arrays.asList("select", "1;"), delimited.words());

delimited = parser.parse("select '1'; --comment", 0);
assertEquals(Arrays.asList("select", "1;"), delimited.words());

delimited = parser.parse("select '1';\n--comment", 0);
assertEquals(Arrays.asList("select", "1;"), delimited.words());

delimited = parser.parse("select '1';\n\n--comment", 0);
assertEquals(Arrays.asList("select", "1;"), delimited.words());

delimited = parser.parse("select '1';\n \n--comment", 0);
assertEquals(Arrays.asList("select", "1;"), delimited.words());

delimited = parser.parse("select '1'\n;\n--comment", 0);
assertEquals(Arrays.asList("select", "1", ";"), delimited.words());

delimited = parser.parse("select '1'\n\n;--comment", 0);
assertEquals(Arrays.asList("select", "1", ";"), delimited.words());

delimited = parser.parse("select '1'\n\n;---comment", 0);
assertEquals(Arrays.asList("select", "1", ";"), delimited.words());

delimited = parser.parse("select '1'\n\n;-- --comment", 0);
assertEquals(Arrays.asList("select", "1", ";"), delimited.words());

delimited = parser.parse("select '1'\n\n;\n--comment", 0);
assertEquals(Arrays.asList("select", "1", ";"), delimited.words());

delimited = parser.parse("select '1'/*comment*/", 0);
assertEquals(Arrays.asList("select", "1"), delimited.words());

delimited = parser.parse("select '1';/*---comment */", 0);
assertEquals(Arrays.asList("select", "1;"), delimited.words());

delimited = parser.parse("select '1';/*comment\n*/\n", 0);
assertEquals(Arrays.asList("select", "1;"), delimited.words());

delimited = parser.parse("select '1';/*comment*/\n\n", 0);
assertEquals(Arrays.asList("select", "1;"), delimited.words());

delimited = parser.parse("select '1'; /*--comment*/", 0);
assertEquals(Arrays.asList("select", "1;"), delimited.words());

delimited = parser.parse("select '1/*' as \"asd\";", 0);
assertEquals(Arrays.asList("select", "1/*", "as", "asd;"), delimited.words());

delimited = parser.parse("select '/*' as \"asd*/\";", 0);
assertEquals(Arrays.asList("select", "/*", "as", "asd*/;"), delimited.words());

delimited = parser.parse("select '1' as \"'a'\\\ns'd\\\n\n\" from t;", 0);
assertEquals(Arrays.asList("select", "1", "as", "'a'\ns'd\n\n", "from", "t;"), delimited.words());
}

@Test(expected = EOFError.class)
public void testMissingOpeningBlockComment() {
parser.setBlockCommentDelims(new DefaultParser.BlockCommentDelims("/*", "*/"));
delimited = parser.parse("1, 2, 3 */", 0);
}
}

0 comments on commit 1315fc0

Please sign in to comment.