+ private boolean noAndWithinBetween() {
+ return !(getToken(1).kind == AND && token_source.parentToken == BETWEEN);
+ }
+
+ /**
+ * Parse given SQL.
+ *
+ * @deprecated This method will be removed in the near future.
+ *
+ * Use {@link #parse(String, ClickHouseProperties)} instead.
+ *
+ * @param sql SQL query
+ * @param properties properties
+ * @return parsed SQL statement
+ */
+ public static ClickHouseSqlStatement parseSingleStatement(String sql, ClickHouseProperties properties) {
+ return parseSingleStatement(sql, properties, null);
+ }
+
+ /**
+ * Parse given SQL.
+ *
+ * @deprecated This method will be removed in the near future.
+ *
+ * Use {@link #parse(String, ClickHouseProperties, ParseHandler)} instead.
+ *
+ * @param sql SQL query
+ * @param properties properties
+ * @param handler parse handler
+ * @return parsed SQL statement
+ */
+ public static ClickHouseSqlStatement parseSingleStatement(
+ String sql, ClickHouseProperties properties, ParseHandler handler) {
+ ClickHouseSqlStatement[] stmts = parse(sql, properties, handler);
+
+ return stmts.length == 1 ? stmts[0] : new ClickHouseSqlStatement(sql, StatementType.UNKNOWN);
+ }
+
+ public static ClickHouseSqlStatement[] parse(String sql, ClickHouseProperties properties) {
+ return parse(sql, properties, null);
+ }
+
+ public static ClickHouseSqlStatement[] parse(String sql, ClickHouseProperties properties, ParseHandler handler) {
+ if (properties == null) {
+ properties = new ClickHouseProperties();
+ }
+
+ ClickHouseSqlStatement[] stmts = new ClickHouseSqlStatement[] {
+ new ClickHouseSqlStatement(sql, StatementType.UNKNOWN) };
+
+ if (!properties.isUseNewParser() || sql == null || sql.isEmpty()) {
+ return stmts;
+ }
+
+ ClickHouseSqlParser p = new ClickHouseSqlParser(sql, properties, handler);
+ try {
+ stmts = p.sql();
+ } catch (Exception e) {
+ if (DEBUG) {
+ throw new IllegalArgumentException(e);
+ } else {
+ log.warn("Failed to parse the given SQL. If you believe the SQL is valid, please feel free to open an issue on Github with the following SQL and exception attached.\n{}", sql, e);
+ }
+ }
+
+ return stmts;
+ }
+
+ public ClickHouseSqlParser(String sql, ClickHouseProperties properties, ParseHandler handler) {
+ this(new StringReader(sql));
+
+ this.properties = properties;
+ this.handler = handler;
+ }
+
+ public void addStatement() {
+ if (token_source.isValid()) {
+ ClickHouseSqlStatement sqlStmt = token_source.build(handler);
+ // FIXME remove the restriction once we can hanlde insertion with format well
+ if (statements.size() == 0 || sqlStmt.isRecognized()) {
+ statements.add(sqlStmt);
+ }
+ } else {
+ token_source.reset();
+ }
+ }
+}
+
+PARSER_END(ClickHouseSqlParser)
+
+TOKEN_MGR_DECLS: {
+ // whitespaces and comments are invalid
+ private int validTokens = 0;
+ // see http://www.engr.mun.ca/~theo/JavaCC-FAQ/javacc-faq-moz.htm#tth_sEc3.17
+ private int commentNestingDepth = 0;
+
+ final java.util.Deque stack = new java.util.LinkedList<>();
+ int parentToken = -1;
+
+ final StringBuilder builder = new StringBuilder();
+
+ StatementType stmtType = StatementType.UNKNOWN;
+ String cluster = null;
+ String database = null;
+ String table = null;
+ String format = null;
+ String outfile = null;
+
+ final List parameters = new ArrayList<>();
+ final Map positions = new HashMap<>();
+
+ public void CommonTokenAction(Token t) {
+ if (t.kind != ClickHouseSqlParserConstants.SEMICOLON) {
+ builder.append(t.image);
+
+ if (t.kind != ClickHouseSqlParserConstants.EOF) {
+ validTokens++;
+ }
+ }
+ }
+
+ void enterToken(int tokenKind) {
+ if (tokenKind < 0) {
+ return;
+ }
+
+ stack.push(parentToken = tokenKind);
+ }
+
+ void leaveToken(int tokenKind) {
+ if (parentToken == tokenKind) {
+ stack.pop();
+ }
+
+ parentToken = stack.isEmpty() ? -1 : stack.getLast();
+ }
+
+ void processMacro(String name, List params, ParseHandler handler) {
+ StringBuilder m = new StringBuilder();
+ m.append('#').append(name);
+
+ int startPos = builder.lastIndexOf(m.toString());
+ int endPos = params.size() > 0 ? builder.indexOf(")", startPos) + 1 : startPos + m.length();
+
+ builder.delete(startPos, endPos);
+ if (handler != null) {
+ String replacement = handler.handleMacro(name, params);
+ if (replacement != null && !replacement.isEmpty()) {
+ builder.insert(startPos, replacement);
+ }
+ }
+ }
+
+ void processParameter(String str, ParseHandler handler) {
+ int pos = builder.lastIndexOf(str);
+ parameters.add(pos);
+
+ if (handler != null) {
+ String replacement = handler.handleParameter(cluster, database, table, parameters.size());
+ if (replacement != null && !replacement.isEmpty()) {
+ builder.deleteCharAt(pos);
+ builder.insert(pos, replacement);
+ }
+ }
+ }
+
+ void append(StringBuilder str) {
+ builder.append(str.toString());
+ }
+
+ void reset() {
+ stack.clear();
+ parentToken = -1;
+
+ builder.setLength(validTokens = 0);
+
+ stmtType = StatementType.UNKNOWN;
+ cluster = null;
+ database = null;
+ table = null;
+ format = null;
+ outfile = null;
+ parameters.clear();
+ positions.clear();
+ }
+
+ ClickHouseSqlStatement build(ParseHandler handler) {
+ String sqlStmt = builder.toString();
+ ClickHouseSqlStatement s = null;
+ if (handler != null) {
+ s = handler.handleStatement(
+ sqlStmt, stmtType, cluster, database, table, format, outfile, parameters, positions);
+ }
+
+ if (s == null) {
+ s = new ClickHouseSqlStatement(
+ sqlStmt, stmtType, cluster, database, table, format, outfile, parameters, positions);
+ }
+
+ // reset variables
+ reset();
+
+ return s;
+ }
+
+ boolean isValid() {
+ return validTokens > 0;
+ }
+
+ void setPosition(String keyword) {
+ if (keyword == null || keyword.isEmpty()) {
+ return;
+ }
+
+ this.positions.put(keyword, builder.lastIndexOf(keyword));
+ }
+}
+
+SKIP: {
+
+ { append(image); }
+ | { append(image); }
+ | "/*" { commentNestingDepth = 1; append(image); }: MULTI_LINE_COMMENT
+}
+
+ SKIP: {
+ "/*" { commentNestingDepth += 1; append(image); }
+ | "*/" { SwitchTo(--commentNestingDepth == 0 ? DEFAULT : MULTI_LINE_COMMENT); append(image); }
+ | < ~[] > { append(image); }
+}
+
+// top-level statements
+ClickHouseSqlStatement[] sql(): {} {
+ stmts()
+ { addStatement(); }
+ (
+ (LOOKAHEAD(2) )+
+ (stmts())?
+ { addStatement(); }
+ )*
+
+ { return statements.toArray(new ClickHouseSqlStatement[statements.size()]); }
+}
+
+void stmts(): { Token t; } {
+ LOOKAHEAD(2) stmt()
+ | LOOKAHEAD(2) anyExprList() // in case there's anything new
+}
+
+void stmt(): {} {
+ alterStmt() { if (token_source.stmtType == StatementType.UNKNOWN) token_source.stmtType = StatementType.ALTER; }
+ | attachStmt() { token_source.stmtType = StatementType.ATTACH; }
+ | checkStmt() { token_source.stmtType = StatementType.CHECK; }
+ | createStmt() { token_source.stmtType = StatementType.CREATE; }
+ | deleteStmt() { token_source.stmtType = StatementType.DELETE; }
+ | describeStmt() { token_source.stmtType = StatementType.DESCRIBE; }
+ | detachStmt() { token_source.stmtType = StatementType.DETACH; }
+ | dropStmt() { token_source.stmtType = StatementType.DROP; }
+ | existsStmt() { token_source.stmtType = StatementType.EXISTS; }
+ | explainStmt() { token_source.stmtType = StatementType.EXPLAIN; }
+ | insertStmt() { token_source.stmtType = StatementType.INSERT; }
+ | grantStmt() { token_source.stmtType = StatementType.GRANT; }
+ | killStmt() { token_source.stmtType = StatementType.KILL; }
+ | optimizeStmt() { token_source.stmtType = StatementType.OPTIMIZE; }
+ | renameStmt() { token_source.stmtType = StatementType.RENAME; }
+ | revokeStmt() { token_source.stmtType = StatementType.REVOKE; }
+ | selectStmt() { token_source.stmtType = StatementType.SELECT; }
+ | setStmt() { token_source.stmtType = StatementType.SET; }
+ | showStmt() { token_source.stmtType = StatementType.SHOW; }
+ | systemStmt() { token_source.stmtType = StatementType.SYSTEM; }
+ | truncateStmt() { token_source.stmtType = StatementType.TRUNCATE; }
+ | updateStmt() { token_source.stmtType = StatementType.UPDATE; }
+ | useStmt() { token_source.stmtType = StatementType.USE; }
+ | watchStmt() { token_source.stmtType = StatementType.WATCH; }
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/alter/
+void alterStmt(): {} {
+
+ (
+ LOOKAHEAD(2)
+ tableIdentifier(true) (LOOKAHEAD(2) clusterClause())? (
+ LOOKAHEAD({ !tokenIn(1, UPDATE, DELETE) }) anyIdentifier()
+ | { token_source.stmtType = StatementType.ALTER_UPDATE; }
+ | { token_source.stmtType = StatementType.ALTER_DELETE; }
+ )
+ )? (anyExprList())?
+}
+
+void clusterClause(): { Token t; } {
+ (LOOKAHEAD(2) t = anyIdentifier() | t = )
+ { token_source.cluster = ClickHouseSqlUtils.unescape(t.image); }
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/attach/
+void attachStmt(): { Token t; } {
+ (
+ LOOKAHEAD(2)
+ (
+ t = { token_source.setPosition(t.image); }
+ |
+ | ( t = { token_source.setPosition(t.image); }) (
+ ()? | ( | )?
+ )
+ )
+ (
+ LOOKAHEAD(2)
+ t = { token_source.setPosition(t.image); }
+ )?
+ )?
+ anyExprList() // not interested
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/check-table/
+void checkStmt(): {} { // not interested
+ anyExprList()
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/create/
+void createStmt(): { Token t; } {
+ (
+ LOOKAHEAD(2)
+ (
+ t = { token_source.setPosition(t.image); }
+ | ( t = { token_source.setPosition(t.image); })? (
+ ()? | ( | )?
+ )
+ | | | | ()? | | ()?
+ )
+ (
+ LOOKAHEAD(2)
+ t = { token_source.setPosition(t.image); }
+ )?
+ )?
+ anyExprList() // not interested
+}
+
+// upcoming lightweight mutation - see https://github.com/ClickHouse/ClickHouse/issues/19627
+void deleteStmt(): {} {
+ tableIdentifier(true) ( anyExprList())?
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/describe-table/
+void describeStmt(): {} {
+ ( | ) { token_source.table = "columns"; }
+ (LOOKAHEAD({ getToken(1).kind == TABLE }) )? (LOOKAHEAD(2) tableIdentifier(true) | anyExprList())
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/detach/
+void detachStmt(): { Token t; } {
+ (
+ LOOKAHEAD(2)
+ (
+ t = { token_source.setPosition(t.image); }
+ | ()? | |
+ )
+ (
+ LOOKAHEAD(2)
+ t = { token_source.setPosition(t.image); }
+ )?
+ )?
+ anyExprList() // not interested
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/drop/
+void dropStmt(): { Token t; } {
+ (
+ LOOKAHEAD(2)
+ (
+ t = { token_source.setPosition(t.image); }
+ | ()? | | | |
+ | ()? | | ()?
+ )
+ (
+ LOOKAHEAD(2)
+ t = { token_source.setPosition(t.image); }
+ )?
+ )?
+ anyExprList() // not interested
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/exists/
+void existsStmt(): {} { // not interested
+ anyExprList()
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/explain/
+void explainStmt(): {} { // not interested
+ anyExprList()
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/grant/
+void grantStmt(): {} { // not interested
+ anyExprList()
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/insert-into/
+void insertStmt(): {} {
+
+ (
+ LOOKAHEAD({ getToken(1).kind == FUNCTION }) functionExpr()
+ | (LOOKAHEAD(2) )? tableIdentifier(true)
+ )
+ (LOOKAHEAD(2) columnExprList() )?
+ dataClause()
+}
+
+void dataClause(): { Token t; } {
+ try {
+ LOOKAHEAD(2) anyIdentifier() (LOOKAHEAD(2) anyExprList())?
+ | LOOKAHEAD(2) t = { token_source.setPosition(t.image); }
+ columnExprList()
+ (
+ LOOKAHEAD(2)
+ ()?
+ columnExprList()
+ )*
+ | anyExprList() // not interested
+ } catch (ParseException e) {
+ // FIXME introduce a lexical state in next release with consideration of delimiter from the context
+ Token nextToken;
+ do {
+ nextToken = getNextToken();
+ } while(nextToken.kind != SEMICOLON && nextToken.kind != EOF);
+ }
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/kill/
+void killStmt(): {} { // not interested
+ anyExprList()
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/optimize/
+void optimizeStmt(): {} { // not interested
+ anyExprList()
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/rename/
+void renameStmt(): {} { // not interested
+ anyExprList()
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/revoke/
+void revokeStmt(): {} { // not interested
+ anyExprList()
+}
+
+// https://clickhouse.tech/docs/en/sql-reference/statements/select/
+void selectStmt(): {} {
+ // FIXME with (select 1), (select 2), 3 select *
+ (withClause())?
+