Skip to content

Commit 7fe50b2

Browse files
Ted-JiangTed-Jiang
authored
[KYLIN-5475] Use javaCC to parse self define DML (#2102)
* [KYLIN-5475] Use javaCC to parse self define DML
1 parent 1dbe113 commit 7fe50b2

File tree

3 files changed

+300
-0
lines changed

3 files changed

+300
-0
lines changed

‎src/query-common/src/main/codegen/javacc/org/apache/kylin/query/util/ManipulationSqlParser.jj

+152
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
options {
20+
IGNORE_CASE = true;
21+
STATIC = false;
22+
UNICODE_INPUT=true;
23+
}
24+
25+
PARSER_BEGIN(ManipulationSqlParser)
26+
package org.apache.kylin.query.util;
27+
28+
import java.io.StringReader;
29+
import java.util.ArrayList;
30+
import java.util.List;
31+
import java.io.Reader;
32+
import org.apache.kylin.query.util.DMLParserResult;
33+
34+
35+
public class ManipulationSqlParser {
36+
public static DMLParserResult evaluate(String src) throws ParseException {
37+
Reader reader = new StringReader(src);
38+
return new ManipulationSqlParser(reader).parse();
39+
}
40+
}
41+
42+
PARSER_END(ManipulationSqlParser)
43+
44+
// Operator type
45+
< DEFAULT >
46+
TOKEN :
47+
{
48+
< SHOW : "SHOW" > |
49+
< LOAD : "LOAD"> |
50+
< DELETE: "DELETE" > |
51+
< CANCEL: "CANCEL" >
52+
}
53+
54+
// Operator unit
55+
< DEFAULT >
56+
TOKEN :
57+
{
58+
< JOB : "JOB" > |
59+
< PARTITION : "PARTITION"> |
60+
< MODEL: "MODEL" >
61+
}
62+
63+
< DEFAULT >
64+
TOKEN: {
65+
// identifier
66+
< DATE:
67+
(["0"-"9"]){4}
68+
"-"
69+
(["0"-"9"]){2}
70+
"-"
71+
(["0"-"9"]){2}
72+
> |
73+
< DEFINENAME: (["a"-"z","A"-"Z","0"-"9"])
74+
(["a"-"z","A"-"Z","0"-"9","_"])*
75+
> |
76+
< JOBID: (["a"-"z","A"-"Z","0"-"9"])
77+
(["a"-"z","A"-"Z","0"-"9","-"])*
78+
>
79+
}
80+
81+
<DEFAULT> SKIP :
82+
{
83+
" "
84+
| "\t"
85+
| "\n"
86+
| "\r"
87+
| "\f"
88+
}
89+
90+
DMLParserResult parse():
91+
{
92+
DMLParserResult res = new DMLParserResult();
93+
List<String> argList = new ArrayList<String>();
94+
Token x, y, sta, end;
95+
}
96+
{
97+
// SHOW JOB job_id
98+
<SHOW> <JOB> x = <JOBID> <EOF>
99+
{
100+
argList.add(x.image);
101+
res.setOperator(DMLParserResult.OPERATOR.SHOW);
102+
res.setUnit(DMLParserResult.UNIT.JOB);
103+
res.setArg(argList);
104+
return res;
105+
} |
106+
// LOAD PARTITION poject_name.model_test (2023-01-01, 2023-02-10)
107+
LOOKAHEAD(2) <LOAD> <PARTITION> x = <DEFINENAME> "." y = <DEFINENAME> "(" sta = <DATE> "," end = <DATE> ")" <EOF>
108+
{
109+
argList.add(x.image);
110+
argList.add(y.image);
111+
argList.add(sta.image);
112+
argList.add(end.image);
113+
res.setOperator(DMLParserResult.OPERATOR.LOAD);
114+
res.setUnit(DMLParserResult.UNIT.PARTITION);
115+
res.setArg(argList);
116+
return res;
117+
} |
118+
// LOAD MODEL poject_name.model_test
119+
<LOAD> <MODEL> x = <DEFINENAME> "." y = <DEFINENAME> <EOF>
120+
{
121+
argList.add(x.image);
122+
argList.add(y.image);
123+
res.setOperator(DMLParserResult.OPERATOR.LOAD);
124+
res.setUnit(DMLParserResult.UNIT.MODEL);
125+
res.setArg(argList);
126+
return res;
127+
} |
128+
// DELETE MODEL poject_name.model_name
129+
<DELETE> <MODEL> x = <DEFINENAME> "." y = <DEFINENAME> <EOF>
130+
{
131+
argList.add(x.image);
132+
argList.add(y.image);
133+
res.setOperator(DMLParserResult.OPERATOR.DELETE);
134+
res.setUnit(DMLParserResult.UNIT.MODEL);
135+
res.setArg(argList);
136+
return res;
137+
} |
138+
// CANCEL JOB job_id
139+
<CANCEL> <JOB> x = <JOBID> <EOF>
140+
{
141+
argList.add(x.image);
142+
res.setOperator(DMLParserResult.OPERATOR.CANCEL);
143+
res.setUnit(DMLParserResult.UNIT.JOB);
144+
res.setArg(argList);
145+
return res;
146+
}
147+
148+
{
149+
return res;
150+
}
151+
152+
}

‎src/query-common/src/main/java/org/apache/kylin/query/util/DMLParserResult.java

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.kylin.query.util;
20+
21+
import java.util.List;
22+
23+
import lombok.Getter;
24+
import lombok.Setter;
25+
26+
@Getter
27+
@Setter
28+
public class DMLParserResult {
29+
// All DML consists with `OPERATOR` + `UNIT` + multi args
30+
public enum OPERATOR {
31+
SHOW, LOAD, DELETE, CANCEL;
32+
}
33+
34+
public enum UNIT {
35+
JOB, PARTITION, MODEL;
36+
}
37+
38+
OPERATOR operator;
39+
UNIT unit;
40+
List<String> arg;
41+
42+
// Notice operator not be Null
43+
public boolean isValid() {
44+
return operator != null;
45+
}
46+
}

‎src/query/src/test/java/org/apache/kylin/query/util/ManipulationSqlParserTest.java

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.kylin.query.util;
20+
21+
import org.junit.Assert;
22+
import org.junit.Test;
23+
24+
public class ManipulationSqlParserTest {
25+
26+
@Test
27+
public void showJob() throws ParseException {
28+
String sql = "SHOW JOB 70ba0c50-15b1-a40c-39a9-5f67f0f813c7-713ab396-1038-eecd-824f-3da66d531db6";
29+
DMLParserResult evaluate = ManipulationSqlParser.evaluate(sql);
30+
Assert.assertEquals(DMLParserResult.OPERATOR.SHOW, evaluate.operator);
31+
Assert.assertEquals(DMLParserResult.UNIT.JOB, evaluate.unit);
32+
Assert.assertEquals(1, evaluate.arg.size());
33+
Assert.assertEquals("70ba0c50-15b1-a40c-39a9-5f67f0f813c7-713ab396-1038-eecd-824f-3da66d531db6",
34+
evaluate.arg.get(0));
35+
}
36+
37+
@Test
38+
public void triggerJob() throws ParseException {
39+
// 1. trigger partial build job with partition col
40+
String sql = "LOAD PARTITION project_name.model_test (2023-01-01, 2023-02-10)";
41+
DMLParserResult evaluate = ManipulationSqlParser.evaluate(sql);
42+
43+
Assert.assertEquals(DMLParserResult.OPERATOR.LOAD, evaluate.operator);
44+
Assert.assertEquals(DMLParserResult.UNIT.PARTITION, evaluate.unit);
45+
Assert.assertEquals(4, evaluate.arg.size());
46+
Assert.assertEquals("project_name", evaluate.arg.get(0));
47+
Assert.assertEquals("model_test", evaluate.arg.get(1));
48+
Assert.assertEquals("2023-01-01", evaluate.arg.get(2));
49+
Assert.assertEquals("2023-02-10", evaluate.arg.get(3));
50+
51+
// 2. trigger full build job
52+
String sql2 = "LOAD MODEL project_name.model_test2";
53+
DMLParserResult evaluate2 = ManipulationSqlParser.evaluate(sql2);
54+
55+
Assert.assertEquals(evaluate2.operator, DMLParserResult.OPERATOR.LOAD);
56+
Assert.assertEquals(evaluate2.unit, DMLParserResult.UNIT.MODEL);
57+
Assert.assertEquals(2, evaluate2.arg.size());
58+
Assert.assertEquals("project_name", evaluate2.arg.get(0));
59+
Assert.assertEquals("model_test2", evaluate2.arg.get(1));
60+
61+
}
62+
63+
@Test
64+
public void cancelJob() throws ParseException {
65+
String sql = "CANCEL JOB 70ba0c50-15b1-a40c-39a9-5f67f0f813c7-713ab396-1038-eecd-824f-3da66d531db6";
66+
DMLParserResult evaluate = ManipulationSqlParser.evaluate(sql);
67+
68+
Assert.assertEquals(DMLParserResult.OPERATOR.CANCEL, evaluate.operator);
69+
Assert.assertEquals(DMLParserResult.UNIT.JOB, evaluate.unit);
70+
Assert.assertEquals(1, evaluate.arg.size());
71+
Assert.assertEquals("70ba0c50-15b1-a40c-39a9-5f67f0f813c7-713ab396-1038-eecd-824f-3da66d531db6",
72+
evaluate.arg.get(0));
73+
}
74+
75+
@Test
76+
public void deleteModel() throws ParseException {
77+
String sql = "DELETE MODEL project_name.model_test";
78+
DMLParserResult evaluate = ManipulationSqlParser.evaluate(sql);
79+
80+
Assert.assertEquals(DMLParserResult.OPERATOR.DELETE, evaluate.operator);
81+
Assert.assertEquals(DMLParserResult.UNIT.MODEL, evaluate.unit);
82+
Assert.assertEquals(2, evaluate.arg.size());
83+
Assert.assertEquals("project_name", evaluate.arg.get(0));
84+
Assert.assertEquals("model_test", evaluate.arg.get(1));
85+
}
86+
87+
@Test
88+
public void testError() {
89+
String sql = "DELETE MODELs model_test";
90+
Assert.assertThrows(ParseException.class, () -> ManipulationSqlParser.evaluate(sql));
91+
}
92+
93+
@Test
94+
public void validDMLParserResult() {
95+
DMLParserResult test = new DMLParserResult();
96+
test.unit = DMLParserResult.UNIT.JOB;
97+
Assert.assertFalse(test.isValid());
98+
99+
test.operator = DMLParserResult.OPERATOR.SHOW;
100+
Assert.assertTrue(test.isValid());
101+
}
102+
}

0 commit comments

Comments
 (0)