Skip to content

Commit 5418807

Browse files
airborne12airborne12
authored
[Fix](inverted index) fix wrong opt for count_on_index #41127 (#41153)
cherry pick from #41127
1 parent 764dcd7 commit 5418807

File tree

4 files changed

+150
-5
lines changed

4 files changed

+150
-5
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java

+9-2
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,10 @@ public List<Rule> buildRules() {
114114
.when(agg -> {
115115
Set<AggregateFunction> funcs = agg.getAggregateFunctions();
116116
return !funcs.isEmpty() && funcs.stream()
117-
.allMatch(f -> f instanceof Count && !f.isDistinct());
117+
.allMatch(f -> f instanceof Count && !f.isDistinct() && (((Count) f).isStar()
118+
|| f.children().isEmpty()
119+
|| (f.children().size() == 1 && f.child(0) instanceof Literal)
120+
|| f.child(0) instanceof Slot));
118121
})
119122
.thenApply(ctx -> {
120123
LogicalAggregate<LogicalFilter<LogicalOlapScan>> agg = ctx.root;
@@ -133,7 +136,11 @@ public List<Rule> buildRules() {
133136
.when(agg -> agg.getGroupByExpressions().isEmpty())
134137
.when(agg -> {
135138
Set<AggregateFunction> funcs = agg.getAggregateFunctions();
136-
return !funcs.isEmpty() && funcs.stream().allMatch(f -> f instanceof Count && !f.isDistinct());
139+
return !funcs.isEmpty() && funcs.stream()
140+
.allMatch(f -> f instanceof Count && !f.isDistinct() && (((Count) f).isStar()
141+
|| f.children().isEmpty()
142+
|| (f.children().size() == 1 && f.child(0) instanceof Literal)
143+
|| f.child(0) instanceof Slot));
137144
})
138145
.thenApply(ctx -> {
139146
LogicalAggregate<LogicalProject<LogicalFilter<LogicalOlapScan>>> agg = ctx.root;

‎regression-test/data/inverted_index_p0/count-on-index.json

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
{"a": "2024-03-25", "b": "ISZHfv2OQ4", "c": "c1", "d": "d1", "e": "e1"}
2+
{"a": "2023-02-20", "b": "0MgsB3vcIf", "c": "c3", "d": "d2", "e": "e2"}
3+
{"a": "2023-11-13", "b": "tczLqYSRhY", "c": "c1", "d": "d2", "e": "e1"}
4+
{"a": "2023-08-13", "b": "f86oV0P4s8", "c": "c2", "d": "d2", "e": "e2"}
5+
{"a": "2023-07-13", "b": "YHD5LihHpK", "c": "c3", "d": "d2", "e": "e2"}
6+
{"a": "2024-07-26", "b": "iSJxD3yKvH", "c": "c1", "d": "d2", "e": "e1"}
7+
{"a": "2023-12-25", "b": "aNiLbwF1vg", "c": "c1", "d": "d2", "e": "e2"}
8+
{"a": "2024-04-04", "b": "DGih88jW0H", "c": "c3", "d": "d3", "e": "e2"}
9+
{"a": "2024-03-30", "b": "kxqqsrD1RH", "c": "c2", "d": "d3", "e": "e1"}
10+
{"a": "2023-10-01", "b": "eySjxEnJvW", "c": "c2", "d": "d3", "e": "e1"}
11+
{"a": "2023-09-19", "b": "lHAJZpcky5", "c": "c2", "d": "d2", "e": "e1"}
12+
{"a": "2023-11-25", "b": "PB4l4NsNAn", "c": "c1", "d": "d3", "e": "e2"}
13+
{"a": "2023-05-10", "b": "TP5M3xQDCj", "c": "c1", "d": "d2", "e": "e1"}
14+
{"a": "2024-08-28", "b": "b1D8xLX4DN", "c": "c3", "d": "d2", "e": "e1"}
15+
{"a": "2023-05-31", "b": "9gTsOoFITb", "c": "c2", "d": "d3", "e": "e2"}
16+
{"a": "2023-01-14", "b": "rsv96RalgR", "c": "c3", "d": "d2", "e": "e2"}
17+
{"a": "2024-08-31", "b": "Uz67DnurlH", "c": "c3", "d": "d2", "e": "e2"}
18+
{"a": "2023-04-05", "b": "kPlM5F56kj", "c": "c2", "d": "d3", "e": "e2"}
19+
{"a": "2023-09-18", "b": "wSGVCB6s3I", "c": "c3", "d": "d2", "e": "e2"}
20+
{"a": "2023-01-28", "b": "2ZpPp1y5G5", "c": "c3", "d": "d2", "e": "e2"}
21+
{"a": "2023-03-27", "b": "VwdMxBUnrc", "c": "c3", "d": "d1", "e": "e1"}
22+
{"a": "2024-03-17", "b": "QPV71OyuZ2", "c": "c1", "d": "d2", "e": "e1"}
23+
{"a": "2023-08-10", "b": "pyhphs1Mj4", "c": "c1", "d": "d1", "e": "e1"}
24+
{"a": "2024-01-26", "b": "wxRO18q0EY", "c": "c2", "d": "d3", "e": "e1"}
25+
{"a": "2023-11-04", "b": "vfVK2TsjTl", "c": "c2", "d": "d1", "e": "e2"}
26+
{"a": "2023-04-27", "b": "eS6vCuQAXP", "c": "c3", "d": "d2", "e": "e1"}
27+
{"a": "2024-02-25", "b": "6dx8DMPK9f", "c": "c2", "d": "d3", "e": "e2"}
28+
{"a": "2023-04-16", "b": "3aJhtwXa7E", "c": "c2", "d": "d3", "e": "e1"}
29+
{"a": "2024-07-08", "b": "Ue9xroXnHI", "c": "c3", "d": "d3", "e": "e2"}
30+
{"a": "2023-08-19", "b": "gVPYCdN2eY", "c": "c2", "d": "d3", "e": "e2"}
31+
{"a": "2024-02-24", "b": "qAaaKQpvd3", "c": "c2", "d": "d2", "e": "e1"}
32+
{"a": "2024-01-10", "b": "XsNcGPnvvC", "c": "c2", "d": "d1", "e": "e2"}
33+
{"a": "2023-03-04", "b": "BD7CEdzIhP", "c": "c2", "d": "d1", "e": "e1"}
34+
{"a": "2023-03-18", "b": "sXESaouuHE", "c": "c2", "d": "d3", "e": "e1"}
35+
{"a": "2023-06-28", "b": "DD3RE2pufi", "c": "c2", "d": "d3", "e": "e2"}
36+
{"a": "2024-04-07", "b": "RdEFKIz8QW", "c": "c2", "d": "d1", "e": "e2"}
37+
{"a": "2024-05-10", "b": "u3OhzAL2LH", "c": "c3", "d": "d1", "e": "e1"}
38+
{"a": "2024-07-20", "b": "U0n5EVKjPm", "c": "c1", "d": "d1", "e": "e1"}
39+
{"a": "2024-08-07", "b": "TXypE2ItVh", "c": "c3", "d": "d2", "e": "e1"}
40+
{"a": "2024-08-08", "b": "8g3hPyCB2B", "c": "c3", "d": "d2", "e": "e1"}
41+
{"a": "2023-01-05", "b": "aT6WlbObnZ", "c": "c3", "d": "d2", "e": "e2"}
42+
{"a": "2023-06-05", "b": "mVqMi8Rzfi", "c": "c1", "d": "d1", "e": "e2"}
43+
{"a": "2024-02-12", "b": "HxpmQ0draG", "c": "c1", "d": "d1", "e": "e1"}
44+
{"a": "2024-05-20", "b": "R5a7gA61KY", "c": "c1", "d": "d2", "e": "e1"}
45+
{"a": "2023-04-19", "b": "QacPa5V0Fj", "c": "c3", "d": "d3", "e": "e1"}
46+
{"a": "2023-07-29", "b": "dwT8GxkWDA", "c": "c1", "d": "d1", "e": "e1"}
47+
{"a": "2024-06-29", "b": "UCRkZWVEhK", "c": "c3", "d": "d1", "e": "e2"}
48+
{"a": "2023-02-22", "b": "yMSAdFkaq9", "c": "c3", "d": "d1", "e": "e1"}
49+
{"a": "2024-04-18", "b": "6Aa4VUyj7b", "c": "c2", "d": "d1", "e": "e2"}
50+
{"a": "2023-09-20", "b": "xet5tOBGLy", "c": "c1", "d": "d1", "e": "e2"}
51+
{"a": "2024-07-09", "b": "kyE5wM71uC", "c": "c1", "d": "d1", "e": "e2"}
52+
{"a": "2024-03-05", "b": "J9UtyRClVj", "c": "c2", "d": "d2", "e": "e2"}
53+
{"a": "2024-08-02", "b": "BnNajVStTq", "c": "c3", "d": "d2", "e": "e2"}
54+
{"a": "2024-03-20", "b": "YFyjBh6JeE", "c": "c1", "d": "d3", "e": "e1"}
55+
{"a": "2024-01-25", "b": "kF462Dpave", "c": "c1", "d": "d1", "e": "e2"}
56+
{"a": "2023-11-20", "b": "uUsipxur13", "c": "c1", "d": "d3", "e": "e2"}
57+
{"a": "2024-08-06", "b": "g4i1sEGImS", "c": "c2", "d": "d3", "e": "e1"}
58+
{"a": "2024-02-09", "b": "NUz4tjvKt5", "c": "c3", "d": "d1", "e": "e2"}
59+
{"a": "2024-04-20", "b": "p72Gn18tWd", "c": "c3", "d": "d3", "e": "e2"}
60+
{"a": "2024-01-22", "b": "3DriXIqvSg", "c": "c2", "d": "d3", "e": "e2"}
61+
{"a": "2024-07-26", "b": "rorCsbghiO", "c": "c1", "d": "d1", "e": "e2"}
62+
{"a": "2023-10-26", "b": "XAWPiEQVmE", "c": "c3", "d": "d1", "e": "e1"}
63+
{"a": "2023-10-30", "b": "L3FWcbrzen", "c": "c1", "d": "d2", "e": "e2"}
64+
{"a": "2024-06-25", "b": "Lj6SZ26GJN", "c": "c3", "d": "d3", "e": "e1"}
65+
{"a": "2023-07-20", "b": "U6nYzFhfwM", "c": "c2", "d": "d2", "e": "e2"}
66+
{"a": "2023-06-26", "b": "J7jWtTmtZT", "c": "c3", "d": "d2", "e": "e2"}
67+
{"a": "2024-05-27", "b": "hDWYIRDHV4", "c": "c2", "d": "d2", "e": "e2"}
68+
{"a": "2023-05-23", "b": "kvjedf4zF8", "c": "c3", "d": "d3", "e": "e1"}
69+
{"a": "2023-02-06", "b": "RsPN2cGb2L", "c": "c2", "d": "d3", "e": "e1"}
70+
{"a": "2024-06-04", "b": "belvas0y6p", "c": "c3", "d": "d1", "e": "e2"}
71+
{"a": "2024-06-17", "b": "J6vYAcFuGZ", "c": "c3", "d": "d2", "e": "e2"}
72+
{"a": "2024-04-18", "b": "qHuHh0Y29i", "c": "c1", "d": "d2", "e": "e1"}
73+
{"a": "2023-08-20", "b": "1GS5UtXMdz", "c": "c2", "d": "d3", "e": "e1"}
74+
{"a": "2024-01-23", "b": "gnjJ4TZ6A6", "c": "c1", "d": "d1", "e": "e1"}
75+
{"a": "2023-02-10", "b": "LX6ddQvIX2", "c": "c2", "d": "d3", "e": "e1"}
76+
{"a": "2024-03-02", "b": "MkImkgiAfm", "c": "c3", "d": "d1", "e": "e2"}
77+
{"a": "2023-11-30", "b": "tGmu0DD8W0", "c": "c3", "d": "d3", "e": "e1"}
78+
{"a": "2023-02-01", "b": "NgzjCOPAku", "c": "c1", "d": "d3", "e": "e2"}
79+
{"a": "2023-01-03", "b": "5mWMiuYwbi", "c": "c3", "d": "d1", "e": "e2"}
80+
{"a": "2023-02-12", "b": "9324TZqLjh", "c": "c3", "d": "d2", "e": "e2"}
81+
{"a": "2023-06-24", "b": "7fAYU4XSma", "c": "c1", "d": "d3", "e": "e1"}
82+
{"a": "2024-08-28", "b": "iNvBMy8AB8", "c": "c1", "d": "d1", "e": "e1"}
83+
{"a": "2024-06-08", "b": "nwJjFx21my", "c": "c3", "d": "d1", "e": "e1"}
84+
{"a": "2023-03-15", "b": "Oonv8eGNIF", "c": "c3", "d": "d2", "e": "e1"}
85+
{"a": "2024-04-22", "b": "6UYDEqQxxf", "c": "c2", "d": "d2", "e": "e2"}
86+
{"a": "2024-05-08", "b": "aVPqpTufJf", "c": "c1", "d": "d2", "e": "e1"}
87+
{"a": "2024-08-02", "b": "z1baLbjnTj", "c": "c3", "d": "d3", "e": "e2"}
88+
{"a": "2023-07-05", "b": "vMLMALySMJ", "c": "c3", "d": "d3", "e": "e1"}
89+
{"a": "2023-08-30", "b": "s7VZKlOG27", "c": "c3", "d": "d3", "e": "e2"}
90+
{"a": "2023-03-06", "b": "gWAuqLvHpJ", "c": "c2", "d": "d1", "e": "e2"}
91+
{"a": "2024-02-24", "b": "2IQL6pazn3", "c": "c1", "d": "d2", "e": "e1"}
92+
{"a": "2024-08-15", "b": "4YbxyhwaRF", "c": "c3", "d": "d2", "e": "e1"}
93+
{"a": "2023-02-28", "b": "wAOOLl8Kqj", "c": "c1", "d": "d2", "e": "e2"}
94+
{"a": "2024-05-31", "b": "FTQKB8rURb", "c": "c3", "d": "d2", "e": "e2"}
95+
{"a": "2023-02-20", "b": "knNNhnMXLN", "c": "c1", "d": "d3", "e": "e2"}
96+
{"a": "2024-07-01", "b": "cxA5xpl6NM", "c": "c2", "d": "d1", "e": "e2"}
97+
{"a": "2023-05-10", "b": "5FnmdQtOA0", "c": "c1", "d": "d2", "e": "e1"}
98+
{"a": "2024-08-21", "b": "ldggIaWfYF", "c": "c3", "d": "d2", "e": "e2"}
99+
{"a": "2024-01-05", "b": "7lwOfCQs5o", "c": "c2", "d": "d3", "e": "e1"}
100+
{"a": "2024-08-21", "b": "2zvnCMAkZG", "c": "c1", "d": "d1", "e": "e2"}

‎regression-test/data/inverted_index_p0/test_count_on_index.out

+3
Original file line numberDiff line numberDiff line change
@@ -74,3 +74,6 @@
7474
-- !sql --
7575
0
7676

77+
-- !sql_bad --
78+
0 1
79+

‎regression-test/suites/inverted_index_p0/test_count_on_index.groovy

+38-3
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ suite("test_count_on_index_httplogs", "p0") {
9393
"""
9494
}
9595

96-
def load_httplogs_data = {table_name, label, read_flag, format_flag, file_name, ignore_failure=false,
96+
def stream_load_data = {table_name, label, read_flag, format_flag, file_name, ignore_failure=false,
9797
expected_succ_rows = -1, load_to_single_tablet = 'true' ->
9898

9999
// load the json data
@@ -137,8 +137,8 @@ suite("test_count_on_index_httplogs", "p0") {
137137
create_httplogs_dup_table.call(testTable_dup)
138138
create_httplogs_unique_table.call(testTable_unique)
139139

140-
load_httplogs_data.call(testTable_dup, 'test_httplogs_load_count_on_index', 'true', 'json', 'documents-1000.json')
141-
load_httplogs_data.call(testTable_unique, 'test_httplogs_load_count_on_index', 'true', 'json', 'documents-1000.json')
140+
stream_load_data.call(testTable_dup, 'test_httplogs_load_count_on_index', 'true', 'json', 'documents-1000.json')
141+
stream_load_data.call(testTable_unique, 'test_httplogs_load_count_on_index', 'true', 'json', 'documents-1000.json')
142142

143143
sql "sync"
144144
sql """set experimental_enable_nereids_planner=true;"""
@@ -272,6 +272,41 @@ suite("test_count_on_index_httplogs", "p0") {
272272
// case4: test compound query when inverted_index_query disable
273273
qt_sql "SELECT COUNT() from ${testTable_dup} where request = 'images' or (size = 0 and status > 400)"
274274
qt_sql "SELECT /*+SET_VAR(enable_inverted_index_query=false) */ COUNT() from ${testTable_dup} where request = 'images' or (size = 0 and status > 400)"
275+
276+
// case5: test complex count to testify bad case
277+
def tableName5 = 'test_count_on_index_bad_case'
278+
sql "DROP TABLE IF EXISTS ${tableName5}"
279+
sql """
280+
CREATE TABLE `${tableName5}` (
281+
`a` DATE NOT NULL COMMENT '',
282+
`b` VARCHAR(4096) NULL COMMENT '',
283+
`c` VARCHAR(4096) NULL COMMENT '',
284+
`d` VARCHAR(4096) NULL COMMENT '',
285+
`e` VARCHAR(4096) NULL COMMENT '',
286+
INDEX idx_a(`a`) USING INVERTED COMMENT '',
287+
INDEX idx_e(`e`) USING INVERTED COMMENT ''
288+
) ENGINE=OLAP
289+
UNIQUE KEY(`a`, `b`)
290+
COMMENT ''
291+
DISTRIBUTED BY HASH(`a`) BUCKETS 3
292+
PROPERTIES (
293+
"replication_allocation" = "tag.location.default: 1"
294+
);
295+
"""
296+
stream_load_data.call(tableName5, 'test_count_on_index_bad_case', 'true', 'json', 'count-on-index.json')
297+
def bad_sql = """
298+
SELECT
299+
COUNT(CASE WHEN c IN ('c1', 'c2', 'c3') AND d = 'd1' THEN b END) AS num1,
300+
COUNT(CASE WHEN e = 'e1' AND c IN ('c1', 'c2', 'c3') THEN b END) AS num2
301+
FROM ${tableName5}
302+
WHERE a = '2024-07-26'
303+
AND e = 'e1';
304+
"""
305+
explain {
306+
sql("${bad_sql}")
307+
contains "pushAggOp=NONE"
308+
}
309+
qt_sql_bad "${bad_sql}"
275310
} finally {
276311
//try_sql("DROP TABLE IF EXISTS ${testTable}")
277312
}

0 commit comments

Comments
 (0)