diff --git a/enginetest/plangen/cmd/plangen/main.go b/enginetest/plangen/cmd/plangen/main.go index 1acc3fea25..615a8e32f8 100644 --- a/enginetest/plangen/cmd/plangen/main.go +++ b/enginetest/plangen/cmd/plangen/main.go @@ -257,7 +257,7 @@ func generatePlansForScriptSuite(spec PlanSpec, w *bytes.Buffer) error { w.WriteString(fmt.Sprintf("\t\tName: \"%s\",\n", tt.Name)) w.WriteString("\t\tSetUpScript: []string{\n") for _, setupQuery := range tt.SetUpScript { - w.WriteString(fmt.Sprintf("\t\t\t\"%s\",\n", setupQuery)) + w.WriteString(fmt.Sprintf("\t\t\t`%s`,\n", setupQuery)) } w.WriteString("\t\t},\n") w.WriteString("\t\tAssertions: []ScriptTestAssertion{\n") @@ -266,7 +266,7 @@ func generatePlansForScriptSuite(spec PlanSpec, w *bytes.Buffer) error { if assertion.Skip { w.WriteString("\t\t\t\tSkip: true,\n") } - w.WriteString(fmt.Sprintf("\t\t\t\tQuery: \"%s\",\n", assertion.Query)) + w.WriteString(fmt.Sprintf("\t\t\t\tQuery: `%s`,\n", assertion.Query)) w.WriteString(fmt.Sprintf("\t\t\t\tExpected: []sql.Row{\n")) for _, expRow := range assertion.Expected { w.WriteString(fmt.Sprintf("\t\t\t\t\t%#v,\n", expRow)) diff --git a/enginetest/queries/query_plan_script_tests.go b/enginetest/queries/query_plan_script_tests.go index 09f7e27fb3..549fb3bd07 100644 --- a/enginetest/queries/query_plan_script_tests.go +++ b/enginetest/queries/query_plan_script_tests.go @@ -24,18 +24,18 @@ var QueryPlanScriptTests = []ScriptTest{ { Name: "test merge join optimization (removing sort node over indexed tables) does not break ordering", SetUpScript: []string{ - "create table t1 (i int primary key);", - "create table t2 (j int primary key);", - "insert into t1 values (1), (2), (3);", - "insert into t2 values (2), (3), (4);", - "create table t3 (i int, j int, primary key (i, j));", - "create table t4 (x int, y int, primary key (x, y));", - "insert into t3 values (1, 1), (1, 2), (2, 2), (3, 3);", - "insert into t4 values (2, 2), (3, 3), (4, 4);", + `create table t1 (i int primary key);`, + `create table t2 (j int primary key);`, + `insert into t1 values (1), (2), (3);`, + `insert into t2 values (2), (3), (4);`, + `create table t3 (i int, j int, primary key (i, j));`, + `create table t4 (x int, y int, primary key (x, y));`, + `insert into t3 values (1, 1), (1, 2), (2, 2), (3, 3);`, + `insert into t4 values (2, 2), (3, 3), (4, 4);`, }, Assertions: []ScriptTestAssertion{ { - Query: "select /*+ MERGE_JOIN(t1, t2) */ * from t1 join t2 on t1.i = t2.j order by t1.i;", + Query: `select /*+ MERGE_JOIN(t1, t2) */ * from t1 join t2 on t1.i = t2.j order by t1.i;`, Expected: []sql.Row{ sql.Row{2, 2}, sql.Row{3, 3}, @@ -63,7 +63,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t1, t2) */ * from t1 join t2 on t1.i = t2.j order by t2.j;", + Query: `select /*+ MERGE_JOIN(t1, t2) */ * from t1 join t2 on t1.i = t2.j order by t2.j;`, Expected: []sql.Row{ sql.Row{2, 2}, sql.Row{3, 3}, @@ -91,7 +91,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t1, t2) */ * from t1 join t2 on t1.i = t2.j order by t1.i desc;", + Query: `select /*+ MERGE_JOIN(t1, t2) */ * from t1 join t2 on t1.i = t2.j order by t1.i desc;`, Expected: []sql.Row{ sql.Row{3, 3}, sql.Row{2, 2}, @@ -121,7 +121,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t1, t2) */ * from t1 join t2 on t1.i = t2.j order by t2.j desc;", + Query: `select /*+ MERGE_JOIN(t1, t2) */ * from t1 join t2 on t1.i = t2.j order by t2.j desc;`, Expected: []sql.Row{ sql.Row{3, 3}, sql.Row{2, 2}, @@ -151,7 +151,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t1, t2) */ * from t1 where ((i in (select j from t2 where j > 2))) order by i desc;", + Query: `select /*+ MERGE_JOIN(t1, t2) */ * from t1 where ((i in (select j from t2 where j > 2))) order by i desc;`, Expected: []sql.Row{ sql.Row{3}, }, @@ -187,7 +187,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i;`, Expected: []sql.Row{ sql.Row{2, 2, 2, 2}, sql.Row{3, 3, 3, 3}, @@ -215,7 +215,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i desc;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i desc;`, Expected: []sql.Row{ sql.Row{3, 3, 3, 3}, sql.Row{2, 2, 2, 2}, @@ -245,7 +245,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t4.x;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t4.x;`, Expected: []sql.Row{ sql.Row{2, 2, 2, 2}, sql.Row{3, 3, 3, 3}, @@ -273,7 +273,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t4.x desc;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t4.x desc;`, Expected: []sql.Row{ sql.Row{3, 3, 3, 3}, sql.Row{2, 2, 2, 2}, @@ -303,7 +303,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i, t3.j;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i, t3.j;`, Expected: []sql.Row{ sql.Row{2, 2, 2, 2}, sql.Row{3, 3, 3, 3}, @@ -331,7 +331,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i desc, t3.j desc;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i desc, t3.j desc;`, Expected: []sql.Row{ sql.Row{3, 3, 3, 3}, sql.Row{2, 2, 2, 2}, @@ -361,7 +361,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t4.x, t4.y;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t4.x, t4.y;`, Expected: []sql.Row{ sql.Row{2, 2, 2, 2}, sql.Row{3, 3, 3, 3}, @@ -389,7 +389,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t4.x desc, t4.y desc;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t4.x desc, t4.y desc;`, Expected: []sql.Row{ sql.Row{3, 3, 3, 3}, sql.Row{2, 2, 2, 2}, @@ -419,7 +419,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i, t4.x;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i, t4.x;`, Expected: []sql.Row{ sql.Row{2, 2, 2, 2}, sql.Row{3, 3, 3, 3}, @@ -448,7 +448,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i, t4.x desc;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i, t4.x desc;`, Expected: []sql.Row{ sql.Row{2, 2, 2, 2}, sql.Row{3, 3, 3, 3}, @@ -477,7 +477,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i, t3.j, t4.x;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i, t3.j, t4.x;`, Expected: []sql.Row{ sql.Row{2, 2, 2, 2}, sql.Row{3, 3, 3, 3}, @@ -506,7 +506,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i, t3.j, t4.x, t4.y;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i, t3.j, t4.x, t4.y;`, Expected: []sql.Row{ sql.Row{2, 2, 2, 2}, sql.Row{3, 3, 3, 3}, @@ -535,7 +535,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.j;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.j;`, Expected: []sql.Row{ sql.Row{2, 2, 2, 2}, sql.Row{3, 3, 3, 3}, @@ -564,7 +564,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t4.y;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t4.y;`, Expected: []sql.Row{ sql.Row{2, 2, 2, 2}, sql.Row{3, 3, 3, 3}, @@ -593,7 +593,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i, t3.j desc;", + Query: `select /*+ MERGE_JOIN(t3, t4) */ * from t3 join t4 on t3.i = t4.x order by t3.i, t3.j desc;`, Expected: []sql.Row{ sql.Row{2, 2, 2, 2}, sql.Row{3, 3, 3, 3}, @@ -622,7 +622,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select * from t1 join t2 order by t1.i;", + Query: `select * from t1 join t2 order by t1.i;`, Expected: []sql.Row{ sql.Row{1, 2}, sql.Row{1, 3}, @@ -650,7 +650,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select * from t1 join t2 order by t2.j;", + Query: `select * from t1 join t2 order by t2.j;`, Expected: []sql.Row{ sql.Row{1, 2}, sql.Row{2, 2}, @@ -680,7 +680,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select * from t1 join t2 order by t1.i desc;", + Query: `select * from t1 join t2 order by t1.i desc;`, Expected: []sql.Row{ sql.Row{3, 2}, sql.Row{3, 3}, @@ -709,7 +709,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select * from t1 join t2 where t1.i > 1 and t1.i < 3 and t2.j > 2 and t2.j < 4 order by t1.i;", + Query: `select * from t1 join t2 where t1.i > 1 and t1.i < 3 and t2.j > 2 and t2.j < 4 order by t1.i;`, Expected: []sql.Row{ sql.Row{2, 3}, }, @@ -733,7 +733,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select * from t3 join t4 where t3.i = 1 and t4.x = 3 order by t3.i;", + Query: `select * from t3 join t4 where t3.i = 1 and t4.x = 3 order by t3.i;`, Expected: []sql.Row{ sql.Row{1, 1, 3, 3}, sql.Row{1, 2, 3, 3}, @@ -758,7 +758,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select * from t3 join t4 where t3.j = 2 and t4.x = 3 order by t3.i desc;", + Query: `select * from t3 join t4 where t3.j = 2 and t4.x = 3 order by t3.i desc;`, Expected: []sql.Row{ sql.Row{2, 2, 3, 3}, sql.Row{1, 2, 3, 3}, @@ -788,7 +788,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select * from t1 inner join t2 where t1.i < t2.j order by t1.i;", + Query: `select * from t1 inner join t2 where t1.i < t2.j order by t1.i;`, Expected: []sql.Row{ sql.Row{1, 2}, sql.Row{1, 3}, @@ -816,7 +816,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select * from t1 inner join t2 where t1.i < t2.j order by t2.j desc;", + Query: `select * from t1 inner join t2 where t1.i < t2.j order by t2.j desc;`, Expected: []sql.Row{ sql.Row{1, 4}, sql.Row{2, 4}, @@ -847,7 +847,7 @@ var QueryPlanScriptTests = []ScriptTest{ "", }, { - Query: "select * from t3 inner join t4 where i != x order by t3.i, t3.j;", + Query: `select * from t3 inner join t4 where i != x order by t3.i, t3.j;`, Expected: []sql.Row{ sql.Row{1, 1, 2, 2}, sql.Row{1, 1, 3, 3}, @@ -881,4 +881,138 @@ var QueryPlanScriptTests = []ScriptTest{ }, }, }, + { + Name: "Recursive CTE inside NOT EXISTS clause with correlated column filter", + SetUpScript: []string{ + `CREATE TABLE issues (id INT PRIMARY KEY, title TEXT, status TEXT);`, + `CREATE TABLE dependencies (issue_id INT, depends_on_id INT, type TEXT);`, + `INSERT INTO issues (id, title, status) VALUES + (1, 'Login API', 'open'), + (2, 'Auth Library', 'in_progress'), + (3, 'User Profile', 'open'), + (4, 'Profile UI', 'open'), + (5, 'Settings Page', 'open'), + (6, 'Marketing Page', 'open'), + (7, 'Old Feature', 'closed');`, + `INSERT INTO dependencies (issue_id, depends_on_id, type) VALUES + (3, 1, 'blocks'), + (3, 2, 'blocks'), + (4, 3, 'parent-child'), + (5, 4, 'parent-child');`, + }, + Assertions: []ScriptTestAssertion{ + { + Query: `WITH RECURSIVE + blocked_directly AS ( + SELECT DISTINCT d.issue_id + FROM dependencies d + JOIN issues blocker ON d.depends_on_id = blocker.id + WHERE d.type = 'blocks' + AND blocker.status IN ('open', 'in_progress', 'blocked', 'deferred', 'hooked') + ), + blocked_transitively AS ( + SELECT issue_id, 0 as depth + FROM blocked_directly + UNION ALL + SELECT d.issue_id, bt.depth + 1 + FROM blocked_transitively bt + JOIN dependencies d ON d.depends_on_id = bt.issue_id + WHERE d.type = 'parent-child' + AND bt.depth < 50 + ) + SELECT i.* + FROM issues i + WHERE i.status = 'open' + AND NOT EXISTS ( + SELECT 1 FROM blocked_transitively WHERE issue_id = i.id + );`, + Expected: []sql.Row{ + sql.Row{1, "Login API", "open"}, + sql.Row{6, "Marketing Page", "open"}, + }, + ExpectedPlan: "AntiJoinIncludingNulls\n" + + " ├─ Eq\n" + + " │ ├─ blocked_transitively.issue_id:3\n" + + " │ └─ i.id:0!null\n" + + " ├─ Filter\n" + + " │ ├─ Eq\n" + + " │ │ ├─ i.status:2\n" + + " │ │ └─ open (longtext)\n" + + " │ └─ TableAlias(i)\n" + + " │ └─ ProcessTable\n" + + " │ └─ Table\n" + + " │ ├─ name: issues\n" + + " │ └─ columns: [id title status]\n" + + " └─ CachedResults\n" + + " └─ SubqueryAlias\n" + + " ├─ name: blocked_transitively\n" + + " ├─ outerVisibility: true\n" + + " ├─ isLateral: false\n" + + " ├─ cacheable: true\n" + + " ├─ colSet: (20,21)\n" + + " ├─ tableId: 7\n" + + " └─ RecursiveCTE\n" + + " └─ Union all\n" + + " ├─ Project\n" + + " │ ├─ columns: [blocked_directly.issue_id:0, 0 (tinyint)->depth:9]\n" + + " │ └─ SubqueryAlias\n" + + " │ ├─ name: blocked_directly\n" + + " │ ├─ outerVisibility: false\n" + + " │ ├─ isLateral: false\n" + + " │ ├─ cacheable: true\n" + + " │ ├─ colSet: (8)\n" + + " │ ├─ tableId: 4\n" + + " │ └─ Distinct\n" + + " │ └─ Project\n" + + " │ ├─ columns: [d.issue_id:0]\n" + + " │ └─ LookupJoin\n" + + " │ ├─ Filter\n" + + " │ │ ├─ Eq\n" + + " │ │ │ ├─ d.type:2\n" + + " │ │ │ └─ blocks (longtext)\n" + + " │ │ └─ TableAlias(d)\n" + + " │ │ └─ Table\n" + + " │ │ ├─ name: dependencies\n" + + " │ │ ├─ columns: [issue_id depends_on_id type]\n" + + " │ │ ├─ colSet: (1-3)\n" + + " │ │ └─ tableId: 1\n" + + " │ └─ Filter\n" + + " │ ├─ HashIn\n" + + " │ │ ├─ blocker.status:1\n" + + " │ │ └─ TUPLE(open (longtext), in_progress (longtext), blocked (longtext), deferred (longtext), hooked (longtext))\n" + + " │ └─ TableAlias(blocker)\n" + + " │ └─ IndexedTableAccess(issues)\n" + + " │ ├─ index: [issues.id]\n" + + " │ ├─ keys: [d.depends_on_id:1]\n" + + " │ ├─ colSet: (4-6)\n" + + " │ ├─ tableId: 2\n" + + " │ └─ Table\n" + + " │ ├─ name: issues\n" + + " │ └─ columns: [id status]\n" + + " └─ Project\n" + + " ├─ columns: [d.issue_id:0, (bt.depth:4!null + 1 (tinyint))->bt.depth + 1:0]\n" + + " └─ InnerJoin\n" + + " ├─ AND\n" + + " │ ├─ LessThan\n" + + " │ │ ├─ bt.depth:7!null\n" + + " │ │ └─ 50 (bigint)\n" + + " │ └─ Eq\n" + + " │ ├─ d.depends_on_id:4\n" + + " │ └─ bt.issue_id:6\n" + + " ├─ Filter\n" + + " │ ├─ Eq\n" + + " │ │ ├─ d.type:2\n" + + " │ │ └─ parent-child (longtext)\n" + + " │ └─ TableAlias(d)\n" + + " │ └─ Table\n" + + " │ ├─ name: dependencies\n" + + " │ ├─ columns: [issue_id depends_on_id type]\n" + + " │ ├─ colSet: (14-16)\n" + + " │ └─ tableId: 8\n" + + " └─ TableAlias(bt)\n" + + " └─ RecursiveTable(blocked_transitively)\n" + + "", + }, + }, + }, } diff --git a/enginetest/queries/query_plans.go b/enginetest/queries/query_plans.go index 475965fae6..cebb3d820f 100644 --- a/enginetest/queries/query_plans.go +++ b/enginetest/queries/query_plans.go @@ -22307,54 +22307,50 @@ WHERE keyless.c0 IN ( " │ ├─ alias-string: with recursive cte (depth, i, j) as (select 0, T1.c0, T1.c1 from keyless as T1 where T1.c0 = 0 union all select cte.depth + 1, cte.i, T2.c1 + 1 from cte, keyless as T2 where cte.depth = T2.c0) select U0.c0 from keyless as U0, cte where cte.j = keyless.c0\n" + " │ └─ Project\n" + " │ ├─ columns: [u0.c0:5]\n" + - " │ └─ CrossHashJoin\n" + + " │ └─ InnerJoin\n" + + " │ ├─ Eq\n" + + " │ │ ├─ cte.j:4\n" + + " │ │ └─ keyless.c0:0\n" + " │ ├─ SubqueryAlias\n" + " │ │ ├─ name: cte\n" + " │ │ ├─ outerVisibility: true\n" + - " │ │ ├─ isLateral: true\n" + - " │ │ ├─ cacheable: false\n" + + " │ │ ├─ isLateral: false\n" + + " │ │ ├─ cacheable: true\n" + " │ │ ├─ colSet: (16-18)\n" + " │ │ ├─ tableId: 5\n" + - " │ │ └─ Filter\n" + - " │ │ ├─ Eq\n" + - " │ │ │ ├─ cte.j:4\n" + - " │ │ │ └─ keyless.c0:0\n" + - " │ │ └─ RecursiveCTE\n" + - " │ │ └─ Union all\n" + - " │ │ ├─ Project\n" + - " │ │ │ ├─ columns: [0 (tinyint), t1.c0:2, t1.c1:3]\n" + - " │ │ │ └─ Filter\n" + - " │ │ │ ├─ Eq\n" + - " │ │ │ │ ├─ t1.c0:2\n" + - " │ │ │ │ └─ 0 (bigint)\n" + - " │ │ │ └─ TableAlias(t1)\n" + - " │ │ │ └─ Table\n" + - " │ │ │ ├─ name: keyless\n" + - " │ │ │ ├─ columns: [c0 c1]\n" + - " │ │ │ ├─ colSet: (3,4)\n" + - " │ │ │ └─ tableId: 2\n" + - " │ │ └─ Project\n" + - " │ │ ├─ columns: [(cte.depth:4!null + 1 (tinyint))->cte.depth + 1:0, cte.i:5, (t2.c1:3 + 1 (tinyint))->T2.c1 + 1:0]\n" + - " │ │ └─ InnerJoin\n" + - " │ │ ├─ Eq\n" + - " │ │ │ ├─ cte.depth:4!null\n" + - " │ │ │ └─ t2.c0:2\n" + - " │ │ ├─ TableAlias(t2)\n" + - " │ │ │ └─ Table\n" + - " │ │ │ ├─ name: keyless\n" + - " │ │ │ ├─ columns: [c0 c1]\n" + - " │ │ │ ├─ colSet: (12,13)\n" + - " │ │ │ └─ tableId: 5\n" + - " │ │ └─ RecursiveTable(cte)\n" + - " │ └─ HashLookup\n" + - " │ ├─ left-key: TUPLE()\n" + - " │ ├─ right-key: TUPLE()\n" + - " │ └─ TableAlias(u0)\n" + - " │ └─ Table\n" + - " │ ├─ name: keyless\n" + - " │ ├─ columns: [c0 c1]\n" + - " │ ├─ colSet: (14,15)\n" + - " │ └─ tableId: 7\n" + + " │ │ └─ RecursiveCTE\n" + + " │ │ └─ Union all\n" + + " │ │ ├─ Project\n" + + " │ │ │ ├─ columns: [0 (tinyint), t1.c0:2, t1.c1:3]\n" + + " │ │ │ └─ Filter\n" + + " │ │ │ ├─ Eq\n" + + " │ │ │ │ ├─ t1.c0:2\n" + + " │ │ │ │ └─ 0 (bigint)\n" + + " │ │ │ └─ TableAlias(t1)\n" + + " │ │ │ └─ Table\n" + + " │ │ │ ├─ name: keyless\n" + + " │ │ │ ├─ columns: [c0 c1]\n" + + " │ │ │ ├─ colSet: (3,4)\n" + + " │ │ │ └─ tableId: 2\n" + + " │ │ └─ Project\n" + + " │ │ ├─ columns: [(cte.depth:4!null + 1 (tinyint))->cte.depth + 1:0, cte.i:5, (t2.c1:3 + 1 (tinyint))->T2.c1 + 1:0]\n" + + " │ │ └─ InnerJoin\n" + + " │ │ ├─ Eq\n" + + " │ │ │ ├─ cte.depth:4!null\n" + + " │ │ │ └─ t2.c0:2\n" + + " │ │ ├─ TableAlias(t2)\n" + + " │ │ │ └─ Table\n" + + " │ │ │ ├─ name: keyless\n" + + " │ │ │ ├─ columns: [c0 c1]\n" + + " │ │ │ ├─ colSet: (12,13)\n" + + " │ │ │ └─ tableId: 5\n" + + " │ │ └─ RecursiveTable(cte)\n" + + " │ └─ TableAlias(u0)\n" + + " │ └─ Table\n" + + " │ ├─ name: keyless\n" + + " │ ├─ columns: [c0]\n" + + " │ ├─ colSet: (14,15)\n" + + " │ └─ tableId: 7\n" + " └─ ProcessTable\n" + " └─ Table\n" + " ├─ name: keyless\n" + @@ -22396,55 +22392,52 @@ WHERE keyless.c0 IN ( " │ ├─ cacheable: false\n" + " │ ├─ alias-string: with recursive cte (depth, i, j) as (select 0, T1.c0, T1.c1 from keyless as T1 where T1.c0 = 0 union all select cte.depth + 1, cte.i, T2.c1 + 1 from cte, keyless as T2 where cte.depth = T2.c0) select U0.c0 from cte, keyless as U0 where cte.j = keyless.c0\n" + " │ └─ Project\n" + - " │ ├─ columns: [u0.c0:5]\n" + - " │ └─ CrossHashJoin\n" + - " │ ├─ SubqueryAlias\n" + - " │ │ ├─ name: cte\n" + - " │ │ ├─ outerVisibility: true\n" + - " │ │ ├─ isLateral: true\n" + - " │ │ ├─ cacheable: false\n" + - " │ │ ├─ colSet: (14-16)\n" + - " │ │ ├─ tableId: 5\n" + - " │ │ └─ Filter\n" + - " │ │ ├─ Eq\n" + - " │ │ │ ├─ cte.j:4\n" + - " │ │ │ └─ keyless.c0:0\n" + - " │ │ └─ RecursiveCTE\n" + - " │ │ └─ Union all\n" + - " │ │ ├─ Project\n" + - " │ │ │ ├─ columns: [0 (tinyint), t1.c0:2, t1.c1:3]\n" + - " │ │ │ └─ Filter\n" + - " │ │ │ ├─ Eq\n" + - " │ │ │ │ ├─ t1.c0:2\n" + - " │ │ │ │ └─ 0 (bigint)\n" + - " │ │ │ └─ TableAlias(t1)\n" + - " │ │ │ └─ Table\n" + - " │ │ │ ├─ name: keyless\n" + - " │ │ │ ├─ columns: [c0 c1]\n" + - " │ │ │ ├─ colSet: (3,4)\n" + - " │ │ │ └─ tableId: 2\n" + - " │ │ └─ Project\n" + - " │ │ ├─ columns: [(cte.depth:4!null + 1 (tinyint))->cte.depth + 1:0, cte.i:5, (t2.c1:3 + 1 (tinyint))->T2.c1 + 1:0]\n" + - " │ │ └─ InnerJoin\n" + - " │ │ ├─ Eq\n" + - " │ │ │ ├─ cte.depth:4!null\n" + - " │ │ │ └─ t2.c0:2\n" + - " │ │ ├─ TableAlias(t2)\n" + - " │ │ │ └─ Table\n" + - " │ │ │ ├─ name: keyless\n" + - " │ │ │ ├─ columns: [c0 c1]\n" + - " │ │ │ ├─ colSet: (12,13)\n" + - " │ │ │ └─ tableId: 5\n" + - " │ │ └─ RecursiveTable(cte)\n" + - " │ └─ HashLookup\n" + - " │ ├─ left-key: TUPLE()\n" + - " │ ├─ right-key: TUPLE()\n" + - " │ └─ TableAlias(u0)\n" + - " │ └─ Table\n" + - " │ ├─ name: keyless\n" + - " │ ├─ columns: [c0]\n" + - " │ ├─ colSet: (17,18)\n" + - " │ └─ tableId: 7\n" + + " │ ├─ columns: [u0.c0:2]\n" + + " │ └─ InnerJoin\n" + + " │ ├─ Eq\n" + + " │ │ ├─ cte.j:5\n" + + " │ │ └─ keyless.c0:0\n" + + " │ ├─ TableAlias(u0)\n" + + " │ │ └─ Table\n" + + " │ │ ├─ name: keyless\n" + + " │ │ ├─ columns: [c0]\n" + + " │ │ ├─ colSet: (17,18)\n" + + " │ │ └─ tableId: 7\n" + + " │ └─ CachedResults\n" + + " │ └─ SubqueryAlias\n" + + " │ ├─ name: cte\n" + + " │ ├─ outerVisibility: true\n" + + " │ ├─ isLateral: false\n" + + " │ ├─ cacheable: true\n" + + " │ ├─ colSet: (14-16)\n" + + " │ ├─ tableId: 5\n" + + " │ └─ RecursiveCTE\n" + + " │ └─ Union all\n" + + " │ ├─ Project\n" + + " │ │ ├─ columns: [0 (tinyint), t1.c0:2, t1.c1:3]\n" + + " │ │ └─ Filter\n" + + " │ │ ├─ Eq\n" + + " │ │ │ ├─ t1.c0:2\n" + + " │ │ │ └─ 0 (bigint)\n" + + " │ │ └─ TableAlias(t1)\n" + + " │ │ └─ Table\n" + + " │ │ ├─ name: keyless\n" + + " │ │ ├─ columns: [c0 c1]\n" + + " │ │ ├─ colSet: (3,4)\n" + + " │ │ └─ tableId: 2\n" + + " │ └─ Project\n" + + " │ ├─ columns: [(cte.depth:4!null + 1 (tinyint))->cte.depth + 1:0, cte.i:5, (t2.c1:3 + 1 (tinyint))->T2.c1 + 1:0]\n" + + " │ └─ InnerJoin\n" + + " │ ├─ Eq\n" + + " │ │ ├─ cte.depth:5!null\n" + + " │ │ └─ t2.c0:3\n" + + " │ ├─ TableAlias(t2)\n" + + " │ │ └─ Table\n" + + " │ │ ├─ name: keyless\n" + + " │ │ ├─ columns: [c0 c1]\n" + + " │ │ ├─ colSet: (12,13)\n" + + " │ │ └─ tableId: 5\n" + + " │ └─ RecursiveTable(cte)\n" + " └─ ProcessTable\n" + " └─ Table\n" + " ├─ name: keyless\n" + diff --git a/enginetest/queries/script_queries.go b/enginetest/queries/script_queries.go index d64982534d..8de21e1f89 100644 --- a/enginetest/queries/script_queries.go +++ b/enginetest/queries/script_queries.go @@ -14781,6 +14781,89 @@ select * from t1 except ( }, }, }, + { + Name: "Subqueries inside NOT EXISTS clause with correlated column filter", + SetUpScript: []string{ + "CREATE TABLE issues (id INT PRIMARY KEY, title TEXT, status TEXT);", + "CREATE TABLE dependencies (issue_id INT, depends_on_id INT, type TEXT);", + `INSERT INTO issues (id, title, status) VALUES + (1, 'Login API', 'open'), + (2, 'Auth Library', 'in_progress'), + (3, 'User Profile', 'open'), + (4, 'Profile UI', 'open'), + (5, 'Settings Page', 'open'), + (6, 'Marketing Page', 'open'), + (7, 'Old Feature', 'closed');`, + `INSERT INTO dependencies (issue_id, depends_on_id, type) VALUES + (3, 1, 'blocks'), + (3, 2, 'blocks'), + (4, 3, 'parent-child'), + (5, 4, 'parent-child');`, + }, + Assertions: []ScriptTestAssertion{ + { + // https://github.com/dolthub/dolt/issues/10472 + Query: `WITH RECURSIVE + blocked_directly AS ( + SELECT DISTINCT d.issue_id + FROM dependencies d + JOIN issues blocker ON d.depends_on_id = blocker.id + WHERE d.type = 'blocks' + AND blocker.status IN ('open', 'in_progress', 'blocked', 'deferred', 'hooked') + ), + blocked_transitively AS ( + SELECT issue_id, 0 as depth + FROM blocked_directly + UNION ALL + SELECT d.issue_id, bt.depth + 1 + FROM blocked_transitively bt + JOIN dependencies d ON d.depends_on_id = bt.issue_id + WHERE d.type = 'parent-child' + AND bt.depth < 50 + ) + SELECT i.* + FROM issues i + WHERE i.status = 'open' + AND NOT EXISTS ( + SELECT 1 FROM blocked_transitively WHERE issue_id = i.id + );`, + Expected: []sql.Row{{1, "Login API", "open"}, {6, "Marketing Page", "open"}}, + }, + { + // fixing in https://github.com/dolthub/go-mysql-server/pull/3427 + Skip: true, + Query: `WITH blocked_directly AS ( + SELECT DISTINCT d.issue_id + FROM dependencies d + JOIN issues blocker ON d.depends_on_id = blocker.id + WHERE d.type = 'blocks' + AND blocker.status IN ('open', 'in_progress', 'blocked', 'deferred', 'hooked') + ) + SELECT i.id + FROM issues i + WHERE i.status = 'open' + AND NOT EXISTS ( + SELECT 1 FROM blocked_directly WHERE issue_id = i.id + );`, + Expected: []sql.Row{{1}, {4}, {5}, {6}}, + }, + { + // fixing in https://github.com/dolthub/go-mysql-server/pull/3427 + Skip: true, + Query: `SELECT i.id + FROM issues i + WHERE i.status = 'open' + AND NOT EXISTS ( + SELECT 1 FROM ( + SELECT DISTINCT d.issue_id + FROM dependencies d + JOIN issues blocker ON d.depends_on_id = blocker.id + WHERE d.type = 'blocks' AND blocker.status IN ('open', 'in_progress', 'blocked', 'deferred', 'hooked') + ) as blocked WHERE blocked.issue_id = i.id);`, + Expected: []sql.Row{{1}, {4}, {5}, {6}}, + }, + }, + }, } var SpatialScriptTests = []ScriptTest{ diff --git a/sql/analyzer/filters.go b/sql/analyzer/filters.go index 2b94f20d41..ece6bff1a6 100644 --- a/sql/analyzer/filters.go +++ b/sql/analyzer/filters.go @@ -72,18 +72,20 @@ func exprToTableFilters(expr sql.Expression, scope *plan.Scope, projectionExpres findGetFields = func(e sql.Expression) bool { f, ok := e.(*expression.GetField) if ok { + id := f.Id() // A GetField that resolves to an outer scope or lateral scope // is effectively constant and can be skipped. - if scope.Correlated().Contains(f.Id()) { + if scope.Correlated().Contains(id) { return true } - if projectionExpression, ok := projectionExpressions[f.Id()]; ok { + if projectionExpression, ok := projectionExpressions[id]; ok { sql.Inspect(projectionExpression, findGetFields) return true } - if !seenTables[f.Table()] { - seenTables[f.Table()] = true - lastTable = f.Table() + table := f.Table() + if !seenTables[table] { + seenTables[table] = true + lastTable = table } } else if _, isSubquery := e.(*plan.Subquery); isSubquery { hasSubquery = true diff --git a/sql/analyzer/pushdown.go b/sql/analyzer/pushdown.go index 699eec50df..4c4ab2464a 100644 --- a/sql/analyzer/pushdown.go +++ b/sql/analyzer/pushdown.go @@ -37,7 +37,7 @@ func pushFilters(ctx *sql.Context, a *Analyzer, n sql.Node, scope *plan.Scope, s } pushdownAboveTables := func(n sql.Node, filters *filterSet) (sql.Node, transform.TreeIdentity, error) { - return transform.NodeWithCtx(n, filterPushdownChildSelector, func(c transform.Context) (sql.Node, transform.TreeIdentity, error) { + return transform.NodeWithCtx(n, filterPushdownSelector, func(c transform.Context) (sql.Node, transform.TreeIdentity, error) { switch node := c.Node.(type) { case *plan.Filter: // Notably, filters are allowed to be pushed through other filters. @@ -156,40 +156,30 @@ func canDoPushdown(n sql.Node) bool { return true } -// Pushing down a filter is incompatible with the secondary table in a Left or Right join. If we push a predicate on the -// secondary table below the join, we end up not evaluating it in all cases (since the secondary table result is -// sometimes null in these types of joins). It must be evaluated only after the join result is computed. This is also -// true with both tables in a Full Outer join, since either table result could be null. -func filterPushdownChildSelector(c transform.Context) bool { - switch c.Node.(type) { - case *plan.Limit: - return false - } - +// filterPushdownSelector determines if it's valid to push a filter down into a node +func filterPushdownSelector(c transform.Context) bool { switch n := c.Parent.(type) { case *plan.TableAlias: return false - case *plan.Window: - // Windows operate across the rows they see and cannot have - // filters pushed below them. Instead, the step will be run - // again by the Transform function, starting at this node. - return false case *plan.JoinNode: - switch { - case n.Op.IsFullOuter(): + // Pushing down a filter is incompatible with the secondary table in a Left or Right join. If we push a + // predicate on the secondary table below the join, we end up not evaluating it in all cases (since the + // secondary table result is sometimes null in these types of joins). It must be evaluated only after the join + // result is computed. + if n.Op.IsLeftOuter() && c.ChildNum != 0 { return false - case n.Op.IsMerge(): - return false - case n.Op.IsLookup(): - if n.JoinType().IsLeftOuter() { - return c.ChildNum == 0 - } - return true - case n.Op.IsLeftOuter(): - return c.ChildNum == 0 - default: } - default: + } + + switch n := c.Node.(type) { + case *plan.Limit, *plan.Window: + // Limit and Window operate across the rows they see and cannot have filters pushed below them. + return false + case *plan.JoinNode: + // Filters cannot be pushed down into FullOuter joins because it is not null-safe and must be evaluated + // after join result is computed. Filters cannot be pushed down into Merge join because they might result into + // an index lookup that is not monotonically sorted on the join condition + return !(n.Op.IsFullOuter() || n.Op.IsMerge()) } return true } @@ -198,7 +188,7 @@ func transformPushdownSubqueryAliasFilters(ctx *sql.Context, a *Analyzer, n sql. var filters *filterSet transformFilterNode := func(n *plan.Filter) (sql.Node, transform.TreeIdentity, error) { - return transform.NodeWithCtx(n, filterPushdownChildSelector, func(c transform.Context) (sql.Node, transform.TreeIdentity, error) { + return transform.NodeWithCtx(n, filterPushdownSelector, func(c transform.Context) (sql.Node, transform.TreeIdentity, error) { switch node := c.Node.(type) { case *plan.Filter: newF := updateFilterNode(ctx, a, node, filters) @@ -207,6 +197,13 @@ func transformPushdownSubqueryAliasFilters(ctx *sql.Context, a *Analyzer, n sql. } return newF, transform.NewTree, nil case *plan.SubqueryAlias: + // TODO: We probably could push filters into a RecursiveCTE to get an IndexedTableAccess where + // applicable. But we currently don't push any filters through at all so pushing filters past the + // SubqueryAlias node doesn't actually do anything except possibly make them uncacheable, which we + // don't want. + if _, ok := node.Child.(*plan.RecursiveCte); ok { + return node, transform.SameTree, nil + } return pushdownFiltersUnderSubqueryAlias(ctx, a, node, filters) default: return node, transform.SameTree, nil diff --git a/sql/plan/subqueryalias.go b/sql/plan/subqueryalias.go index a3566f6b1a..4d01e33481 100644 --- a/sql/plan/subqueryalias.go +++ b/sql/plan/subqueryalias.go @@ -37,7 +37,6 @@ type SubqueryAlias struct { // expression and is eligible to have visibility to outer scopes of the query. OuterScopeVisibility bool Volatile bool - CacheableCTESource bool IsLateral bool } diff --git a/sql/planbuilder/cte.go b/sql/planbuilder/cte.go index 7e6619ddc7..fdcb4bb6c9 100644 --- a/sql/planbuilder/cte.go +++ b/sql/planbuilder/cte.go @@ -93,11 +93,7 @@ func (b *Builder) buildRecursiveCte(inScope *scope, union *ast.SetOp, name strin switch n := cteScope.node.(type) { case *plan.SetOp: - sq := plan.NewSubqueryAlias(name, "", n) b.qFlags.Set(sql.QFlagRelSubquery) - sq = sq.WithColumnNames(columns) - sq = sq.WithCorrelated(sqScope.correlated()) - sq = sq.WithVolatile(sqScope.volatile()) tabId := cteScope.addTable(name) var colset sql.ColSet @@ -107,7 +103,9 @@ func (b *Builder) buildRecursiveCte(inScope *scope, union *ast.SetOp, name strin colset.Add(sql.ColumnId(c.id)) scopeMapping[sql.ColumnId(c.id)] = c.scalarGf() } - cteScope.node = sq.WithScopeMapping(scopeMapping).WithId(tabId).WithColumns(colset) + cteScope.node = plan.NewSubqueryAlias(name, "", n). + WithColumnNames(columns).WithCorrelated(sqScope.correlated()).WithVolatile(sqScope.volatile()). + WithScopeMapping(scopeMapping).WithId(tabId).WithColumns(colset) } b.renameSource(cteScope, name, columns) return cteScope @@ -191,20 +189,15 @@ func (b *Builder) buildRecursiveCte(inScope *scope, union *ast.SetOp, name strin sortFields = append(sortFields, sf) } - rcte := plan.NewRecursiveCte(rInit, rightScope.node, name, columns, distinct, limit, sortFields) - rcte = rcte.WithSchema(recSch).WithWorking(rTable) corr := leftSqScope.correlated().Union(rightInScope.correlated()) vol := leftSqScope.activeSubquery.volatile || rightInScope.activeSubquery.volatile - rcteId := rcte.WithId(tableId).WithColumns(cols) - - sq := plan.NewSubqueryAlias(name, "", rcteId) b.qFlags.Set(sql.QFlagRelSubquery) - sq = sq.WithColumnNames(columns) - sq = sq.WithCorrelated(corr) - sq = sq.WithVolatile(vol) - sq = sq.WithScopeMapping(scopeMapping) - cteScope.node = sq.WithId(tableId).WithColumns(cols) + cteScope.node = plan.NewSubqueryAlias(name, "", + plan.NewRecursiveCte(rInit, rightScope.node, name, columns, distinct, limit, sortFields). + WithSchema(recSch).WithWorking(rTable).WithId(tableId).WithColumns(cols)). + WithColumnNames(columns).WithCorrelated(corr).WithVolatile(vol).WithScopeMapping(scopeMapping). + WithId(tableId).WithColumns(cols) b.renameSource(cteScope, name, columns) return cteScope }