From 7abd6ea63c17615696d9431af093ed423800bbe7 Mon Sep 17 00:00:00 2001 From: JackTan25 Date: Wed, 13 Dec 2023 18:10:37 +0800 Subject: [PATCH 1/4] add explian for merge into --- .../sql/src/planner/format/display_plan.rs | 90 ++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/src/query/sql/src/planner/format/display_plan.rs b/src/query/sql/src/planner/format/display_plan.rs index 7ef00a7e6ea78..9039888fae3b8 100644 --- a/src/query/sql/src/planner/format/display_plan.rs +++ b/src/query/sql/src/planner/format/display_plan.rs @@ -20,8 +20,10 @@ use common_exception::Result; use common_expression::types::DataType; use common_expression::types::NumberDataType; use common_expression::ROW_ID_COL_NAME; +use itertools::Itertools; use crate::binder::ColumnBindingBuilder; +use crate::format_scalar; use crate::optimizer::SExpr; use crate::planner::format::display_rel_operator::FormatContext; use crate::plans::BoundColumnRef; @@ -29,6 +31,7 @@ use crate::plans::CreateTablePlan; use crate::plans::DeletePlan; use crate::plans::EvalScalar; use crate::plans::Filter; +use crate::plans::MergeInto; use crate::plans::Plan; use crate::plans::RelOperator; use crate::plans::ScalarItem; @@ -110,7 +113,7 @@ impl Plan { // Insert Plan::Insert(_) => Ok("Insert".to_string()), Plan::Replace(_) => Ok("Replace".to_string()), - Plan::MergeInto(_) => Ok("MergeInto".to_string()), + Plan::MergeInto(merge_into) => format_merge_into(merge_into), Plan::Delete(delete) => format_delete(delete), Plan::Update(_) => Ok("Update".to_string()), @@ -264,3 +267,88 @@ fn format_create_table(create_table: &CreateTablePlan) -> Result { None => Ok("CreateTable".to_string()), } } + +fn format_merge_into(merge_into: &MergeInto) -> Result { + // add merge into target_table + let table_index = merge_into + .meta_data + .read() + .get_table_index( + Some(merge_into.database.as_str()), + merge_into.table.as_str(), + ) + .unwrap(); + + let table_entry = merge_into.meta_data.read().table(table_index).clone(); + let target_table_format = FormatContext::Text(format!( + "target_table: {}.{}.{}", + table_entry.catalog(), + table_entry.database(), + table_entry.name(), + )); + + // add macthed clauses + let mut matched_children = Vec::with_capacity(merge_into.matched_evaluators.len() as usize); + let taregt_schema = table_entry.table().schema(); + for evaluator in &merge_into.matched_evaluators { + let condition_format = evaluator.condition.as_ref().map_or_else( + || "condition: None".to_string(), + |predicate| format!("condition: {}", format_scalar(predicate)), + ); + if evaluator.update.is_none() { + matched_children.push(FormatTreeNode::new(FormatContext::Text(format!( + "matched delete: [{}]", + condition_format + )))); + } else { + let update_format = evaluator + .update + .as_ref() + .unwrap() + .iter() + .map(|(field_idx, expr)| { + format!( + "{} = {}", + taregt_schema.field(*field_idx).name().to_string(), + format_scalar(expr) + ) + }) + .join(","); + matched_children.push(FormatTreeNode::new(FormatContext::Text(format!( + "matched update: [{},update set {}]", + condition_format, update_format + )))); + } + } + // add unmacthed clauses + let mut unmatched_children = Vec::with_capacity(merge_into.unmatched_evaluators.len() as usize); + for evaluator in &merge_into.unmatched_evaluators { + let condition_format = evaluator.condition.as_ref().map_or_else( + || "condition: None".to_string(), + |predicate| format!("condition: {}", format_scalar(predicate)), + ); + let insert_schema_format = evaluator + .source_schema + .fields + .iter() + .map(|field| field.name()) + .join(","); + let values_format = evaluator.values.iter().map(format_scalar).join(","); + let unmatched_format = format!( + "insert into ({}) values({})", + insert_schema_format, values_format + ); + unmatched_children.push(FormatTreeNode::new(FormatContext::Text(format!( + "unmatched insert: [{},{}]", + condition_format, unmatched_format + )))); + } + let s_expr = merge_into.input.as_ref(); + let input_format_child = s_expr.to_format_tree(&merge_into.meta_data); + let all_children = vec![matched_children, unmatched_children, vec![ + input_format_child, + ]] + .concat(); + let res = FormatTreeNode::with_children(target_table_format, all_children).format_pretty()?; + Ok(format!("MergeInto:\n{res}")) +} From 2b0087a0deef9180354c9bf8c5ba5863d6f8cd99 Mon Sep 17 00:00:00 2001 From: JackTan25 Date: Wed, 13 Dec 2023 19:16:46 +0800 Subject: [PATCH 2/4] add test --- .../sql/src/planner/format/display_plan.rs | 8 +++---- ...merge_into_without_distributed_enable.test | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/query/sql/src/planner/format/display_plan.rs b/src/query/sql/src/planner/format/display_plan.rs index 9039888fae3b8..28493a9069ce9 100644 --- a/src/query/sql/src/planner/format/display_plan.rs +++ b/src/query/sql/src/planner/format/display_plan.rs @@ -288,7 +288,7 @@ fn format_merge_into(merge_into: &MergeInto) -> Result { )); // add macthed clauses - let mut matched_children = Vec::with_capacity(merge_into.matched_evaluators.len() as usize); + let mut matched_children = Vec::with_capacity(merge_into.matched_evaluators.len()); let taregt_schema = table_entry.table().schema(); for evaluator in &merge_into.matched_evaluators { let condition_format = evaluator.condition.as_ref().map_or_else( @@ -309,7 +309,7 @@ fn format_merge_into(merge_into: &MergeInto) -> Result { .map(|(field_idx, expr)| { format!( "{} = {}", - taregt_schema.field(*field_idx).name().to_string(), + taregt_schema.field(*field_idx).name(), format_scalar(expr) ) }) @@ -321,7 +321,7 @@ fn format_merge_into(merge_into: &MergeInto) -> Result { } } // add unmacthed clauses - let mut unmatched_children = Vec::with_capacity(merge_into.unmatched_evaluators.len() as usize); + let mut unmatched_children = Vec::with_capacity(merge_into.unmatched_evaluators.len()); for evaluator in &merge_into.unmatched_evaluators { let condition_format = evaluator.condition.as_ref().map_or_else( || "condition: None".to_string(), @@ -345,7 +345,7 @@ fn format_merge_into(merge_into: &MergeInto) -> Result { } let s_expr = merge_into.input.as_ref(); let input_format_child = s_expr.to_format_tree(&merge_into.meta_data); - let all_children = vec![matched_children, unmatched_children, vec![ + let all_children = [matched_children, unmatched_children, vec![ input_format_child, ]] .concat(); diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0036_merge_into_without_distributed_enable.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0036_merge_into_without_distributed_enable.test index b80104e9f28ff..089cd192974d4 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0036_merge_into_without_distributed_enable.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0036_merge_into_without_distributed_enable.test @@ -484,6 +484,30 @@ MERGE INTO salaries USING (SELECT * FROM employees) as employees ON salaries.emp ---- 2 2 +query T +explain MERGE INTO salaries USING (SELECT * FROM employees) as employees ON salaries.employee_id = employees.employee_id WHEN MATCHED AND employees.department = 'HR' THEN UPDATE SET salaries.salary = salaries.salary + 1000.00 WHEN MATCHED THEN UPDATE SET salaries.salary = salaries.salary + 500.00 WHEN NOT MATCHED THEN INSERT (employee_id, salary) VALUES (employees.employee_id, 55000.00); +---- +MergeInto: +target_table: default.default.salaries +├── matched update: [condition: eq(employees.department (#2), 'HR'),update set salary = plus(salaries.salary (#4), 1000.00)] +├── matched update: [condition: None,update set salary = plus(salaries.salary (#4), 500.00)] +├── unmatched insert: [condition: None,insert into (employee_id,salary) values(CAST(employees.employee_id (#0) AS Int32 NULL),CAST(55000.00 AS Decimal(10, 2) NULL))] +└── HashJoin: RIGHT OUTER + ├── equi conditions: [eq(salaries.employee_id (#3), employees.employee_id (#0))] + ├── non-equi conditions: [] + ├── LogicalGet + │ ├── table: default.default.salaries + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── EvalScalar + ├── scalars: [employees.employee_id (#0), employees.employee_name (#1), employees.department (#2)] + └── LogicalGet + ├── table: default.default.employees + ├── filters: [] + ├── order by: [] + └── limit: NONE + query TTT select * from salaries order by employee_id; ---- From 0dc020db8e068baf36056152402ff2785aff8028 Mon Sep 17 00:00:00 2001 From: JackTan25 Date: Wed, 13 Dec 2023 19:35:53 +0800 Subject: [PATCH 3/4] move test --- ...merge_into_without_distributed_enable.test | 24 ----------- .../mode/standalone/explain/merge_into.test | 40 +++++++++++++++++++ 2 files changed, 40 insertions(+), 24 deletions(-) create mode 100644 tests/sqllogictests/suites/mode/standalone/explain/merge_into.test diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0036_merge_into_without_distributed_enable.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0036_merge_into_without_distributed_enable.test index 089cd192974d4..b80104e9f28ff 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0036_merge_into_without_distributed_enable.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0036_merge_into_without_distributed_enable.test @@ -484,30 +484,6 @@ MERGE INTO salaries USING (SELECT * FROM employees) as employees ON salaries.emp ---- 2 2 -query T -explain MERGE INTO salaries USING (SELECT * FROM employees) as employees ON salaries.employee_id = employees.employee_id WHEN MATCHED AND employees.department = 'HR' THEN UPDATE SET salaries.salary = salaries.salary + 1000.00 WHEN MATCHED THEN UPDATE SET salaries.salary = salaries.salary + 500.00 WHEN NOT MATCHED THEN INSERT (employee_id, salary) VALUES (employees.employee_id, 55000.00); ----- -MergeInto: -target_table: default.default.salaries -├── matched update: [condition: eq(employees.department (#2), 'HR'),update set salary = plus(salaries.salary (#4), 1000.00)] -├── matched update: [condition: None,update set salary = plus(salaries.salary (#4), 500.00)] -├── unmatched insert: [condition: None,insert into (employee_id,salary) values(CAST(employees.employee_id (#0) AS Int32 NULL),CAST(55000.00 AS Decimal(10, 2) NULL))] -└── HashJoin: RIGHT OUTER - ├── equi conditions: [eq(salaries.employee_id (#3), employees.employee_id (#0))] - ├── non-equi conditions: [] - ├── LogicalGet - │ ├── table: default.default.salaries - │ ├── filters: [] - │ ├── order by: [] - │ └── limit: NONE - └── EvalScalar - ├── scalars: [employees.employee_id (#0), employees.employee_name (#1), employees.department (#2)] - └── LogicalGet - ├── table: default.default.employees - ├── filters: [] - ├── order by: [] - └── limit: NONE - query TTT select * from salaries order by employee_id; ---- diff --git a/tests/sqllogictests/suites/mode/standalone/explain/merge_into.test b/tests/sqllogictests/suites/mode/standalone/explain/merge_into.test new file mode 100644 index 0000000000000..e7b8dd0e52892 --- /dev/null +++ b/tests/sqllogictests/suites/mode/standalone/explain/merge_into.test @@ -0,0 +1,40 @@ +statement ok +CREATE TABLE employees2 (employee_id INT, employee_name VARCHAR(255),department VARCHAR(255)); + +statement ok +CREATE TABLE salaries2 (employee_id INT,salary DECIMAL(10, 2)); + +statement ok +INSERT INTO employees2 VALUES(1, 'Alice', 'HR'),(2, 'Bob', 'IT'),(3, 'Charlie', 'Finance'),(4, 'David', 'HR'); + +statement ok +INSERT INTO salaries2 VALUES(1, 50000.00),(2, 60000.00); + +query TT +MERGE INTO salaries2 USING (SELECT * FROM employees2) as employees2 ON salaries2.employee_id = employees2.employee_id WHEN MATCHED AND employees2.department = 'HR' THEN UPDATE SET salaries2.salary = salaries2.salary + 1000.00 WHEN MATCHED THEN UPDATE SET salaries2.salary = salaries2.salary + 500.00 WHEN NOT MATCHED THEN INSERT (employee_id, salary) VALUES (employees2.employee_id, 55000.00); +---- +2 2 + +query T +explain MERGE INTO salaries2 USING (SELECT * FROM employees2) as employees2 ON salaries2.employee_id = employees2.employee_id WHEN MATCHED AND employees2.department = 'HR' THEN UPDATE SET salaries2.salary = salaries2.salary + 1000.00 WHEN MATCHED THEN UPDATE SET salaries2.salary = salaries2.salary + 500.00 WHEN NOT MATCHED THEN INSERT (employee_id, salary) VALUES (employees2.employee_id, 55000.00); +---- +MergeInto: +target_table: default.default.salaries2 +├── matched update: [condition: eq(employees2.department (#2), 'HR'),update set salary = plus(salaries2.salary (#4), 1000.00)] +├── matched update: [condition: None,update set salary = plus(salaries2.salary (#4), 500.00)] +├── unmatched insert: [condition: None,insert into (employee_id,salary) values(CAST(employees2.employee_id (#0) AS Int32 NULL),CAST(55000.00 AS Decimal(10, 2) NULL))] +└── HashJoin: RIGHT OUTER + ├── equi conditions: [eq(salaries2.employee_id (#3), employees2.employee_id (#0))] + ├── non-equi conditions: [] + ├── LogicalGet + │ ├── table: default.default.salaries2 + │ ├── filters: [] + │ ├── order by: [] + │ └── limit: NONE + └── EvalScalar + ├── scalars: [employees2.employee_id (#0), employees2.employee_name (#1), employees2.department (#2)] + └── LogicalGet + ├── table: default.default.employees2 + ├── filters: [] + ├── order by: [] + └── limit: NONE \ No newline at end of file From 7c5d97f2e7ebfc9d13eae03caa5a1aac54eb2586 Mon Sep 17 00:00:00 2001 From: JackTan25 Date: Wed, 13 Dec 2023 19:37:30 +0800 Subject: [PATCH 4/4] add setting --- .../suites/mode/standalone/explain/merge_into.test | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/sqllogictests/suites/mode/standalone/explain/merge_into.test b/tests/sqllogictests/suites/mode/standalone/explain/merge_into.test index e7b8dd0e52892..7d59f85d75e0b 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/merge_into.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/merge_into.test @@ -1,3 +1,6 @@ +statement ok +set enable_experimental_merge_into = 1; + statement ok CREATE TABLE employees2 (employee_id INT, employee_name VARCHAR(255),department VARCHAR(255)); @@ -37,4 +40,7 @@ target_table: default.default.salaries2 ├── table: default.default.employees2 ├── filters: [] ├── order by: [] - └── limit: NONE \ No newline at end of file + └── limit: NONE + +statement ok +set enable_experimental_merge_into = 0; \ No newline at end of file