diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q05.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q05.plan.txt index ebed85d1de978..aeff998bfd1f8 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q05.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q05.plan.txt @@ -1,57 +1,56 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) - remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) - partial aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, ROUND_ROBIN, []) - final aggregation over (s_store_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [s_store_id]) - partial aggregation over (s_store_id) +remote exchange (GATHER, SINGLE, []) + final aggregation over (channel$gid, groupid, id$gid) + local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + partial aggregation over (channel$gid, groupid, id$gid) + local exchange (REPARTITION, ROUND_ROBIN, []) + final aggregation over (s_store_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [s_store_id]) + partial aggregation over (s_store_id) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - remote exchange (REPARTITION, ROUND_ROBIN, []) - scan store_sales - scan store_returns - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + remote exchange (REPARTITION, ROUND_ROBIN, []) + scan store_sales + scan store_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan store - final aggregation over (cp_catalog_page_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [cp_catalog_page_id]) - partial aggregation over (cp_catalog_page_id) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan store + final aggregation over (cp_catalog_page_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [cp_catalog_page_id]) + partial aggregation over (cp_catalog_page_id) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - remote exchange (REPARTITION, ROUND_ROBIN, []) - scan catalog_sales - scan catalog_returns - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + remote exchange (REPARTITION, ROUND_ROBIN, []) + scan catalog_sales + scan catalog_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan catalog_page - final aggregation over (web_site_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [web_site_id]) - partial aggregation over (web_site_id) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan catalog_page + final aggregation over (web_site_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [web_site_id]) + partial aggregation over (web_site_id) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - remote exchange (REPARTITION, ROUND_ROBIN, []) - scan web_sales - join (RIGHT, PARTITIONED): - remote exchange (REPARTITION, HASH, [ws_item_sk_216, ws_order_number_230]) - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [wr_item_sk, wr_order_number]) - scan web_returns - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + remote exchange (REPARTITION, ROUND_ROBIN, []) + scan web_sales + join (RIGHT, PARTITIONED): + remote exchange (REPARTITION, HASH, [ws_item_sk_216, ws_order_number_230]) + scan web_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [wr_item_sk, wr_order_number]) + scan web_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan web_site + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan web_site diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_1.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_1.plan.txt index 9d5966e6652b2..f86567d03c0b8 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_1.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_1.plan.txt @@ -1,244 +1,243 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743) - local exchange (REPARTITION, HASH, [channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743]) - remote exchange (REPARTITION, HASH, [channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743]) - partial aggregation over (channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743) - local exchange (REPARTITION, ROUND_ROBIN, []) - cross join: - final aggregation over (i_brand_id, i_category_id, i_class_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_brand_id, i_category_id, i_class_id]) - partial aggregation over (i_brand_id, i_category_id, i_class_id) - semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, [ss_item_sk]) +remote exchange (GATHER, SINGLE, []) + final aggregation over (channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743) + local exchange (REPARTITION, HASH, [channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743]) + remote exchange (REPARTITION, HASH, [channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743]) + partial aggregation over (channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743) + local exchange (REPARTITION, ROUND_ROBIN, []) + cross join: + final aggregation over (i_brand_id, i_category_id, i_class_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [i_brand_id, i_category_id, i_class_id]) + partial aggregation over (i_brand_id, i_category_id, i_class_id) + semijoin (PARTITIONED): + remote exchange (REPARTITION, HASH, [ss_item_sk]) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_item_sk_1]) - join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, [i_brand_id_8, i_category_id_12, i_class_id_10]) - scan item - final aggregation over (expr_216, expr_217, expr_218) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_brand_id_53, i_category_id_57, i_class_id_55]) - partial aggregation over (i_brand_id_53, i_category_id_57, i_class_id_55) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [i_item_sk_1]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, [i_brand_id_8, i_category_id_12, i_class_id_10]) + scan item + final aggregation over (expr_216, expr_217, expr_218) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [i_brand_id_53, i_category_id_57, i_class_id_55]) + partial aggregation over (i_brand_id_53, i_category_id_57, i_class_id_55) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - remote exchange (REPARTITION, HASH, [i_brand_id_108, i_category_id_112, i_class_id_110]) - partial aggregation over (i_brand_id_108, i_category_id_112, i_class_id_110) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + remote exchange (REPARTITION, HASH, [i_brand_id_108, i_category_id_112, i_class_id_110]) + partial aggregation over (i_brand_id_108, i_category_id_112, i_class_id_110) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - remote exchange (REPARTITION, HASH, [i_brand_id_167, i_category_id_171, i_class_id_169]) - partial aggregation over (i_brand_id_167, i_category_id_171, i_class_id_169) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + remote exchange (REPARTITION, HASH, [i_brand_id_167, i_category_id_171, i_class_id_169]) + partial aggregation over (i_brand_id_167, i_category_id_171, i_class_id_169) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + final aggregation over () + local exchange (GATHER, SINGLE, []) + remote exchange (GATHER, SINGLE, []) + partial aggregation over () + join (INNER, REPLICATED): + scan store_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + partial aggregation over () + join (INNER, REPLICATED): + scan catalog_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + partial aggregation over () + join (INNER, REPLICATED): + scan web_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + cross join: + final aggregation over (i_brand_id_508, i_category_id_512, i_class_id_510) local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - final aggregation over () - local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - partial aggregation over () - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - partial aggregation over () + remote exchange (REPARTITION, HASH, [i_brand_id_508, i_category_id_512, i_class_id_510]) + partial aggregation over (i_brand_id_508, i_category_id_512, i_class_id_510) + semijoin (PARTITIONED): + remote exchange (REPARTITION, HASH, [cs_item_sk_482]) + join (INNER, REPLICATED): join (INNER, REPLICATED): scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim - partial aggregation over () - join (INNER, REPLICATED): - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - cross join: - final aggregation over (i_brand_id_508, i_category_id_512, i_class_id_510) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_brand_id_508, i_category_id_512, i_class_id_510]) - partial aggregation over (i_brand_id_508, i_category_id_512, i_class_id_510) - semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, [cs_item_sk_482]) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan catalog_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [i_item_sk_552]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, [i_brand_id_559, i_category_id_563, i_class_id_561]) + scan item + final aggregation over (expr_836, expr_837, expr_838) local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan item - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_item_sk_552]) - join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, [i_brand_id_559, i_category_id_563, i_class_id_561]) - scan item - final aggregation over (expr_836, expr_837, expr_838) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_brand_id_604, i_category_id_608, i_class_id_606]) - partial aggregation over (i_brand_id_604, i_category_id_608, i_class_id_606) + remote exchange (REPARTITION, HASH, [i_brand_id_604, i_category_id_608, i_class_id_606]) + partial aggregation over (i_brand_id_604, i_category_id_608, i_class_id_606) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - remote exchange (REPARTITION, HASH, [i_brand_id_694, i_category_id_698, i_class_id_696]) - partial aggregation over (i_brand_id_694, i_category_id_698, i_class_id_696) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + remote exchange (REPARTITION, HASH, [i_brand_id_694, i_category_id_698, i_class_id_696]) + partial aggregation over (i_brand_id_694, i_category_id_698, i_class_id_696) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - remote exchange (REPARTITION, HASH, [i_brand_id_787, i_category_id_791, i_class_id_789]) - partial aggregation over (i_brand_id_787, i_category_id_791, i_class_id_789) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + remote exchange (REPARTITION, HASH, [i_brand_id_787, i_category_id_791, i_class_id_789]) + partial aggregation over (i_brand_id_787, i_category_id_791, i_class_id_789) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + final aggregation over () + local exchange (GATHER, SINGLE, []) + remote exchange (GATHER, SINGLE, []) + partial aggregation over () + join (INNER, REPLICATED): + scan store_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + partial aggregation over () + join (INNER, REPLICATED): + scan catalog_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + partial aggregation over () + join (INNER, REPLICATED): + scan web_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + cross join: + final aggregation over (i_brand_id_1135, i_category_id_1139, i_class_id_1137) local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - final aggregation over () - local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - partial aggregation over () - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - partial aggregation over () - join (INNER, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - partial aggregation over () + remote exchange (REPARTITION, HASH, [i_brand_id_1135, i_category_id_1139, i_class_id_1137]) + partial aggregation over (i_brand_id_1135, i_category_id_1139, i_class_id_1137) + semijoin (PARTITIONED): + remote exchange (REPARTITION, HASH, [ws_item_sk_1097]) + join (INNER, REPLICATED): join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim - cross join: - final aggregation over (i_brand_id_1135, i_category_id_1139, i_class_id_1137) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_brand_id_1135, i_category_id_1139, i_class_id_1137]) - partial aggregation over (i_brand_id_1135, i_category_id_1139, i_class_id_1137) - semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, [ws_item_sk_1097]) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan web_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [i_item_sk_1179]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, [i_brand_id_1186, i_category_id_1190, i_class_id_1188]) + scan item + final aggregation over (expr_1463, expr_1464, expr_1465) local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan item - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_item_sk_1179]) - join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, [i_brand_id_1186, i_category_id_1190, i_class_id_1188]) - scan item - final aggregation over (expr_1463, expr_1464, expr_1465) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_brand_id_1231, i_category_id_1235, i_class_id_1233]) - partial aggregation over (i_brand_id_1231, i_category_id_1235, i_class_id_1233) + remote exchange (REPARTITION, HASH, [i_brand_id_1231, i_category_id_1235, i_class_id_1233]) + partial aggregation over (i_brand_id_1231, i_category_id_1235, i_class_id_1233) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - remote exchange (REPARTITION, HASH, [i_brand_id_1321, i_category_id_1325, i_class_id_1323]) - partial aggregation over (i_brand_id_1321, i_category_id_1325, i_class_id_1323) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + remote exchange (REPARTITION, HASH, [i_brand_id_1321, i_category_id_1325, i_class_id_1323]) + partial aggregation over (i_brand_id_1321, i_category_id_1325, i_class_id_1323) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - remote exchange (REPARTITION, HASH, [i_brand_id_1414, i_category_id_1418, i_class_id_1416]) - partial aggregation over (i_brand_id_1414, i_category_id_1418, i_class_id_1416) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + remote exchange (REPARTITION, HASH, [i_brand_id_1414, i_category_id_1418, i_class_id_1416]) + partial aggregation over (i_brand_id_1414, i_category_id_1418, i_class_id_1416) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - final aggregation over () - local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - partial aggregation over () - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - partial aggregation over () - join (INNER, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - partial aggregation over () - join (INNER, REPLICATED): - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + final aggregation over () + local exchange (GATHER, SINGLE, []) + remote exchange (GATHER, SINGLE, []) + partial aggregation over () + join (INNER, REPLICATED): + scan store_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + partial aggregation over () + join (INNER, REPLICATED): + scan catalog_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + partial aggregation over () + join (INNER, REPLICATED): + scan web_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q18.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q18.plan.txt index 7d4c99ffd8b9d..c9d1e8288375e 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q18.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q18.plan.txt @@ -1,33 +1,32 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid) - local exchange (REPARTITION, HASH, [ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid]) - remote exchange (REPARTITION, HASH, [ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid]) - partial aggregation over (ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid) +remote exchange (GATHER, SINGLE, []) + final aggregation over (ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid) + local exchange (REPARTITION, HASH, [ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid]) + remote exchange (REPARTITION, HASH, [ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid]) + partial aggregation over (ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid) + join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan customer_demographics + scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, [c_current_cdemo_sk]) - join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, [c_current_addr_sk]) - scan customer - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [ca_address_sk]) - scan customer_address - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [cd_demo_sk_0]) - scan customer_demographics + scan customer_demographics local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, [c_current_cdemo_sk]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, [c_current_addr_sk]) + scan customer + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [ca_address_sk]) + scan customer_address + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [cd_demo_sk_0]) + scan customer_demographics local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q22.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q22.plan.txt index 38c414f6d58fb..28ca93776bcf0 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q22.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q22.plan.txt @@ -1,15 +1,14 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid) - local exchange (REPARTITION, HASH, [groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid]) - remote exchange (REPARTITION, HASH, [groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid]) - partial aggregation over (groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid) +remote exchange (GATHER, SINGLE, []) + final aggregation over (groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid) + local exchange (REPARTITION, HASH, [groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid]) + remote exchange (REPARTITION, HASH, [groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid]) + partial aggregation over (groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan inventory - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan inventory local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q27.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q27.plan.txt index 042bd1c805181..7e3746fafa38a 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q27.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q27.plan.txt @@ -1,24 +1,23 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (groupid, i_item_id$gid, s_state$gid) - local exchange (REPARTITION, HASH, [groupid, i_item_id$gid, s_state$gid]) - remote exchange (REPARTITION, HASH, [groupid, i_item_id$gid, s_state$gid]) - partial aggregation over (groupid, i_item_id$gid, s_state$gid) - join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, [ss_item_sk]) +remote exchange (GATHER, SINGLE, []) + final aggregation over (groupid, i_item_id$gid, s_state$gid) + local exchange (REPARTITION, HASH, [groupid, i_item_id$gid, s_state$gid]) + remote exchange (REPARTITION, HASH, [groupid, i_item_id$gid, s_state$gid]) + partial aggregation over (groupid, i_item_id$gid, s_state$gid) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, [ss_item_sk]) + join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan customer_demographics + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan store + scan customer_demographics local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_item_sk]) - scan item + scan store + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [i_item_sk]) + scan item diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q77.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q77.plan.txt index 765e1f82b3284..ea5a1a88c2650 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q77.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q77.plan.txt @@ -1,82 +1,81 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) - remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) - partial aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, ROUND_ROBIN, []) - join (LEFT, PARTITIONED): - final aggregation over (s_store_sk) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [s_store_sk]) - partial aggregation over (s_store_sk) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan store - final aggregation over (s_store_sk_46) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [s_store_sk_46]) - partial aggregation over (s_store_sk_46) +remote exchange (GATHER, SINGLE, []) + final aggregation over (channel$gid, groupid, id$gid) + local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + partial aggregation over (channel$gid, groupid, id$gid) + local exchange (REPARTITION, ROUND_ROBIN, []) + join (LEFT, PARTITIONED): + final aggregation over (s_store_sk) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [s_store_sk]) + partial aggregation over (s_store_sk) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_returns - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan store - cross join: - final aggregation over (cs_call_center_sk) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [cs_call_center_sk]) - partial aggregation over (cs_call_center_sk) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan store + final aggregation over (s_store_sk_46) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [s_store_sk_46]) + partial aggregation over (s_store_sk_46) + join (INNER, REPLICATED): join (INNER, REPLICATED): - scan catalog_sales + scan store_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan store + cross join: + final aggregation over (cs_call_center_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - final aggregation over (cr_call_center_sk) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [cr_call_center_sk]) - partial aggregation over (cr_call_center_sk) - join (INNER, REPLICATED): - scan catalog_returns - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - join (LEFT, PARTITIONED): - final aggregation over (wp_web_page_sk) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [wp_web_page_sk]) - partial aggregation over (wp_web_page_sk) - join (INNER, REPLICATED): + remote exchange (REPARTITION, HASH, [cs_call_center_sk]) + partial aggregation over (cs_call_center_sk) + join (INNER, REPLICATED): + scan catalog_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + final aggregation over (cr_call_center_sk) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [cr_call_center_sk]) + partial aggregation over (cr_call_center_sk) join (INNER, REPLICATED): - scan web_sales + scan catalog_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim + join (LEFT, PARTITIONED): + final aggregation over (wp_web_page_sk) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [wp_web_page_sk]) + partial aggregation over (wp_web_page_sk) + join (INNER, REPLICATED): + join (INNER, REPLICATED): + scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan web_page - final aggregation over (wp_web_page_sk_298) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [wp_web_page_sk_298]) - partial aggregation over (wp_web_page_sk_298) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan web_page + final aggregation over (wp_web_page_sk_298) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [wp_web_page_sk_298]) + partial aggregation over (wp_web_page_sk_298) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan web_returns - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan web_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan web_page + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan web_page diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q80.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q80.plan.txt index 99c9b4a65e450..b198e899773ae 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q80.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q80.plan.txt @@ -1,82 +1,81 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) - remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) - partial aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, ROUND_ROBIN, []) - final aggregation over (s_store_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [s_store_id]) - partial aggregation over (s_store_id) +remote exchange (GATHER, SINGLE, []) + final aggregation over (channel$gid, groupid, id$gid) + local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + partial aggregation over (channel$gid, groupid, id$gid) + local exchange (REPARTITION, ROUND_ROBIN, []) + final aggregation over (s_store_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [s_store_id]) + partial aggregation over (s_store_id) + join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - join (LEFT, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan store_returns + join (LEFT, REPLICATED): + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan store_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan promotion + scan item local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan store - final aggregation over (cp_catalog_page_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [cp_catalog_page_id]) - partial aggregation over (cp_catalog_page_id) + scan promotion + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan store + final aggregation over (cp_catalog_page_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [cp_catalog_page_id]) + partial aggregation over (cp_catalog_page_id) + join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - join (LEFT, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan catalog_returns + join (LEFT, REPLICATED): + scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan catalog_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan promotion + scan item local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan catalog_page - final aggregation over (web_site_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [web_site_id]) - partial aggregation over (web_site_id) + scan promotion + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan catalog_page + final aggregation over (web_site_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [web_site_id]) + partial aggregation over (web_site_id) + join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - join (LEFT, REPLICATED): - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan web_returns + join (LEFT, REPLICATED): + scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan web_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan promotion + scan item local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan web_site + scan promotion + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan web_site diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index a3a2bda9ceecd..ea73a92df1270 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -170,6 +170,7 @@ public final class SystemSessionProperties public static final String INLINE_SQL_FUNCTIONS = "inline_sql_functions"; public static final String REMOTE_FUNCTIONS_ENABLED = "remote_functions_enabled"; public static final String CHECK_ACCESS_CONTROL_ON_UTILIZED_COLUMNS_ONLY = "check_access_control_on_utilized_columns_only"; + public static final String SKIP_REDUNDANT_SORT = "skip_redundant_sort"; private final List> sessionProperties; @@ -870,6 +871,11 @@ public SystemSessionProperties( "Enable warning for query relying on legacy type coercion", featuresConfig.isLegacyDateTimestampToVarcharCoercion(), true), + booleanProperty( + SKIP_REDUNDANT_SORT, + "Skip redundant sort operations", + featuresConfig.isSkipRedundantSort(), + false), booleanProperty( INLINE_SQL_FUNCTIONS, "Inline SQL function definition at plan time", @@ -887,6 +893,11 @@ public SystemSessionProperties( false)); } + public static boolean isSkipRedundantSort(Session session) + { + return session.getSystemProperty(SKIP_REDUNDANT_SORT, Boolean.class); + } + public List> getSessionProperties() { return sessionProperties; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java index 2292e0159192c..3fc7224538a7f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java @@ -99,6 +99,7 @@ public class Analysis private final Map, Expression> where = new LinkedHashMap<>(); private final Map, Expression> having = new LinkedHashMap<>(); private final Map, List> orderByExpressions = new LinkedHashMap<>(); + private final Set> redundantOrderBy = new HashSet<>(); private final Map, List> outputExpressions = new LinkedHashMap<>(); private final Map, List> windowFunctions = new LinkedHashMap<>(); private final Map, List> orderByWindowFunctions = new LinkedHashMap<>(); @@ -698,6 +699,16 @@ public Map>> getTableCol return isCheckAccessControlOnUtilizedColumnsOnly(session) ? utilizedTableColumnReferences : tableColumnReferences; } + public void markRedundantOrderBy(OrderBy orderBy) + { + redundantOrderBy.add(NodeRef.of(orderBy)); + } + + public boolean isOrderByRedundant(OrderBy orderBy) + { + return redundantOrderBy.contains(NodeRef.of(orderBy)); + } + @Immutable public static final class Insert { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index 25b0dd652319e..545b400a14276 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -171,6 +171,7 @@ public class FeaturesConfig private boolean pushdownDereferenceEnabled; private boolean inlineSqlFunctions = true; private boolean checkAccessControlOnUtilizedColumnsOnly; + private boolean skipRedundantSort = true; private String warnOnNoTableLayoutFilter = ""; @@ -1464,4 +1465,16 @@ public FeaturesConfig setCheckAccessControlOnUtilizedColumnsOnly(boolean checkAc this.checkAccessControlOnUtilizedColumnsOnly = checkAccessControlOnUtilizedColumnsOnly; return this; } + + public boolean isSkipRedundantSort() + { + return skipRedundantSort; + } + + @Config("optimizer.skip-redundant-sort") + public FeaturesConfig setSkipRedundantSort(boolean value) + { + this.skipRedundantSort = value; + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java index bbb3b7cf9d5c9..f697e3c759eb5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java @@ -163,6 +163,7 @@ import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; import static com.facebook.presto.spi.StandardWarningCode.PERFORMANCE_WARNING; +import static com.facebook.presto.spi.StandardWarningCode.REDUNDANT_ORDER_BY; import static com.facebook.presto.spi.function.FunctionKind.AGGREGATE; import static com.facebook.presto.spi.function.FunctionKind.WINDOW; import static com.facebook.presto.sql.NodeUtils.getSortItemsFromOrderBy; @@ -827,12 +828,16 @@ protected Scope visitQuery(Query node, Optional scope) { Scope withScope = analyzeWith(node, scope); Scope queryBodyScope = process(node.getQueryBody(), withScope); + List orderByExpressions = emptyList(); if (node.getOrderBy().isPresent()) { - analyzeOrderBy(node, queryBodyScope); - } - else { - analysis.setOrderByExpressions(node, emptyList()); + orderByExpressions = analyzeOrderBy(node, getSortItemsFromOrderBy(node.getOrderBy()), queryBodyScope); + if (queryBodyScope.getOuterQueryParent().isPresent() && !node.getLimit().isPresent()) { + // not the root scope and ORDER BY is ineffective + analysis.markRedundantOrderBy(node.getOrderBy().get()); + warningCollector.add(new PrestoWarning(REDUNDANT_ORDER_BY, "ORDER BY in subquery may have no effect")); + } } + analysis.setOrderByExpressions(node, orderByExpressions); // Input fields == Output fields analysis.setOutputExpressions(node, descriptorToFields(queryBodyScope)); @@ -1122,12 +1127,22 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional List orderByExpressions = emptyList(); Optional orderByScope = Optional.empty(); if (node.getOrderBy().isPresent()) { - orderByScope = Optional.of(computeAndAssignOrderByScope(node.getOrderBy().get(), sourceScope, outputScope)); - orderByExpressions = analyzeOrderBy(node, orderByScope.get(), outputExpressions); - } - else { - analysis.setOrderByExpressions(node, emptyList()); + if (node.getSelect().isDistinct()) { + verifySelectDistinct(node, outputExpressions); + } + + OrderBy orderBy = node.getOrderBy().get(); + orderByScope = Optional.of(computeAndAssignOrderByScope(orderBy, sourceScope, outputScope)); + + orderByExpressions = analyzeOrderBy(node, orderBy.getSortItems(), orderByScope.get()); + + if (sourceScope.getOuterQueryParent().isPresent() && !node.getLimit().isPresent()) { + // not the root scope and ORDER BY is ineffective + analysis.markRedundantOrderBy(orderBy); + warningCollector.add(new PrestoWarning(REDUNDANT_ORDER_BY, "ORDER BY in subquery may have no effect")); + } } + analysis.setOrderByExpressions(node, orderByExpressions); List sourceExpressions = new ArrayList<>(outputExpressions); node.getHaving().ifPresent(sourceExpressions::add); @@ -2178,27 +2193,6 @@ private Scope analyzeWith(Query node, Optional scope) return withScope; } - private void analyzeOrderBy(Query node, Scope orderByScope) - { - checkState(node.getOrderBy().isPresent(), "orderBy is absent"); - - List sortItems = getSortItemsFromOrderBy(node.getOrderBy()); - analyzeOrderBy(node, sortItems, orderByScope); - } - - private List analyzeOrderBy(QuerySpecification node, Scope orderByScope, List outputExpressions) - { - checkState(node.getOrderBy().isPresent(), "orderBy is absent"); - - List sortItems = getSortItemsFromOrderBy(node.getOrderBy()); - - if (node.getSelect().isDistinct()) { - verifySelectDistinct(node, outputExpressions); - } - - return analyzeOrderBy(node, sortItems, orderByScope); - } - private void verifySelectDistinct(QuerySpecification node, List outputExpressions) { for (SortItem item : node.getOrderBy().get().getSortItems()) { @@ -2250,7 +2244,6 @@ private List analyzeOrderBy(Node node, List sortItems, Sco } List orderByFields = orderByFieldsBuilder.build(); - analysis.setOrderByExpressions(node, orderByFields); return orderByFields; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index f7d4e66159a64..f6d3d7ed65ed8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -35,8 +35,10 @@ import com.facebook.presto.sql.planner.iterative.rule.DetermineJoinDistributionType; import com.facebook.presto.sql.planner.iterative.rule.DetermineSemiJoinDistributionType; import com.facebook.presto.sql.planner.iterative.rule.EliminateCrossJoins; +import com.facebook.presto.sql.planner.iterative.rule.EvaluateZeroDistinctLimit; import com.facebook.presto.sql.planner.iterative.rule.EvaluateZeroLimit; import com.facebook.presto.sql.planner.iterative.rule.EvaluateZeroSample; +import com.facebook.presto.sql.planner.iterative.rule.EvaluateZeroTopN; import com.facebook.presto.sql.planner.iterative.rule.ExtractSpatialJoins; import com.facebook.presto.sql.planner.iterative.rule.GatherAndMergeWindows; import com.facebook.presto.sql.planner.iterative.rule.ImplementBernoulliSampleAsFilter; @@ -87,7 +89,11 @@ import com.facebook.presto.sql.planner.iterative.rule.PushTopNThroughUnion; import com.facebook.presto.sql.planner.iterative.rule.RemoveEmptyDelete; import com.facebook.presto.sql.planner.iterative.rule.RemoveFullSample; +import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantDistinctLimit; import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantIdentityProjections; +import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantLimit; +import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantSort; +import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantTopN; import com.facebook.presto.sql.planner.iterative.rule.RemoveTrivialFilters; import com.facebook.presto.sql.planner.iterative.rule.RemoveUnreferencedScalarApplyNodes; import com.facebook.presto.sql.planner.iterative.rule.RemoveUnreferencedScalarLateralNodes; @@ -285,11 +291,6 @@ public PlanOptimizers( statsCalculator, estimatedExchangesCostCalculator, new CanonicalizeExpressions().rules()), - new IterativeOptimizer( - ruleStats, - statsCalculator, - estimatedExchangesCostCalculator, - ImmutableSet.of(new EvaluateZeroLimit())), new IterativeOptimizer( ruleStats, statsCalculator, @@ -301,6 +302,9 @@ public PlanOptimizers( new RemoveRedundantIdentityProjections(), new RemoveFullSample(), new EvaluateZeroSample(), + new EvaluateZeroLimit(), + new EvaluateZeroDistinctLimit(), + new EvaluateZeroTopN(), new PushLimitThroughProject(), new MergeLimits(), new MergeLimitWithSort(), @@ -310,6 +314,10 @@ public PlanOptimizers( new PushLimitThroughSemiJoin(), new PushLimitThroughUnion(), new RemoveTrivialFilters(), + new RemoveRedundantLimit(), + new RemoveRedundantSort(), + new RemoveRedundantTopN(), + new RemoveRedundantDistinctLimit(), new ImplementFilteredAggregations(), new SingleDistinctAggregationToGroupBy(), new MultipleDistinctAggregationToMarkDistinct(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java index cca55ff631f82..35cf93f236560 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java @@ -79,6 +79,7 @@ import java.util.Set; import java.util.stream.IntStream; +import static com.facebook.presto.SystemSessionProperties.isSkipRedundantSort; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; import static com.facebook.presto.spi.plan.AggregationNode.groupingSets; @@ -895,7 +896,7 @@ private PlanBuilder sort(PlanBuilder subPlan, QuerySpecification node) private PlanBuilder sort(PlanBuilder subPlan, Optional orderBy, List orderByExpressions) { - if (!orderBy.isPresent()) { + if (!orderBy.isPresent() || (isSkipRedundantSort(session)) && analysis.isOrderByRedundant(orderBy.get())) { return subPlan; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroDistinctLimit.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroDistinctLimit.java new file mode 100644 index 0000000000000..7ccd6f7f1b11d --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroDistinctLimit.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.google.common.collect.ImmutableList; + +import static com.facebook.presto.sql.planner.plan.Patterns.DistinctLimit.count; +import static com.facebook.presto.sql.planner.plan.Patterns.distinctLimit; + +public class EvaluateZeroDistinctLimit + implements Rule +{ + private static final Pattern PATTERN = distinctLimit() + .with(count().equalTo(0L)); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(DistinctLimitNode node, Captures captures, Context context) + { + return Result.ofPlanNode(new ValuesNode(node.getId(), node.getOutputVariables(), ImmutableList.of())); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroTopN.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroTopN.java new file mode 100644 index 0000000000000..5e5c84995e4f5 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroTopN.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.google.common.collect.ImmutableList; + +import static com.facebook.presto.sql.planner.plan.Patterns.TopN.count; +import static com.facebook.presto.sql.planner.plan.Patterns.topN; + +public class EvaluateZeroTopN + implements Rule +{ + private static final Pattern PATTERN = topN() + .with(count().equalTo(0L)); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(TopNNode node, Captures captures, Context context) + { + return Result.ofPlanNode(new ValuesNode(node.getId(), node.getOutputVariables(), ImmutableList.of())); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantDistinctLimit.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantDistinctLimit.java new file mode 100644 index 0000000000000..76a71b4c09cbb --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantDistinctLimit.java @@ -0,0 +1,71 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.Optional; + +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; +import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isAtMost; +import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isScalar; +import static com.facebook.presto.sql.planner.plan.Patterns.distinctLimit; +import static com.google.common.base.Preconditions.checkArgument; + +/** + * Replace DistinctLimit node + * 1. With a empty ValuesNode when count is 0 + * 2. With a Distinct node when the subplan is guaranteed to produce fewer rows than count + * 3. With its source when the subplan produces only one row + */ +public class RemoveRedundantDistinctLimit + implements Rule +{ + private static final Pattern PATTERN = distinctLimit(); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(DistinctLimitNode node, Captures captures, Context context) + { + checkArgument(!node.getHashVariable().isPresent(), "HashSymbol should be empty"); + + if (isScalar(node.getSource(), context.getLookup())) { + return Result.ofPlanNode(node.getSource()); + } + if (isAtMost(node.getSource(), context.getLookup(), node.getLimit())) { + return Result.ofPlanNode(new AggregationNode( + node.getId(), + node.getSource(), + ImmutableMap.of(), + singleGroupingSet(node.getDistinctVariables()), + ImmutableList.of(), + SINGLE, + node.getHashVariable(), + Optional.empty())); + } + return Result.empty(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantLimit.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantLimit.java new file mode 100644 index 0000000000000..b62a97f3544ca --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantLimit.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.sql.planner.iterative.Rule; + +import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isAtMost; +import static com.facebook.presto.sql.planner.plan.Patterns.limit; + +/** + * Remove Limit node when the subplan is guaranteed to produce fewer rows than the limit and + * replace the plan with empty values if the limit count is 0. + */ +public class RemoveRedundantLimit + implements Rule +{ + // Applies to both LimitNode with ties and LimitNode without ties. + private static final Pattern PATTERN = limit(); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(LimitNode limit, Captures captures, Context context) + { + if (isAtMost(limit.getSource(), context.getLookup(), limit.getCount())) { + return Result.ofPlanNode(limit.getSource()); + } + return Result.empty(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantSort.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantSort.java new file mode 100644 index 0000000000000..52ec593fb6a27 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantSort.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.planner.plan.SortNode; + +import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isAtMostScalar; +import static com.facebook.presto.sql.planner.plan.Patterns.sort; + +public class RemoveRedundantSort + implements Rule +{ + private static final Pattern PATTERN = sort(); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(SortNode node, Captures captures, Context context) + { + if (isAtMostScalar(node.getSource(), context.getLookup())) { + return Result.ofPlanNode(node.getSource()); + } + return Result.empty(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantTopN.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantTopN.java new file mode 100644 index 0000000000000..f0bcd203d214a --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantTopN.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.planner.plan.SortNode; + +import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isAtMost; +import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isAtMostScalar; +import static com.facebook.presto.sql.planner.plan.Patterns.topN; + +/** + * Replace TopN node + * 1. With its source when the subplan is at most one row + * 2. With a Sort node when the subplan is guaranteed to produce fewer rows than N + */ +public class RemoveRedundantTopN + implements Rule +{ + private static final Pattern PATTERN = topN(); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(TopNNode node, Captures captures, Context context) + { + if (isAtMostScalar(node.getSource(), context.getLookup())) { + return Result.ofPlanNode(node.getSource()); + } + if (isAtMost(node.getSource(), context.getLookup(), node.getCount())) { + return Result.ofPlanNode(new SortNode(context.getIdAllocator().getNextId(), node.getSource(), node.getOrderingScheme(), false)); + } + return Result.empty(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/QueryCardinalityUtil.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/QueryCardinalityUtil.java index b1f03a51cdcf7..843c8b4aac118 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/QueryCardinalityUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/QueryCardinalityUtil.java @@ -24,6 +24,7 @@ import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; +import com.facebook.presto.sql.planner.plan.SampleNode; import com.google.common.collect.Range; import static com.facebook.presto.sql.planner.iterative.Lookup.noLookup; @@ -118,6 +119,15 @@ public Range visitExchange(ExchangeNode node, Void context) return Range.atLeast(0L); } + @Override + public Range visitSample(SampleNode node, Void context) + { + if (node.getSampleRatio() == 0.0) { + return Range.atMost(0L); + } + return Range.atLeast(0L); + } + @Override public Range visitProject(ProjectNode node, Void context) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java index a68b59d7fc2d4..175e659f205ce 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java @@ -16,6 +16,7 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.matching.Property; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.IntersectNode; import com.facebook.presto.spi.plan.LimitNode; @@ -160,6 +161,11 @@ public static Pattern union() return typeOf(UnionNode.class); } + public static Pattern distinctLimit() + { + return typeOf(DistinctLimitNode.class); + } + public static Pattern intersect() { return typeOf(IntersectNode.class); @@ -197,6 +203,14 @@ public static Property> sources() return property("sources", PlanNode::getSources); } + public static class DistinctLimit + { + public static Property count() + { + return property("limit", DistinctLimitNode::getLimit); + } + } + public static class Aggregation { public static Property> groupingColumns() @@ -266,6 +280,11 @@ public static Property step() { return property("step", TopNNode::getStep); } + + public static Property count() + { + return property("count", TopNNode::getCount); + } } public static class Values diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index 207caa9aeb721..323a78941ed67 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -148,6 +148,7 @@ public void testDefaults() .setOptimizeCommonSubExpressions(true) .setPreferDistributedUnion(true) .setOptimizeNullsInJoin(false) + .setSkipRedundantSort(true) .setWarnOnNoTableLayoutFilter("") .setInlineSqlFunctions(true) .setCheckAccessControlOnUtilizedColumnsOnly(false)); @@ -254,6 +255,7 @@ public void testExplicitPropertyMappings() .put("warn-on-no-table-layout-filter", "ry@nlikestheyankees,ds") .put("inline-sql-functions", "false") .put("check-access-control-on-utilized-columns-only", "true") + .put("optimizer.skip-redundant-sort", "false") .build(); FeaturesConfig expected = new FeaturesConfig() @@ -351,9 +353,11 @@ public void testExplicitPropertyMappings() .setOptimizeCommonSubExpressions(false) .setPreferDistributedUnion(false) .setOptimizeNullsInJoin(true) + .setSkipRedundantSort(false) .setWarnOnNoTableLayoutFilter("ry@nlikestheyankees,ds") .setInlineSqlFunctions(false) - .setCheckAccessControlOnUtilizedColumnsOnly(true); + .setCheckAccessControlOnUtilizedColumnsOnly(true) + .setSkipRedundantSort(false); assertFullMapping(properties, expected); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java index 7e0554d3062fd..afba4fefeecc0 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java @@ -17,8 +17,11 @@ import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType; @@ -33,6 +36,7 @@ import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.tree.LongLiteral; import com.facebook.presto.tests.QueryTemplate; @@ -59,6 +63,7 @@ import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.sql.planner.LogicalPlanner.Stage.OPTIMIZED; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.any; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyNot; @@ -73,6 +78,7 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.functionCall; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.limit; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.markDistinct; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.output; @@ -582,7 +588,7 @@ public void testJoinOutputPruning() private void assertPlanContainsNoApplyOrAnyJoin(String sql) { assertFalse( - searchFrom(plan(sql, LogicalPlanner.Stage.OPTIMIZED).getRoot()) + searchFrom(plan(sql, OPTIMIZED).getRoot()) .where(isInstanceOfAny(ApplyNode.class, JoinNode.class, IndexJoinNode.class, SemiJoinNode.class, LateralJoinNode.class)) .matches(), "Unexpected node for query: " + sql); @@ -593,7 +599,7 @@ public void testCorrelatedSubqueries() { assertPlan( "SELECT orderkey FROM orders WHERE 3 = (SELECT orderkey)", - LogicalPlanner.Stage.OPTIMIZED, + OPTIMIZED, any( filter( "X = BIGINT '3'", @@ -751,7 +757,7 @@ public void testDoubleNestedCorrelatedSubqueries() assertPlan( "SELECT orderkey FROM orders o " + "WHERE 3 IN (SELECT o.custkey FROM lineitem l WHERE (SELECT l.orderkey = o.orderkey))", - LogicalPlanner.Stage.OPTIMIZED, + OPTIMIZED, anyTree( filter("OUTER_FILTER", apply(ImmutableList.of("C", "O"), @@ -1075,14 +1081,7 @@ public void testComplexOrderBy() " SUM(REDUCE(col1, ROW(0),(l, r) -> l, x -> 1)) " + " )", output( - project( - exchange( - exchange( - sort( - exchange( - project( - aggregation(ImmutableMap.of(), - project(values("col1"))))))))))); + (values("col1")))); } @Test @@ -1143,4 +1142,102 @@ public void testJoinNullFilters() ImmutableMap.of( "REGION_REGIONKEY", "regionkey")))))); } + + @Test + public void testRedundantLimitNodeRemoval() + { + String query = "SELECT count(*) FROM orders LIMIT 10"; + assertFalse( + searchFrom(plan(query, OPTIMIZED).getRoot()) + .where(LimitNode.class::isInstance) + .matches(), + format("Unexpected limit node for query: '%s'", query)); + + assertPlan( + "SELECT orderkey, count(*) FROM orders GROUP BY orderkey LIMIT 10", + output( + limit(10, + anyTree( + tableScan("orders"))))); + + assertPlan( + "SELECT * FROM (VALUES 1,2,3,4,5,6) AS t1 LIMIT 10", + output( + values(ImmutableList.of("x")))); + } + + @Test + public void testRemoveSingleRowSort() + { + String query = "SELECT count(*) FROM orders ORDER BY 1"; + assertFalse( + searchFrom(plan(query, OPTIMIZED).getRoot()) + .where(isInstanceOfAny(SortNode.class)) + .matches(), + format("Unexpected sort node for query: '%s'", query)); + + assertPlan( + "SELECT orderkey, count(*) FROM orders GROUP BY orderkey ORDER BY 1", + anyTree( + node(SortNode.class, + anyTree( + tableScan("orders"))))); + } + + @Test + public void testRedundantTopNNodeRemoval() + { + String query = "SELECT count(*) FROM orders ORDER BY 1 LIMIT 10"; + assertFalse( + searchFrom(plan(query, OPTIMIZED).getRoot()) + .where(isInstanceOfAny(TopNNode.class, SortNode.class)) + .matches(), + format("Unexpected TopN node for query: '%s'", query)); + + assertPlan( + "SELECT orderkey, count(*) FROM orders GROUP BY orderkey ORDER BY 1 LIMIT 10", + output( + node(TopNNode.class, + anyTree( + tableScan("orders"))))); + + assertPlan( + "SELECT orderkey, count(*) FROM orders GROUP BY orderkey ORDER BY 1 LIMIT 0", + output( + node(ValuesNode.class))); + + query = "SELECT * FROM (VALUES 1,2,3,4,5,6) AS t1 ORDER BY 1 LIMIT 10"; + assertPlan( + query, + output( + node(TopNNode.class, + node(TopNNode.class, + node(ValuesNode.class))))); + } + + @Test + public void testRedundantDistinctLimitNodeRemoval() + { + String query = "SELECT distinct(c) FROM (SELECT count(*) as c FROM orders) LIMIT 10"; + assertFalse( + searchFrom(plan(query, OPTIMIZED).getRoot()) + .where(isInstanceOfAny(DistinctLimitNode.class)) + .matches(), + format("Unexpected DistinctLimit node for query: '%s'", query)); + + assertPlan( + "SELECT distinct(c) FROM (SELECT count(*) as c FROM orders GROUP BY orderkey) LIMIT 10", + output( + node(DistinctLimitNode.class, + anyTree( + tableScan("orders"))))); + + assertPlan( + "SELECT distinct(id) FROM (VALUES 1, 2, 3, 4, 5, 6) as t1 (id) LIMIT 10", + output( + node(ProjectNode.class, + node(AggregationNode.class, + node(ProjectNode.class, + values(ImmutableList.of("x"))))))); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestOrderBy.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestOrderBy.java new file mode 100644 index 0000000000000..31b2c98155a21 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestOrderBy.java @@ -0,0 +1,97 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner; + +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.sql.planner.assertions.BasePlanTest; +import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; +import com.facebook.presto.sql.planner.plan.ExchangeNode; +import com.facebook.presto.sql.planner.plan.SortNode; +import org.testng.annotations.Test; + +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.output; + +public class TestOrderBy + extends BasePlanTest +{ + @Test + public void testRedundantOrderByInSubquery() + { + assertPlan("SELECT * FROM (SELECT * FROM (VALUES 1, 2, 3) t(x) ORDER BY x)", + output( + node(ValuesNode.class))); + } + + @Test + public void testRequiredOrderByInSubquery() + { + assertPlan("SELECT * FROM (SELECT * FROM (VALUES 1, 2, 3) t(x) ORDER BY x LIMIT 1)", + output( + node(TopNNode.class, + anyTree( + node(ValuesNode.class))))); + } + + @Test + public void testRedundantOrderByInScalarSubquery() + { + assertPlan("SELECT (SELECT * FROM (VALUES 1, 2, 3) t(x) ORDER BY x) FROM (VALUES 10)", + output( + node(EnforceSingleRowNode.class, + node(ValuesNode.class)))); + } + + @Test + public void testRequiredOrderByInScalarSubquery() + { + assertPlan("SELECT (SELECT * FROM (VALUES 1, 2, 3) t(x) ORDER BY x LIMIT 1) FROM (VALUES 10)", + output( + anyTree( + node(TopNNode.class, + node(ValuesNode.class))))); + } + + @Test + public void testRequiredOrderByInUnion() + { + assertPlan("VALUES 1 " + + "UNION ALL " + + "VALUES 2 " + + "ORDER BY 1 ", + output( + anyTree( + node(SortNode.class, + node(ExchangeNode.class, + node(ValuesNode.class), + node(ValuesNode.class)))))); + } + + @Test + public void testRedundantOrderByInUnion() + { + assertPlan("SELECT * FROM (" + + " VALUES 1 " + + " UNION ALL " + + " VALUES 2 " + + " ORDER BY 1 " + + ")", + output( + node(ExchangeNode.class, + node(ValuesNode.class), + node(ValuesNode.class)))); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantDistinctLimit.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantDistinctLimit.java new file mode 100644 index 0000000000000..2ba5e968e1df9 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantDistinctLimit.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; + +public class TestRemoveRedundantDistinctLimit + extends BaseRuleTest +{ + @Test + public void test() + { + tester().assertThat(new RemoveRedundantDistinctLimit()) + .on(p -> + p.distinctLimit( + 10, + ImmutableList.of(p.variable("c")), + p.values(1, p.variable("c")))) + .matches(node(ValuesNode.class)); + + tester().assertThat(new RemoveRedundantDistinctLimit()) + .on(p -> + p.distinctLimit( + 10, + ImmutableList.of(p.variable("c")), + p.values(6, p.variable("c")))) + .matches( + node(AggregationNode.class, + node(ValuesNode.class))); + + tester().assertThat(new RemoveRedundantDistinctLimit()) + .on(p -> + p.distinctLimit( + 0, + ImmutableList.of(p.variable("c")), + p.values(1, p.variable("c")))) + .matches(node(ValuesNode.class)); + } + + @Test + public void doesNotFire() + { + tester().assertThat(new RemoveRedundantDistinctLimit()) + .on(p -> + p.distinctLimit( + 10, + ImmutableList.of(p.variable("c")), + p.values(100, p.variable("c")))) + .doesNotFire(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantLimit.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantLimit.java new file mode 100644 index 0000000000000..f019edad11017 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantLimit.java @@ -0,0 +1,92 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; + +public class TestRemoveRedundantLimit + extends BaseRuleTest +{ + @Test + public void test() + { + tester().assertThat(new RemoveRedundantLimit()) + .on(p -> + p.limit( + 10, + p.aggregation(builder -> builder + .addAggregation(p.variable("c"), expression("count(foo)"), ImmutableList.of(BIGINT)) + .globalGrouping() + .source(p.values(p.variable("foo")))))) + .matches( + node(AggregationNode.class, + node(ValuesNode.class))); + } + + @Test + public void testRemoveLimitWithTies() + { + tester().assertThat(new RemoveRedundantLimit()) + .on(p -> { + VariableReferenceExpression c = p.variable("c"); + return p.limit( + 10, + p.values(5, c)); + }) + .matches(values("c")); + } + + @Test + public void testForZeroLimit() + { + tester().assertThat(new EvaluateZeroLimit()) + .on(p -> + p.limit( + 0, + p.filter( + expression("b > 5"), + p.values( + ImmutableList.of(p.variable("a"), p.variable("b")), + ImmutableList.of( + constantExpressions(BIGINT, 1L, 10L), + constantExpressions(BIGINT, 2L, 11L)))))) + .matches(values(ImmutableMap.of())); + } + + @Test + public void doesNotFire() + { + tester().assertThat(new EvaluateZeroLimit()) + .on(p -> + p.limit( + 10, + p.aggregation(builder -> builder + .addAggregation(p.variable("c"), expression("count(foo)"), ImmutableList.of(BIGINT)) + .singleGroupingSet(p.variable("foo")) + .source(p.values(20, p.variable("foo")))))) + .doesNotFire(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantSort.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantSort.java new file mode 100644 index 0000000000000..081d38bf3b0f8 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantSort.java @@ -0,0 +1,69 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; + +public class TestRemoveRedundantSort + extends BaseRuleTest +{ + @Test + public void test() + { + tester().assertThat(new RemoveRedundantSort()) + .on(p -> + p.sort( + ImmutableList.of(p.variable("c")), + p.aggregation(builder -> builder + .addAggregation(p.variable("c"), expression("count(foo)"), ImmutableList.of(BIGINT)) + .globalGrouping() + .source(p.values(p.variable("foo")))))) + .matches( + node(AggregationNode.class, + node(ValuesNode.class))); + } + + @Test + public void testForZeroCardinality() + { + tester().assertThat(new RemoveRedundantSort()) + .on(p -> + p.sort( + ImmutableList.of(p.variable("c")), + p.values(p.variable("foo")))) + .matches(node(ValuesNode.class)); + } + + @Test + public void doesNotFire() + { + tester().assertThat(new RemoveRedundantSort()) + .on(p -> + p.sort( + ImmutableList.of(p.variable("c")), + p.aggregation(builder -> builder + .addAggregation(p.variable("c"), expression("count(foo)"), ImmutableList.of(BIGINT)) + .singleGroupingSet(p.variable("foo")) + .source(p.values(20, p.variable("foo")))))) + .doesNotFire(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantTopN.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantTopN.java new file mode 100644 index 0000000000000..df3622517de1a --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantTopN.java @@ -0,0 +1,100 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.planner.plan.SortNode; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; + +public class TestRemoveRedundantTopN + extends BaseRuleTest +{ + @Test + public void test() + { + tester().assertThat(new RemoveRedundantTopN()) + .on(p -> + p.topN( + 10, + ImmutableList.of(p.variable("c")), + p.aggregation(builder -> builder + .addAggregation(p.variable("c"), expression("count(foo)"), ImmutableList.of(BIGINT)) + .globalGrouping() + .source(p.values(p.variable("foo")))))) + .matches( + node(AggregationNode.class, + node(ValuesNode.class))); + + tester().assertThat(new RemoveRedundantTopN()) + .on(p -> + p.topN( + 10, + ImmutableList.of(p.variable("a")), + p.filter( + expression("b > 5"), + p.values( + ImmutableList.of(p.variable("a"), p.variable("b")), + ImmutableList.of( + constantExpressions(BIGINT, 1L, 10L), + constantExpressions(BIGINT, 2L, 11L)))))) + .matches( + node(SortNode.class, + node(FilterNode.class, + node(ValuesNode.class)))); + } + + @Test + public void testZeroTopN() + { + tester().assertThat(new EvaluateZeroTopN()) + .on(p -> + p.topN( + 0, + ImmutableList.of(p.variable("a")), + p.filter( + expression("b > 5"), + p.values( + ImmutableList.of(p.variable("a"), p.variable("b")), + ImmutableList.of( + constantExpressions(BIGINT, 1L, 10L), + constantExpressions(BIGINT, 2L, 11L)))))) + .matches(values(ImmutableMap.of())); + } + + @Test + public void doesNotFire() + { + tester().assertThat(new RemoveRedundantTopN()) + .on(p -> + p.topN( + 10, + ImmutableList.of(p.variable("c")), + p.aggregation(builder -> builder + .addAggregation(p.variable("c"), expression("count(foo)"), ImmutableList.of(BIGINT)) + .singleGroupingSet(p.variable("foo")) + .source(p.values(20, p.variable("foo")))))) + .doesNotFire(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java index 9f6111e6cc470..20b69c55f316f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java @@ -14,7 +14,6 @@ package com.facebook.presto.sql.planner.iterative.rule.test; import com.facebook.presto.Session; -import com.facebook.presto.common.block.SortOrder; import com.facebook.presto.common.function.OperatorType; import com.facebook.presto.common.predicate.TupleDomain; import com.facebook.presto.common.type.Type; @@ -30,6 +29,7 @@ import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.IntersectNode; import com.facebook.presto.spi.plan.LimitNode; @@ -70,6 +70,7 @@ import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.UnnestNode; @@ -97,6 +98,7 @@ import java.util.function.Consumer; import java.util.stream.Stream; +import static com.facebook.presto.common.block.SortOrder.ASC_NULLS_FIRST; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; import static com.facebook.presto.metadata.FunctionAndTypeManager.qualifyObjectName; @@ -238,6 +240,16 @@ public EnforceSingleRowNode enforceSingleRow(PlanNode source) return new EnforceSingleRowNode(idAllocator.getNextId(), source); } + public SortNode sort(List orderBy, PlanNode source) + { + ImmutableList ordering = orderBy.stream().map(variable -> new Ordering(variable, ASC_NULLS_FIRST)).collect(toImmutableList()); + return new SortNode( + idAllocator.getNextId(), + source, + new OrderingScheme(ordering), + false); + } + public LimitNode limit(long limit, PlanNode source) { return new LimitNode(idAllocator.getNextId(), source, limit, FINAL); @@ -249,7 +261,7 @@ public TopNNode topN(long count, List orderBy, Plan idAllocator.getNextId(), source, count, - new OrderingScheme(orderBy.stream().map(variable -> new Ordering(variable, SortOrder.ASC_NULLS_FIRST)).collect(toImmutableList())), + new OrderingScheme(orderBy.stream().map(variable -> new Ordering(variable, ASC_NULLS_FIRST)).collect(toImmutableList())), TopNNode.Step.SINGLE); } @@ -302,6 +314,17 @@ public CallExpression comparison(OperatorType operatorType, RowExpression left, return call(operatorType.getOperator(), functionHandle, left.getType(), left, right); } + public DistinctLimitNode distinctLimit(long count, List distinctSymbols, PlanNode source) + { + return new DistinctLimitNode( + idAllocator.getNextId(), + source, + count, + false, + distinctSymbols, + Optional.empty()); + } + public class AggregationBuilder { private final TypeProvider types; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java b/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java index d19f642063fdd..f92ca4791edbb 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java @@ -90,7 +90,7 @@ public void testUnsupportedSubqueriesWithCoercions() { // coercion from subquery symbol type to correlation type assertions.assertFails( - "select (select count(*) from (values 1) t(a) where t.a=t2.b limit 1) from (values 1.0) t2(b)", + "select (select count(*) from (values 1) t(a) where t.a=t2.b group by a limit 1) from (values 1.0) t2(b)", UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); // coercion from t.a (null) to integer assertions.assertFails( @@ -101,12 +101,19 @@ public void testUnsupportedSubqueriesWithCoercions() @Test public void testCorrelatedSubqueriesWithLimit() { + // coercion from subquery symbol type to correlation type + assertions.assertQuery( + "select (select count(*) from (values 1) t(a) where t.a=t2.b limit 1) from (values 1.0) t2(b)", + "VALUES BIGINT '1'"); assertions.assertQuery( "select (select t.a from (values 1, 2) t(a) where t.a=t2.b limit 1) from (values 1) t2(b)", "VALUES 1"); + assertions.assertQuery( + "SELECT (SELECT t.a FROM (VALUES 1, 2) t(a) WHERE t.a=t2.b LIMIT 2) FROM (VALUES 1) t2(b)", + "VALUES 1"); // cannot enforce limit 2 on correlated subquery assertions.assertFails( - "select (select t.a from (values 1, 2) t(a) where t.a=t2.b limit 2) from (values 1) t2(b)", + "select (select t.a from (values 1, 2, 3) t(a) where t.a=t2.b limit 2) from (values 1) t2(b)", UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); assertions.assertQuery( "select (select sum(t.a) from (values 1, 2) t(a) where t.a=t2.b group by t.a limit 2) from (values 1) t2(b)", @@ -118,10 +125,12 @@ public void testCorrelatedSubqueriesWithLimit() "select EXISTS(select 1 from (values 1, 1, 3) t(a) where t.a=t2.b limit 1) from (values 1, 2) t2(b)", "VALUES true, false", false); - // TransformCorrelatedScalarAggregationToJoin does not fire since limit is above aggregation node assertions.assertFails( - "select (select count(*) from (values 1, 1, 3) t(a) where t.a=t2.b limit 1) from (values 1) t2(b)", + "select (select count(*) from (values 1, 1, 3) t(a) where t.a=t2.b group by a limit 1) from (values 1.0) t2(b)", UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); + assertions.assertQuery( + "SELECT (SELECT count(*) FROM (VALUES 1, 1, 3) t(a) WHERE t.a=t2.b LIMIT 1) FROM (VALUES 1) t2(b)", + "VALUES BIGINT '2'"); assertExistsRewrittenToAggregationBelowJoin( "SELECT EXISTS(SELECT 1 FROM (values ('x', 1)) u(x, cid) WHERE x = 'x' AND t.cid = cid LIMIT 1) " + "FROM (values 1) t(cid)", diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/StandardWarningCode.java b/presto-spi/src/main/java/com/facebook/presto/spi/StandardWarningCode.java index 7d1e923726ce8..dd5440564355d 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/StandardWarningCode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/StandardWarningCode.java @@ -19,7 +19,8 @@ public enum StandardWarningCode TOO_MANY_STAGES(0x0000_0001), PARSER_WARNING(0x0000_0002), PERFORMANCE_WARNING(0x0000_0003), - SEMANTIC_WARNING(0x0000_0004) + SEMANTIC_WARNING(0x0000_0004), + REDUNDANT_ORDER_BY(0x0000_0005) /**/; private final WarningCode warningCode; diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestOrderByQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestOrderByQueries.java index 2040fe550b325..738ee418d1340 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestOrderByQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestOrderByQueries.java @@ -80,7 +80,6 @@ public void testOrderByWithOutputColumnReference() assertQueryOrdered("SELECT max(a) FROM (values (1,2), (2,1)) t(a,b) GROUP BY b ORDER BY b", "VALUES 2, 1"); assertQueryOrdered("SELECT max(a) FROM (values (1,2), (2,1)) t(a,b) GROUP BY t.b ORDER BY t.b*1.0", "VALUES 2, 1"); assertQueryOrdered("SELECT -(a+b) AS a, -(a+b) AS b, a+b FROM (values (41, 42), (-41, -42)) t(a,b) GROUP BY a+b ORDER BY a+b", "VALUES (-83, -83, 83), (83, 83, -83)"); - assertQueryOrdered("SELECT c.a FROM (SELECT CAST(ROW(-a.a) AS ROW(a BIGINT)) a FROM (VALUES (2), (1)) a(a) GROUP BY a.a ORDER BY a.a) t(c)", "VALUES -2, -1"); assertQueryOrdered("SELECT -a AS a FROM (values (1,2),(3,2)) t(a,b) GROUP BY GROUPING SETS ((a), (a, b)) ORDER BY -a", "VALUES -1, -1, -3, -3"); assertQueryOrdered("SELECT a AS foo FROM (values (1,2),(3,2)) t(a,b) GROUP BY GROUPING SETS ((a), (a, b)) HAVING b IS NOT NULL ORDER BY -a", "VALUES 3, 1"); assertQueryOrdered("SELECT max(a) FROM (values (1,2),(3,2)) t(a,b) ORDER BY max(-a)", "VALUES 3"); @@ -244,7 +243,7 @@ public void testUndistributedOrderBy() @Test public void testOrderLimitCompaction() { - assertQueryOrdered("SELECT * FROM (SELECT * FROM orders ORDER BY orderkey) LIMIT 10"); + assertQueryOrdered("SELECT * FROM (SELECT * FROM orders ORDER BY orderkey LIMIT 10)"); } @Test diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java index c8416839d8aad..33cf242ec8b4e 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java @@ -3389,9 +3389,12 @@ public void testCorrelatedScalarSubqueries() // two level of nesting assertQuery("SELECT * FROM nation n WHERE 2 = (SELECT (SELECT 2 * n.nationkey))"); + // redundant LIMIT in subquery + assertQuery("SELECT (SELECT count(*) FROM (VALUES (7,1)) t(orderkey, value) WHERE orderkey = corr_key LIMIT 1) FROM (values 7) t(corr_key)"); + // explicit LIMIT in subquery assertQueryFails( - "SELECT (SELECT count(*) FROM (VALUES (7,1)) t(orderkey, value) WHERE orderkey = corr_key LIMIT 1) FROM (values 7) t(corr_key)", + "SELECT (SELECT count(*) FROM (VALUES (7,1)) t(orderkey, value) WHERE orderkey = corr_key GROUP BY value LIMIT 2) FROM (values 7) t(corr_key)", "line 1:9: Given correlated subquery is not supported"); }