Skip to content

Commit 6a4e483

Browse files
krvikashebyhr
krvikash
authored and
ebyhr
committed
Add projection pushdown tests in BaseConnectorTest
1 parent 9dc8375 commit 6a4e483

File tree

14 files changed

+261
-266
lines changed

14 files changed

+261
-266
lines changed

‎plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryConnectorTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior)
8787
return true;
8888

8989
case SUPPORTS_TOPN_PUSHDOWN:
90+
case SUPPORTS_DEREFERENCE_PUSHDOWN:
9091
return false;
9192

9293
case SUPPORTS_RENAME_SCHEMA:

‎plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConnectorTest.java‎

Lines changed: 0 additions & 186 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@
5656
import static io.trino.spi.type.VarcharType.VARCHAR;
5757
import static io.trino.testing.DataProviders.toDataProvider;
5858
import static io.trino.testing.MaterializedResult.resultBuilder;
59-
import static io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder;
6059
import static io.trino.testing.QueryAssertions.copyTpchTables;
6160
import static io.trino.testing.TestingAccessControlManager.TestingPrivilegeType.EXECUTE_FUNCTION;
6261
import static io.trino.testing.TestingAccessControlManager.TestingPrivilegeType.SELECT_COLUMN;
@@ -952,19 +951,6 @@ public void testSettingChangeDataFeedEnabledProperty()
952951
.contains("change_data_feed_enabled = true");
953952
}
954953

955-
@Test
956-
public void testProjectionPushdown()
957-
{
958-
String tableName = "test_projection_pushdown_" + randomNameSuffix();
959-
960-
assertUpdate("CREATE TABLE " + tableName + " (id BIGINT, root ROW(f1 BIGINT, f2 BIGINT))");
961-
assertUpdate("INSERT INTO " + tableName + " VALUES (1, ROW(1, 2)), (1, NULl), (1, ROW(NULL, 4))", 3);
962-
963-
assertQuery("SELECT root.f1, id FROM " + tableName, "VALUES (1, 1), (NULL, 1), (NULL, 1)");
964-
965-
assertUpdate("DROP TABLE " + tableName);
966-
}
967-
968954
@Test
969955
public void testProjectionPushdownOnPartitionedTables()
970956
{
@@ -981,178 +967,6 @@ public void testProjectionPushdownOnPartitionedTables()
981967
assertUpdate("DROP TABLE " + tableNamePartitioningAtEnd);
982968
}
983969

984-
@Test
985-
public void testProjectionWithCaseSensitiveField()
986-
{
987-
// TODO consider moving this in BaseConnectorTest
988-
String tableName = "test_projection_with_case_sensitive_field_" + randomNameSuffix();
989-
990-
assertUpdate("CREATE TABLE " + tableName + " (id INT, a ROW(\"UPPER_CASE\" INT, \"lower_case\" INT, \"MiXeD_cAsE\" INT))");
991-
assertUpdate("INSERT INTO " + tableName + " VALUES (1, ROW(2, 3, 4)), (5, ROW(6, 7, 8))", 2);
992-
993-
String expected = "VALUES (2, 3, 4), (6, 7, 8)";
994-
assertQuery("SELECT a.UPPER_CASE, a.lower_case, a.MiXeD_cAsE FROM " + tableName, expected);
995-
assertQuery("SELECT a.upper_case, a.lower_case, a.mixed_case FROM " + tableName, expected);
996-
assertQuery("SELECT a.UPPER_CASE, a.LOWER_CASE, a.MIXED_CASE FROM " + tableName, expected);
997-
assertUpdate("DROP TABLE " + tableName);
998-
}
999-
1000-
@Test
1001-
public void testProjectionPushdownMultipleRows()
1002-
{
1003-
String tableName = "test_projection_pushdown_multiple_rows_" + randomNameSuffix();
1004-
1005-
assertUpdate("CREATE TABLE " + tableName +
1006-
" (id BIGINT, nested1 ROW(child1 BIGINT, child2 VARCHAR, child3 INT), nested2 ROW(child1 DOUBLE, child2 BOOLEAN, child3 DATE))");
1007-
assertUpdate("INSERT INTO " + tableName + " VALUES" +
1008-
" (1, ROW(10, 'a', 100), ROW(10.10, true, DATE '2023-04-19'))," +
1009-
" (2, ROW(20, 'b', 200), ROW(20.20, false, DATE '1990-04-20'))," +
1010-
" (4, ROW(40, NULL, 400), NULL)," +
1011-
" (5, NULL, ROW(NULL, true, NULL))",
1012-
4);
1013-
1014-
// Select one field from one row field
1015-
assertQuery("SELECT id, nested1.child1 FROM " + tableName, "VALUES (1, 10), (2, 20), (4, 40), (5, NULL)");
1016-
assertQuery("SELECT nested2.child3, id FROM " + tableName, "VALUES (DATE '2023-04-19', 1), (DATE '1990-04-20', 2), (NULL, 4), (NULL, 5)");
1017-
1018-
// Select one field each from multiple row fields
1019-
assertQuery("SELECT nested2.child1, id, nested1.child2 FROM " + tableName, "VALUES (10.10, 1, 'a'), (20.20, 2, 'b'), (NULL, 4, NULL), (NULL, 5, NULL)");
1020-
1021-
// Select multiple fields from one row field
1022-
assertQuery("SELECT nested1.child3, id, nested1.child2 FROM " + tableName, "VALUES (100, 1, 'a'), (200, 2, 'b'), (400, 4, NULL), (NULL, 5, NULL)");
1023-
assertQuery(
1024-
"SELECT nested2.child2, nested2.child3, id FROM " + tableName,
1025-
"VALUES (true, DATE '2023-04-19' , 1), (false, DATE '1990-04-20', 2), (NULL, NULL, 4), (true, NULL, 5)");
1026-
1027-
// Select multiple fields from multiple row fields
1028-
assertQuery(
1029-
"SELECT id, nested2.child1, nested1.child3, nested2.child2, nested1.child1 FROM " + tableName,
1030-
"VALUES (1, 10.10, 100, true, 10), (2, 20.20, 200, false, 20), (4, NULL, 400, NULL, 40), (5, NULL, NULL, true, NULL)");
1031-
1032-
// Select only nested fields
1033-
assertQuery("SELECT nested2.child2, nested1.child3 FROM " + tableName, "VALUES (true, 100), (false, 200), (NULL, 400), (true, NULL)");
1034-
1035-
assertUpdate("DROP TABLE " + tableName);
1036-
}
1037-
1038-
@Test
1039-
public void testReadHighlyNestedData()
1040-
{
1041-
// TODO consider moving this in BaseConnectorTest
1042-
String tableName = "test_highly_nested_data_" + randomNameSuffix();
1043-
1044-
assertUpdate("CREATE TABLE " + tableName + " (id INT, row1_t ROW(f1 INT, f2 INT, row2_t ROW (f1 INT, f2 INT, row3_t ROW(f1 INT, f2 INT))))");
1045-
assertUpdate("INSERT INTO " + tableName + " VALUES (1, ROW(2, 3, ROW(4, 5, ROW(6, 7)))), (11, ROW(12, 13, ROW(14, 15, ROW(16, 17))))", 2);
1046-
assertUpdate("INSERT INTO " + tableName + " VALUES (21, ROW(22, 23, ROW(24, 25, ROW(26, 27))))", 1);
1047-
1048-
// Test select projected columns, with and without their parent column
1049-
assertQuery("SELECT id, row1_t.row2_t.row3_t.f2 FROM " + tableName, "VALUES (1, 7), (11, 17), (21, 27)");
1050-
assertQuery("SELECT id, row1_t.row2_t.row3_t.f2, CAST(row1_t AS JSON) FROM " + tableName,
1051-
"VALUES (1, 7, '{\"f1\":2,\"f2\":3,\"row2_t\":{\"f1\":4,\"f2\":5,\"row3_t\":{\"f1\":6,\"f2\":7}}}'), " +
1052-
"(11, 17, '{\"f1\":12,\"f2\":13,\"row2_t\":{\"f1\":14,\"f2\":15,\"row3_t\":{\"f1\":16,\"f2\":17}}}'), " +
1053-
"(21, 27, '{\"f1\":22,\"f2\":23,\"row2_t\":{\"f1\":24,\"f2\":25,\"row3_t\":{\"f1\":26,\"f2\":27}}}')");
1054-
1055-
// Test predicates on immediate child column and deeper nested column
1056-
assertQuery("SELECT id, CAST(row1_t.row2_t.row3_t AS JSON) FROM " + tableName + " WHERE row1_t.row2_t.row3_t.f2 = 27", "VALUES (21, '{\"f1\":26,\"f2\":27}')");
1057-
assertQuery("SELECT id, CAST(row1_t.row2_t.row3_t AS JSON) FROM " + tableName + " WHERE row1_t.row2_t.row3_t.f2 > 20", "VALUES (21, '{\"f1\":26,\"f2\":27}')");
1058-
assertQuery("SELECT id, CAST(row1_t AS JSON) FROM " + tableName + " WHERE row1_t.row2_t.row3_t.f2 = 27",
1059-
"VALUES (21, '{\"f1\":22,\"f2\":23,\"row2_t\":{\"f1\":24,\"f2\":25,\"row3_t\":{\"f1\":26,\"f2\":27}}}')");
1060-
assertQuery("SELECT id, CAST(row1_t AS JSON) FROM " + tableName + " WHERE row1_t.row2_t.row3_t.f2 > 20",
1061-
"VALUES (21, '{\"f1\":22,\"f2\":23,\"row2_t\":{\"f1\":24,\"f2\":25,\"row3_t\":{\"f1\":26,\"f2\":27}}}')");
1062-
1063-
// Test predicates on parent columns
1064-
assertQuery("SELECT id, row1_t.row2_t.row3_t.f1 FROM " + tableName + " WHERE row1_t.row2_t.row3_t = ROW(16, 17)", "VALUES (11, 16)");
1065-
assertQuery("SELECT id, row1_t.row2_t.row3_t.f1 FROM " + tableName + " WHERE row1_t = ROW(22, 23, ROW(24, 25, ROW(26, 27)))", "VALUES (21, 26)");
1066-
1067-
// Explain highly nested select
1068-
assertExplain(
1069-
"EXPLAIN SELECT id, row1_t.row2_t.row3_t.f1, row1_t.row2_t.f1, row1_t.row2_t.row3_t.f2 FROM " + tableName + " WHERE row1_t.row2_t.row3_t.f2 = 27",
1070-
"ScanFilter\\[table = (.*), filterPredicate = \\(\"row1_t#row2_t#row3_t#f2\" = 27\\)]",
1071-
"id(.*) := id:integer:REGULAR",
1072-
"row1_t#row2_t#f1 := row1_t#row2_t#f1:integer:REGULAR",
1073-
"row1_t#row2_t#row3_t#f1 := row1_t#row2_t#row3_t#f1:integer:REGULAR",
1074-
"row1_t#row2_t#row3_t#f2 := row1_t#row2_t#row3_t#f2:integer:REGULAR");
1075-
1076-
assertUpdate("DROP TABLE " + tableName);
1077-
}
1078-
1079-
@Test
1080-
public void testProjectionPushdownReadsLessData()
1081-
{
1082-
// TODO consider moving this in BaseConnectorTest
1083-
String tableName = "test_projection_pushdown_reads_less_data_" + randomNameSuffix();
1084-
1085-
assertUpdate("CREATE TABLE " + tableName + " (id INT, root ROW(leaf1 BIGINT, leaf2 BIGINT))");
1086-
assertUpdate("INSERT INTO " + tableName + " SELECT val, ROW(val + 1, val + 2) FROM UNNEST(SEQUENCE(1, 10)) AS t(val)", 10);
1087-
1088-
MaterializedResult expectedResult = computeActual("SELECT val + 2 FROM UNNEST(SEQUENCE(1, 10)) AS t(val)");
1089-
String selectQuery = "SELECT root.leaf2 FROM " + tableName;
1090-
Session sessionWithoutPushdown = Session.builder(getSession())
1091-
.setCatalogSessionProperty(getSession().getCatalog().orElseThrow(), "projection_pushdown_enabled", "false")
1092-
.build();
1093-
1094-
assertQueryStats(
1095-
getSession(),
1096-
selectQuery,
1097-
statsWithPushdown -> {
1098-
DataSize physicalInputDataSizeWithPushdown = statsWithPushdown.getPhysicalInputDataSize();
1099-
DataSize processedDataSizeWithPushdown = statsWithPushdown.getProcessedInputDataSize();
1100-
assertQueryStats(
1101-
sessionWithoutPushdown,
1102-
selectQuery,
1103-
statsWithoutPushdown -> {
1104-
assertThat(statsWithoutPushdown.getPhysicalInputDataSize()).isGreaterThan(physicalInputDataSizeWithPushdown);
1105-
assertThat(statsWithoutPushdown.getProcessedInputDataSize()).isGreaterThan(processedDataSizeWithPushdown);
1106-
},
1107-
results -> assertEquals(results.getOnlyColumnAsSet(), expectedResult.getOnlyColumnAsSet()));
1108-
},
1109-
results -> assertEquals(results.getOnlyColumnAsSet(), expectedResult.getOnlyColumnAsSet()));
1110-
1111-
assertUpdate("DROP TABLE " + tableName);
1112-
}
1113-
1114-
@Test
1115-
public void testProjectionPushdownPhysicalInputSize()
1116-
{
1117-
// TODO consider moving this in BaseConnectorTest
1118-
String tableName = "test_projection_pushdown_physical_input_size_" + randomNameSuffix();
1119-
1120-
assertUpdate("CREATE TABLE " + tableName + " (id INT, root ROW(leaf1 BIGINT, leaf2 BIGINT))");
1121-
assertUpdate("INSERT INTO " + tableName + " SELECT val, ROW(val + 1, val + 2) FROM UNNEST(SEQUENCE(1, 10)) AS t(val)", 10);
1122-
1123-
// Verify that the physical input size is smaller when reading the root.leaf1 field compared to reading the root field
1124-
assertQueryStats(
1125-
getSession(),
1126-
"SELECT root FROM " + tableName,
1127-
statsWithSelectRootField -> {
1128-
assertQueryStats(
1129-
getSession(),
1130-
"SELECT root.leaf1 FROM " + tableName,
1131-
statsWithSelectLeafField -> {
1132-
assertThat(statsWithSelectLeafField.getPhysicalInputDataSize()).isLessThan(statsWithSelectRootField.getPhysicalInputDataSize());
1133-
},
1134-
results -> assertEquals(results.getOnlyColumnAsSet(), computeActual("SELECT val + 1 FROM UNNEST(SEQUENCE(1, 10)) AS t(val)").getOnlyColumnAsSet()));
1135-
},
1136-
results -> assertEquals(results.getOnlyColumnAsSet(), computeActual("SELECT ROW(val + 1, val + 2) FROM UNNEST(SEQUENCE(1, 10)) AS t(val)").getOnlyColumnAsSet()));
1137-
1138-
// Verify that the physical input size is the same when reading the root field compared to reading both the root and root.leaf1 fields
1139-
assertQueryStats(
1140-
getSession(),
1141-
"SELECT root FROM " + tableName,
1142-
statsWithSelectRootField -> {
1143-
assertQueryStats(
1144-
getSession(),
1145-
"SELECT root, root.leaf1 FROM " + tableName,
1146-
statsWithSelectRootAndLeafField -> {
1147-
assertThat(statsWithSelectRootAndLeafField.getPhysicalInputDataSize()).isEqualTo(statsWithSelectRootField.getPhysicalInputDataSize());
1148-
},
1149-
results -> assertEqualsIgnoreOrder(results.getMaterializedRows(), computeActual("SELECT ROW(val + 1, val + 2), val + 1 FROM UNNEST(SEQUENCE(1, 10)) AS t(val)").getMaterializedRows()));
1150-
},
1151-
results -> assertEquals(results.getOnlyColumnAsSet(), computeActual("SELECT ROW(val + 1, val + 2) FROM UNNEST(SEQUENCE(1, 10)) AS t(val)").getOnlyColumnAsSet()));
1152-
1153-
assertUpdate("DROP TABLE " + tableName);
1154-
}
1155-
1156970
@Test
1157971
public void testProjectionPushdownColumnReorderInSchemaAndDataFile()
1158972
{

‎plugin/trino-druid/src/test/java/io/trino/plugin/druid/BaseDruidConnectorTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior)
9191
case SUPPORTS_TRUNCATE:
9292
return true; // TODO ATM Truncate doesn't work in Druid, but testTruncateTable doesn't fail as CREATE TABLE is not supported
9393

94+
case SUPPORTS_ROW_TYPE:
95+
return false;
96+
9497
default:
9598
return super.hasBehavior(connectorBehavior);
9699
}

‎plugin/trino-elasticsearch/src/test/java/io/trino/plugin/elasticsearch/BaseElasticsearchConnectorTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,9 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior)
130130
case SUPPORTS_COMMENT_ON_COLUMN:
131131
return false;
132132

133+
case SUPPORTS_ROW_TYPE:
134+
return false;
135+
133136
default:
134137
return super.hasBehavior(connectorBehavior);
135138
}

‎plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8801,6 +8801,14 @@ private String getTableLocation(String tableName)
88018801
return (String) computeScalar("SELECT DISTINCT regexp_replace(\"$path\", '/[^/]*$', '') FROM " + tableName);
88028802
}
88038803

8804+
@Override
8805+
protected boolean supportsPhysicalPushdown()
8806+
{
8807+
// Hive table is created using default format which is ORC. Currently ORC reader has issue
8808+
// pruning dereferenced struct fields https://github.com/trinodb/trino/issues/17201
8809+
return false;
8810+
}
8811+
88048812
private static final class BucketedFilterTestSetup
88058813
{
88068814
private final String typeName;

‎plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/BaseHudiConnectorTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior)
3939
return false;
4040

4141
case SUPPORTS_TOPN_PUSHDOWN:
42+
case SUPPORTS_DEREFERENCE_PUSHDOWN:
4243
return false;
4344

4445
case SUPPORTS_CREATE_SCHEMA:

0 commit comments

Comments
 (0)