From d7d9ea3d7b8393556ea19454abb8a2c8d222cff6 Mon Sep 17 00:00:00 2001 From: Karol Sobczak Date: Mon, 29 Mar 2021 12:48:18 +0200 Subject: [PATCH 001/146] Update javadoc to match rule --- .../iterative/rule/TransformCorrelatedInPredicateToJoin.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java index 1f3641c3cd7e..8467237e625f 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java @@ -81,9 +81,9 @@ * Into: * 
  * - Project (output: CASE WHEN (countmatches > 0) THEN true WHEN (countnullmatches > 0) THEN null ELSE false END)
- *   - Aggregate (countmatches=count(*) where a, b not null; countnullmatches where a,b null but buildSideKnownNonNull is not null)
+ *   - Aggregate (countmatches=count(*) where a, b not null; countnullmatches where (a is null or b is null) but buildSideKnownNonNull is not null)
  *     grouping by (A'.*)
- *     - LeftJoin on (A and B correlation condition)
+ *     - LeftJoin on (a = B.b, A and B correlation condition)
  *       - AssignUniqueId (A')
  *         - A
  *
From 2ac1cec7503854c151e49cd4cc04f3d9452a6cb4 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Tue, 30 Mar 2021 10:32:22 +0200 Subject: [PATCH 002/146] Remove redundant suppression --- .../src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java index 26efb600646b..cb453baedaea 100644 --- a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java +++ b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java @@ -987,7 +987,6 @@ public void testGetSuperTypes() } @Test - @SuppressWarnings("resource") public void testGetSchemasMetadataCalls() throws Exception { @@ -1052,7 +1051,6 @@ public void testGetSchemasMetadataCalls() } @Test - @SuppressWarnings("resource") public void testGetTablesMetadataCalls() throws Exception { @@ -1181,7 +1179,6 @@ public void testGetTablesMetadataCalls() } @Test - @SuppressWarnings("resource") public void testGetColumnsMetadataCalls() throws Exception { From 093abcf271bd049ca89a2242e8bb3a7024556e5c Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Tue, 30 Mar 2021 10:56:31 +0200 Subject: [PATCH 003/146] Use existing connection in assertMetadataCalls Before the change, each invocation of `assertMetadataCalls` opened a new connection, even though the test setup provides one. --- .../trino/jdbc/TestTrinoDatabaseMetaData.java | 68 ++++++++++++++----- 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java index cb453baedaea..55eda66db269 100644 --- a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java +++ b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java @@ -994,6 +994,7 @@ public void testGetSchemasMetadataCalls() // No filter assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getSchemas(null, null), list("TABLE_CATALOG", "TABLE_SCHEM")), @@ -1002,6 +1003,7 @@ public void testGetSchemasMetadataCalls() // Equality predicate on catalog name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getSchemas(COUNTING_CATALOG, null), list("TABLE_CATALOG", "TABLE_SCHEM")), @@ -1014,6 +1016,7 @@ public void testGetSchemasMetadataCalls() // Equality predicate on schema name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getSchemas(COUNTING_CATALOG, "test\\_schema%"), list("TABLE_CATALOG", "TABLE_SCHEM")), @@ -1025,6 +1028,7 @@ public void testGetSchemasMetadataCalls() // LIKE predicate on schema name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getSchemas(COUNTING_CATALOG, "test_sch_ma1"), list("TABLE_CATALOG", "TABLE_SCHEM")), @@ -1034,6 +1038,7 @@ public void testGetSchemasMetadataCalls() // Empty schema name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getSchemas(COUNTING_CATALOG, ""), list("TABLE_CATALOG", "TABLE_SCHEM")), @@ -1043,6 +1048,7 @@ public void testGetSchemasMetadataCalls() // catalog does not exist assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getSchemas("wrong", null), list("TABLE_CATALOG", "TABLE_SCHEM")), @@ -1058,6 +1064,7 @@ public void testGetTablesMetadataCalls() // No filter assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getTables(null, null, null, null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), @@ -1067,6 +1074,7 @@ public void testGetTablesMetadataCalls() // Equality predicate on catalog name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, null, null, null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), @@ -1076,6 +1084,7 @@ public void testGetTablesMetadataCalls() // Equality predicate on schema name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, "test\\_schema1", null, null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), @@ -1088,6 +1097,7 @@ public void testGetTablesMetadataCalls() // LIKE predicate on schema name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, "test_sch_ma1", null, null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), @@ -1101,6 +1111,7 @@ public void testGetTablesMetadataCalls() // Equality predicate on table name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, null, "test\\_table1", null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), @@ -1113,6 +1124,7 @@ public void testGetTablesMetadataCalls() // LIKE predicate on table name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, null, "test_t_ble1", null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), @@ -1125,6 +1137,7 @@ public void testGetTablesMetadataCalls() // Equality predicate on schema name and table name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, "test\\_schema1", "test\\_table1", null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), @@ -1133,6 +1146,7 @@ public void testGetTablesMetadataCalls() // LIKE predicate on schema name and table name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, "test_schema1", "test_table1", null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), @@ -1143,6 +1157,7 @@ public void testGetTablesMetadataCalls() // catalog does not exist assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getTables("wrong", null, null, null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), @@ -1151,6 +1166,7 @@ public void testGetTablesMetadataCalls() // empty schema name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, "", null, null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), @@ -1161,6 +1177,7 @@ public void testGetTablesMetadataCalls() // empty table name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, null, "", null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), @@ -1171,6 +1188,7 @@ public void testGetTablesMetadataCalls() // no table types selected assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, null, null, new String[0]), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), @@ -1186,6 +1204,7 @@ public void testGetColumnsMetadataCalls() // No filter assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(null, null, null, null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1196,6 +1215,7 @@ public void testGetColumnsMetadataCalls() // Equality predicate on catalog name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(COUNTING_CATALOG, null, null, null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1206,6 +1226,7 @@ public void testGetColumnsMetadataCalls() // Equality predicate on catalog name, schema name and table name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(COUNTING_CATALOG, "test\\_schema1", "test\\_table1", null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1218,6 +1239,7 @@ public void testGetColumnsMetadataCalls() // Equality predicate on catalog name, schema name, table name and column name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(COUNTING_CATALOG, "test\\_schema1", "test\\_table1", "column\\_17"), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1228,6 +1250,7 @@ public void testGetColumnsMetadataCalls() // Equality predicate on catalog name, LIKE predicate on schema name, table name and column name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(COUNTING_CATALOG, "test_schema1", "test_table1", "column_17"), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1239,6 +1262,7 @@ public void testGetColumnsMetadataCalls() // LIKE predicate on schema name and table name, but no predicate on catalog name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(null, "test_schema1", "test_table1", null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1252,6 +1276,7 @@ public void testGetColumnsMetadataCalls() // LIKE predicate on schema name, but no predicate on catalog name and table name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(null, "test_schema1", null, null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1267,6 +1292,7 @@ public void testGetColumnsMetadataCalls() // LIKE predicate on table name, but no predicate on catalog name and schema name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(null, null, "test_table1", null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1282,6 +1308,7 @@ public void testGetColumnsMetadataCalls() // Equality predicate on schema name and table name, but no predicate on catalog name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(null, "test\\_schema1", "test\\_table1", null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1294,6 +1321,7 @@ public void testGetColumnsMetadataCalls() // catalog does not exist assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns("wrong", null, null, null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1302,6 +1330,7 @@ public void testGetColumnsMetadataCalls() // schema does not exist assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(COUNTING_CATALOG, "wrong\\_schema1", "test\\_table1", null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1311,6 +1340,7 @@ public void testGetColumnsMetadataCalls() // schema does not exist assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(COUNTING_CATALOG, "wrong_schema1", "test_table1", null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1322,6 +1352,7 @@ public void testGetColumnsMetadataCalls() // empty schema name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(COUNTING_CATALOG, "", null, null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1333,6 +1364,7 @@ public void testGetColumnsMetadataCalls() // empty table name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(COUNTING_CATALOG, null, "", null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1344,6 +1376,7 @@ public void testGetColumnsMetadataCalls() // empty column name assertMetadataCalls( + connection, readMetaData( databaseMetaData -> databaseMetaData.getColumns(COUNTING_CATALOG, null, null, ""), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), @@ -1393,19 +1426,23 @@ private Set captureQueries(Callable action) .collect(toImmutableSet()); } - private void assertMetadataCalls(MetaDataCallback>> callback, MetadataCallsCount expectedMetadataCallsCount) - throws Exception + private void assertMetadataCalls(Connection connection, MetaDataCallback>> callback, MetadataCallsCount expectedMetadataCallsCount) { assertMetadataCalls( + connection, callback, actual -> {}, expectedMetadataCallsCount); } - private void assertMetadataCalls(MetaDataCallback>> callback, Collection> expected, MetadataCallsCount expectedMetadataCallsCount) - throws Exception + private void assertMetadataCalls( + Connection connection, + MetaDataCallback>> callback, + Collection> expected, + MetadataCallsCount expectedMetadataCallsCount) { assertMetadataCalls( + connection, callback, actual -> assertThat(ImmutableMultiset.copyOf(requireNonNull(actual, "actual is null"))) .isEqualTo(ImmutableMultiset.copyOf(requireNonNull(expected, "expected is null"))), @@ -1413,23 +1450,20 @@ private void assertMetadataCalls(MetaDataCallback>> callback, Consumer>> resultsVerification, MetadataCallsCount expectedMetadataCallsCount) - throws Exception { - MetadataCallsCount actualMetadataCallsCount; - try (Connection connection = createConnection()) { - actualMetadataCallsCount = countingMockConnector.runCounting(() -> { - try { - Collection> actual = callback.apply(connection.getMetaData()); - resultsVerification.accept(actual); - } - catch (SQLException e) { - throw new RuntimeException(e); - } - }); - } + MetadataCallsCount actualMetadataCallsCount = countingMockConnector.runCounting(() -> { + try { + Collection> actual = callback.apply(connection.getMetaData()); + resultsVerification.accept(actual); + } + catch (SQLException e) { + throw new RuntimeException(e); + } + }); assertEquals(actualMetadataCallsCount, expectedMetadataCallsCount); } From 28233c8c579d34df04d2834b3ed5bee888ad611e Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Tue, 30 Mar 2021 11:16:43 +0200 Subject: [PATCH 004/146] Count getTableHandle calls in metadata calls tests --- .../trino/jdbc/TestTrinoDatabaseMetaData.java | 4 +- .../trino/testing/CountingMockConnector.java | 108 ++++++++++++++++-- .../tests/TestInformationSchemaConnector.java | 13 ++- 3 files changed, 109 insertions(+), 16 deletions(-) diff --git a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java index 55eda66db269..aaa0e2ada351 100644 --- a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java +++ b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java @@ -1142,7 +1142,8 @@ public void testGetTablesMetadataCalls() databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, "test\\_schema1", "test\\_table1", null), list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), list(list(COUNTING_CATALOG, "test_schema1", "test_table1", "TABLE")), - new MetadataCallsCount()); + new MetadataCallsCount() + .withGetTableHandleCount(1)); // LIKE predicate on schema name and table name assertMetadataCalls( @@ -1304,6 +1305,7 @@ public void testGetColumnsMetadataCalls() new MetadataCallsCount() .withListSchemasCount(3) .withListTablesCount(8) + .withGetTableHandleCount(2) .withGetColumnsCount(2)); // Equality predicate on schema name and table name, but no predicate on catalog name diff --git a/testing/trino-testing/src/main/java/io/trino/testing/CountingMockConnector.java b/testing/trino-testing/src/main/java/io/trino/testing/CountingMockConnector.java index 35522a7d7582..b71b07cdb24b 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/CountingMockConnector.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/CountingMockConnector.java @@ -34,6 +34,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static io.trino.connector.MockConnectorFactory.Builder.defaultGetColumns; +import static io.trino.connector.MockConnectorFactory.Builder.defaultGetTableHandle; import static io.trino.spi.security.PrincipalType.USER; public class CountingMockConnector @@ -54,6 +55,7 @@ public class CountingMockConnector private final AtomicLong listSchemasCallsCounter = new AtomicLong(); private final AtomicLong listTablesCallsCounter = new AtomicLong(); + private final AtomicLong getTableHandleCallsCounter = new AtomicLong(); private final AtomicLong getColumnsCallsCounter = new AtomicLong(); private final ListRoleGrantsCounter listRoleGranstCounter = new ListRoleGrantsCounter(); @@ -81,13 +83,16 @@ public MetadataCallsCount runCounting(Runnable runnable) synchronized (lock) { listSchemasCallsCounter.set(0); listTablesCallsCounter.set(0); + getTableHandleCallsCounter.set(0); getColumnsCallsCounter.set(0); listRoleGranstCounter.reset(); runnable.run(); - return new MetadataCallsCount(listSchemasCallsCounter.get(), + return new MetadataCallsCount( + listSchemasCallsCounter.get(), listTablesCallsCounter.get(), + getTableHandleCallsCounter.get(), getColumnsCallsCounter.get(), listRoleGranstCounter.listRowGrantsCallsCounter.get(), listRoleGranstCounter.rolesPushedCounter.get(), @@ -113,6 +118,10 @@ private ConnectorFactory getConnectorFactory() } return ImmutableList.of(); }) + .withGetTableHandle((connectorSession, schemaTableName) -> { + getTableHandleCallsCounter.incrementAndGet(); + return defaultGetTableHandle().apply(connectorSession, schemaTableName); + }) .withGetColumns(schemaTableName -> { getColumnsCallsCounter.incrementAndGet(); return defaultGetColumns().apply(schemaTableName); @@ -130,6 +139,7 @@ public static final class MetadataCallsCount { private final long listSchemasCount; private final long listTablesCount; + private final long getTableHandleCount; private final long getColumnsCount; private final long listRoleGrantsCount; private final long rolesPushedCount; @@ -138,11 +148,13 @@ public static final class MetadataCallsCount public MetadataCallsCount() { - this(0, 0, 0, 0, 0, 0, 0); + this(0, 0, 0, 0, 0, 0, 0, 0); } - public MetadataCallsCount(long listSchemasCount, + public MetadataCallsCount( + long listSchemasCount, long listTablesCount, + long getTableHandleCount, long getColumnsCount, long listRoleGrantsCount, long rolesPushedCount, @@ -151,6 +163,7 @@ public MetadataCallsCount(long listSchemasCount, { this.listSchemasCount = listSchemasCount; this.listTablesCount = listTablesCount; + this.getTableHandleCount = getTableHandleCount; this.getColumnsCount = getColumnsCount; this.listRoleGrantsCount = listRoleGrantsCount; this.rolesPushedCount = rolesPushedCount; @@ -160,37 +173,106 @@ public MetadataCallsCount(long listSchemasCount, public MetadataCallsCount withListSchemasCount(long listSchemasCount) { - return new MetadataCallsCount(listSchemasCount, listTablesCount, getColumnsCount, listRoleGrantsCount, rolesPushedCount, granteesPushedCount, limitPushedCount); + return new MetadataCallsCount( + listSchemasCount, + listTablesCount, + getTableHandleCount, + getColumnsCount, + listRoleGrantsCount, + rolesPushedCount, + granteesPushedCount, + limitPushedCount); } public MetadataCallsCount withListTablesCount(long listTablesCount) { - return new MetadataCallsCount(listSchemasCount, listTablesCount, getColumnsCount, listRoleGrantsCount, rolesPushedCount, granteesPushedCount, limitPushedCount); + return new MetadataCallsCount( + listSchemasCount, + listTablesCount, + getTableHandleCount, + getColumnsCount, + listRoleGrantsCount, + rolesPushedCount, + granteesPushedCount, + limitPushedCount); + } + + public MetadataCallsCount withGetTableHandleCount(long getTableHandleCount) + { + return new MetadataCallsCount( + listSchemasCount, + listTablesCount, + getTableHandleCount, + getColumnsCount, + listRoleGrantsCount, + rolesPushedCount, + granteesPushedCount, + limitPushedCount); } public MetadataCallsCount withGetColumnsCount(long getColumnsCount) { - return new MetadataCallsCount(listSchemasCount, listTablesCount, getColumnsCount, listRoleGrantsCount, rolesPushedCount, granteesPushedCount, limitPushedCount); + return new MetadataCallsCount( + listSchemasCount, + listTablesCount, + getTableHandleCount, + getColumnsCount, + listRoleGrantsCount, + rolesPushedCount, + granteesPushedCount, + limitPushedCount); } public MetadataCallsCount withListRoleGrantsCount(long listRoleGrantsCount) { - return new MetadataCallsCount(listSchemasCount, listTablesCount, getColumnsCount, listRoleGrantsCount, rolesPushedCount, granteesPushedCount, limitPushedCount); + return new MetadataCallsCount( + listSchemasCount, + listTablesCount, + getTableHandleCount, + getColumnsCount, + listRoleGrantsCount, + rolesPushedCount, + granteesPushedCount, + limitPushedCount); } public MetadataCallsCount withRolesPushedCount(long rolesPushedCount) { - return new MetadataCallsCount(listSchemasCount, listTablesCount, getColumnsCount, listRoleGrantsCount, rolesPushedCount, granteesPushedCount, limitPushedCount); + return new MetadataCallsCount( + listSchemasCount, + listTablesCount, + getTableHandleCount, + getColumnsCount, + listRoleGrantsCount, + rolesPushedCount, + granteesPushedCount, + limitPushedCount); } public MetadataCallsCount withGranteesPushedCount(long granteesPushedCount) { - return new MetadataCallsCount(listSchemasCount, listTablesCount, getColumnsCount, listRoleGrantsCount, rolesPushedCount, granteesPushedCount, limitPushedCount); + return new MetadataCallsCount( + listSchemasCount, + listTablesCount, + getTableHandleCount, + getColumnsCount, + listRoleGrantsCount, + rolesPushedCount, + granteesPushedCount, + limitPushedCount); } public MetadataCallsCount withLimitPushedCount(long limitPushedCount) { - return new MetadataCallsCount(listSchemasCount, listTablesCount, getColumnsCount, listRoleGrantsCount, rolesPushedCount, granteesPushedCount, limitPushedCount); + return new MetadataCallsCount( + listSchemasCount, + listTablesCount, + getTableHandleCount, + getColumnsCount, + listRoleGrantsCount, + rolesPushedCount, + granteesPushedCount, + limitPushedCount); } @Override @@ -205,6 +287,7 @@ public boolean equals(Object o) MetadataCallsCount that = (MetadataCallsCount) o; return listSchemasCount == that.listSchemasCount && listTablesCount == that.listTablesCount && + getTableHandleCount == that.getTableHandleCount && getColumnsCount == that.getColumnsCount && listRoleGrantsCount == that.listRoleGrantsCount && rolesPushedCount == that.rolesPushedCount && @@ -215,8 +298,10 @@ public boolean equals(Object o) @Override public int hashCode() { - return Objects.hash(listSchemasCount, + return Objects.hash( + listSchemasCount, listTablesCount, + getTableHandleCount, getColumnsCount, listRoleGrantsCount, rolesPushedCount, @@ -230,6 +315,7 @@ public String toString() return toStringHelper(this) .add("listSchemasCount", listSchemasCount) .add("listTablesCount", listTablesCount) + .add("getTableHandleCount", getTableHandleCount) .add("getColumnsCount", getColumnsCount) .add("listRoleGrantsCount", listRoleGrantsCount) .add("rolesPushedCount", rolesPushedCount) diff --git a/testing/trino-tests/src/test/java/io/trino/tests/TestInformationSchemaConnector.java b/testing/trino-tests/src/test/java/io/trino/tests/TestInformationSchemaConnector.java index 9ede07fcd826..894b0679fd7c 100644 --- a/testing/trino-tests/src/test/java/io/trino/tests/TestInformationSchemaConnector.java +++ b/testing/trino-tests/src/test/java/io/trino/tests/TestInformationSchemaConnector.java @@ -185,7 +185,8 @@ public void testMetadataCalls() "SELECT count(*) from test_catalog.information_schema.tables WHERE table_name = 'test_table1'", "VALUES 2", new MetadataCallsCount() - .withListSchemasCount(1)); + .withListSchemasCount(1) + .withGetTableHandleCount(2)); assertMetadataCalls( "SELECT count(*) from test_catalog.information_schema.tables WHERE table_name LIKE 'test_t_ble1'", "VALUES 2", @@ -196,12 +197,14 @@ public void testMetadataCalls() "SELECT count(*) from test_catalog.information_schema.tables WHERE table_name LIKE 'test_t_ble1' AND table_name IN ('test_table1', 'test_table2')", "VALUES 2", new MetadataCallsCount() - .withListSchemasCount(1)); + .withListSchemasCount(1) + .withGetTableHandleCount(4)); assertMetadataCalls( "SELECT count(*) from test_catalog.information_schema.columns WHERE table_schema = 'test_schema1' AND table_name = 'test_table1'", "VALUES 100", new MetadataCallsCount() .withListTablesCount(1) + .withGetTableHandleCount(1) .withGetColumnsCount(1)); assertMetadataCalls( "SELECT count(*) from test_catalog.information_schema.columns WHERE table_catalog = 'wrong'", @@ -211,12 +214,14 @@ public void testMetadataCalls() "SELECT count(*) from test_catalog.information_schema.columns WHERE table_catalog = 'test_catalog' AND table_schema = 'wrong_schema1' AND table_name = 'test_table1'", "VALUES 0", new MetadataCallsCount() - .withListTablesCount(1)); + .withListTablesCount(1) + .withGetTableHandleCount(1)); assertMetadataCalls( "SELECT count(*) from test_catalog.information_schema.columns WHERE table_catalog IN ('wrong', 'test_catalog') AND table_schema = 'wrong_schema1' AND table_name = 'test_table1'", "VALUES 0", new MetadataCallsCount() - .withListTablesCount(1)); + .withListTablesCount(1) + .withGetTableHandleCount(1)); assertMetadataCalls( "SELECT count(*) FROM (SELECT * from test_catalog.information_schema.columns LIMIT 1)", "VALUES 1", From 266e00100b2b4704162c237852fb01e1c044d338 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Fri, 26 Mar 2021 21:59:50 +0100 Subject: [PATCH 005/146] Add compatibility with misbehaving client applications Number of `DatabaseMetaData` methods accept name patterns (e.g. `schemaPattern`, `tableNamePattern`, `columnNamePattern`, etc.). Values provided, if ought to be treated as literals, should have underscores `_` and percent signs `%` escaped with `DatabaseMetaData.getSearchStringEscape()`. Sadly, many client applications do not do that, which results in metadata queries which cannot be answered efficiently by the server. This commit adds `assumeLiteralNamesInMetadataCallsForNonConformingClients`, a compatibility flag as a workaround for such misbehaving client applications. --- .../io/trino/jdbc/ConnectionProperties.java | 11 +++ .../java/io/trino/jdbc/TrinoConnection.java | 4 +- .../io/trino/jdbc/TrinoDatabaseMetaData.java | 55 ++++++++++- .../java/io/trino/jdbc/TrinoDriverUri.java | 7 ++ .../trino/jdbc/TestTrinoDatabaseMetaData.java | 96 +++++++++++++++++++ 5 files changed, 171 insertions(+), 2 deletions(-) diff --git a/client/trino-jdbc/src/main/java/io/trino/jdbc/ConnectionProperties.java b/client/trino-jdbc/src/main/java/io/trino/jdbc/ConnectionProperties.java index c3db9662374b..0c1f3bea085d 100644 --- a/client/trino-jdbc/src/main/java/io/trino/jdbc/ConnectionProperties.java +++ b/client/trino-jdbc/src/main/java/io/trino/jdbc/ConnectionProperties.java @@ -55,6 +55,7 @@ enum SslVerificationMode public static final ConnectionProperty HTTP_PROXY = new HttpProxy(); public static final ConnectionProperty APPLICATION_NAME_PREFIX = new ApplicationNamePrefix(); public static final ConnectionProperty DISABLE_COMPRESSION = new DisableCompression(); + public static final ConnectionProperty ASSUME_LITERAL_NAMES_IN_METADATA_CALLS_FOR_NON_CONFORMING_CLIENTS = new AssumeLiteralNamesInMetadataCallsForNonConformingClients(); public static final ConnectionProperty SSL = new Ssl(); public static final ConnectionProperty SSL_VERIFICATION = new SslVerification(); public static final ConnectionProperty SSL_KEY_STORE_PATH = new SslKeyStorePath(); @@ -89,6 +90,7 @@ enum SslVerificationMode .add(HTTP_PROXY) .add(APPLICATION_NAME_PREFIX) .add(DISABLE_COMPRESSION) + .add(ASSUME_LITERAL_NAMES_IN_METADATA_CALLS_FOR_NON_CONFORMING_CLIENTS) .add(SSL) .add(SSL_VERIFICATION) .add(SSL_KEY_STORE_PATH) @@ -273,6 +275,15 @@ public DisableCompression() } } + private static class AssumeLiteralNamesInMetadataCallsForNonConformingClients + extends AbstractConnectionProperty + { + public AssumeLiteralNamesInMetadataCallsForNonConformingClients() + { + super("assumeLiteralNamesInMetadataCallsForNonConformingClients", NOT_REQUIRED, ALLOWED, BOOLEAN_CONVERTER); + } + } + private static class Ssl extends AbstractConnectionProperty { diff --git a/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoConnection.java b/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoConnection.java index 81c2b8cc148a..bea84728d960 100644 --- a/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoConnection.java +++ b/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoConnection.java @@ -91,6 +91,7 @@ public class TrinoConnection private final String user; private final Optional sessionUser; private final boolean compressionDisabled; + private final boolean assumeLiteralNamesInMetadataCallsForNonConformingClients; private final Map extraCredentials; private final Optional applicationNamePrefix; private final Optional source; @@ -115,6 +116,7 @@ public class TrinoConnection this.source = uri.getSource(); this.extraCredentials = uri.getExtraCredentials(); this.compressionDisabled = uri.isCompressionDisabled(); + this.assumeLiteralNamesInMetadataCallsForNonConformingClients = uri.isAssumeLiteralNamesInMetadataCallsForNonConformingClients(); this.queryExecutor = requireNonNull(queryExecutor, "queryExecutor is null"); uri.getClientInfo().ifPresent(tags -> clientInfo.put(CLIENT_INFO, tags)); uri.getClientTags().ifPresent(tags -> clientInfo.put(CLIENT_TAGS, tags)); @@ -238,7 +240,7 @@ public boolean isClosed() public DatabaseMetaData getMetaData() throws SQLException { - return new TrinoDatabaseMetaData(this); + return new TrinoDatabaseMetaData(this, assumeLiteralNamesInMetadataCallsForNonConformingClients); } @Override diff --git a/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoDatabaseMetaData.java b/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoDatabaseMetaData.java index 49033d3ca876..9782234da699 100644 --- a/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoDatabaseMetaData.java +++ b/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoDatabaseMetaData.java @@ -20,6 +20,8 @@ import io.trino.client.ClientTypeSignatureParameter; import io.trino.client.Column; +import javax.annotation.Nullable; + import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.ResultSet; @@ -31,6 +33,7 @@ import java.util.List; import java.util.stream.Stream; +import static com.google.common.base.Verify.verify; import static com.google.common.collect.Lists.newArrayList; import static io.trino.client.ClientTypeSignature.VARCHAR_UNBOUNDED_LENGTH; import static io.trino.jdbc.DriverInfo.DRIVER_NAME; @@ -46,10 +49,12 @@ public class TrinoDatabaseMetaData private static final String SEARCH_STRING_ESCAPE = "\\"; private final TrinoConnection connection; + private final boolean assumeLiteralNamesInMetadataCallsForNonConformingClients; - TrinoDatabaseMetaData(TrinoConnection connection) + TrinoDatabaseMetaData(TrinoConnection connection, boolean assumeLiteralNamesInMetadataCallsForNonConformingClients) { this.connection = requireNonNull(connection, "connection is null"); + this.assumeLiteralNamesInMetadataCallsForNonConformingClients = assumeLiteralNamesInMetadataCallsForNonConformingClients; } @Override @@ -897,6 +902,8 @@ public boolean dataDefinitionIgnoredInTransactions() public ResultSet getProcedures(String catalog, String schemaPattern, String procedureNamePattern) throws SQLException { + schemaPattern = escapeIfNecessary(schemaPattern); + procedureNamePattern = escapeIfNecessary(procedureNamePattern); return selectEmpty("" + "SELECT PROCEDURE_CAT, PROCEDURE_SCHEM, PROCEDURE_NAME,\n " + " null, null, null, REMARKS, PROCEDURE_TYPE, SPECIFIC_NAME\n" + @@ -908,6 +915,9 @@ public ResultSet getProcedures(String catalog, String schemaPattern, String proc public ResultSet getProcedureColumns(String catalog, String schemaPattern, String procedureNamePattern, String columnNamePattern) throws SQLException { + schemaPattern = escapeIfNecessary(schemaPattern); + procedureNamePattern = escapeIfNecessary(procedureNamePattern); + columnNamePattern = escapeIfNecessary(columnNamePattern); return selectEmpty("" + "SELECT PROCEDURE_CAT, PROCEDURE_SCHEM, PROCEDURE_NAME, " + " COLUMN_NAME, COLUMN_TYPE, DATA_TYPE, TYPE_NAME,\n" + @@ -922,6 +932,8 @@ public ResultSet getProcedureColumns(String catalog, String schemaPattern, Strin public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) throws SQLException { + schemaPattern = escapeIfNecessary(schemaPattern); + tableNamePattern = escapeIfNecessary(tableNamePattern); StringBuilder query = new StringBuilder("" + "SELECT TABLE_CAT, TABLE_SCHEM, TABLE_NAME, TABLE_TYPE, REMARKS,\n" + " TYPE_CAT, TYPE_SCHEM, TYPE_NAME, " + @@ -974,6 +986,9 @@ public ResultSet getTableTypes() public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { + schemaPattern = escapeIfNecessary(schemaPattern); + tableNamePattern = escapeIfNecessary(tableNamePattern); + columnNamePattern = escapeIfNecessary(columnNamePattern); StringBuilder query = new StringBuilder("" + "SELECT TABLE_CAT, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, DATA_TYPE,\n" + " TYPE_NAME, COLUMN_SIZE, BUFFER_LENGTH, DECIMAL_DIGITS, NUM_PREC_RADIX,\n" + @@ -999,6 +1014,7 @@ public ResultSet getColumns(String catalog, String schemaPattern, String tableNa public ResultSet getColumnPrivileges(String catalog, String schema, String table, String columnNamePattern) throws SQLException { + columnNamePattern = escapeIfNecessary(columnNamePattern); throw new SQLFeatureNotSupportedException("privileges not supported"); } @@ -1006,6 +1022,8 @@ public ResultSet getColumnPrivileges(String catalog, String schema, String table public ResultSet getTablePrivileges(String catalog, String schemaPattern, String tableNamePattern) throws SQLException { + schemaPattern = escapeIfNecessary(schemaPattern); + tableNamePattern = escapeIfNecessary(tableNamePattern); throw new SQLFeatureNotSupportedException("privileges not supported"); } @@ -1168,6 +1186,8 @@ public boolean supportsBatchUpdates() public ResultSet getUDTs(String catalog, String schemaPattern, String typeNamePattern, int[] types) throws SQLException { + schemaPattern = escapeIfNecessary(schemaPattern); + typeNamePattern = escapeIfNecessary(typeNamePattern); return selectEmpty("" + "SELECT TYPE_CAT, TYPE_SCHEM, TYPE_NAME,\n" + " CLASS_NAME, DATA_TYPE, REMARKS, BASE_TYPE\n" + @@ -1214,6 +1234,8 @@ public boolean supportsGetGeneratedKeys() public ResultSet getSuperTypes(String catalog, String schemaPattern, String typeNamePattern) throws SQLException { + schemaPattern = escapeIfNecessary(schemaPattern); + typeNamePattern = escapeIfNecessary(typeNamePattern); return selectEmpty("" + "SELECT TYPE_CAT, TYPE_SCHEM, TYPE_NAME,\n" + " SUPERTYPE_CAT, SUPERTYPE_SCHEM, SUPERTYPE_NAME\n" + @@ -1225,6 +1247,8 @@ public ResultSet getSuperTypes(String catalog, String schemaPattern, String type public ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern) throws SQLException { + schemaPattern = escapeIfNecessary(schemaPattern); + tableNamePattern = escapeIfNecessary(tableNamePattern); return selectEmpty("" + "SELECT TABLE_CAT, TABLE_SCHEM, TABLE_NAME, SUPERTABLE_NAME\n" + "FROM system.jdbc.super_tables\n" + @@ -1235,6 +1259,9 @@ public ResultSet getSuperTables(String catalog, String schemaPattern, String tab public ResultSet getAttributes(String catalog, String schemaPattern, String typeNamePattern, String attributeNamePattern) throws SQLException { + schemaPattern = escapeIfNecessary(schemaPattern); + typeNamePattern = escapeIfNecessary(typeNamePattern); + attributeNamePattern = escapeIfNecessary(attributeNamePattern); return selectEmpty("" + "SELECT TYPE_CAT, TYPE_SCHEM, TYPE_NAME, ATTR_NAME, DATA_TYPE,\n" + " ATTR_TYPE_NAME, ATTR_SIZE, DECIMAL_DIGITS, NUM_PREC_RADIX, NULLABLE,\n" + @@ -1332,6 +1359,7 @@ public RowIdLifetime getRowIdLifetime() public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLException { + schemaPattern = escapeIfNecessary(schemaPattern); StringBuilder query = new StringBuilder("" + "SELECT TABLE_SCHEM, TABLE_CATALOG\n" + "FROM system.jdbc.schemas"); @@ -1391,6 +1419,8 @@ public ResultSet getClientInfoProperties() public ResultSet getFunctions(String catalog, String schemaPattern, String functionNamePattern) throws SQLException { + schemaPattern = escapeIfNecessary(schemaPattern); + functionNamePattern = escapeIfNecessary(functionNamePattern); // TODO: implement this throw new NotImplementedException("DatabaseMetaData", "getFunctions"); } @@ -1399,6 +1429,9 @@ public ResultSet getFunctions(String catalog, String schemaPattern, String funct public ResultSet getFunctionColumns(String catalog, String schemaPattern, String functionNamePattern, String columnNamePattern) throws SQLException { + schemaPattern = escapeIfNecessary(schemaPattern); + functionNamePattern = escapeIfNecessary(functionNamePattern); + columnNamePattern = escapeIfNecessary(columnNamePattern); // TODO: implement this throw new NotImplementedException("DatabaseMetaData", "getFunctionColumns"); } @@ -1407,6 +1440,9 @@ public ResultSet getFunctionColumns(String catalog, String schemaPattern, String public ResultSet getPseudoColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { + schemaPattern = escapeIfNecessary(schemaPattern); + tableNamePattern = escapeIfNecessary(tableNamePattern); + columnNamePattern = escapeIfNecessary(columnNamePattern); return selectEmpty("" + "SELECT TABLE_CAT, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, DATA_TYPE,\n" + " COLUMN_SIZE, DECIMAL_DIGITS, NUM_PREC_RADIX, COLUMN_USAGE, REMARKS,\n" + @@ -1485,6 +1521,23 @@ private static void optionalStringInFilter(List filters, String columnNa filters.add(filter.toString()); } + @Nullable + private String escapeIfNecessary(@Nullable String namePattern) + { + return escapeIfNecessary(assumeLiteralNamesInMetadataCallsForNonConformingClients, namePattern); + } + + @Nullable + static String escapeIfNecessary(boolean assumeLiteralNamesInMetadataCallsForNonConformingClients, @Nullable String namePattern) + { + if (namePattern == null || !assumeLiteralNamesInMetadataCallsForNonConformingClients) { + return namePattern; + } + //noinspection ConstantConditions + verify(SEARCH_STRING_ESCAPE.equals("\\")); + return namePattern.replaceAll("[_%\\\\]", "\\\\$0"); + } + private static void optionalStringLikeFilter(List filters, String columnName, String value) { if (value != null) { diff --git a/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoDriverUri.java b/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoDriverUri.java index 9a2af8cb55d0..23bf7440a46a 100644 --- a/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoDriverUri.java +++ b/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoDriverUri.java @@ -53,6 +53,7 @@ import static io.trino.client.OkHttpUtil.tokenAuth; import static io.trino.jdbc.ConnectionProperties.ACCESS_TOKEN; import static io.trino.jdbc.ConnectionProperties.APPLICATION_NAME_PREFIX; +import static io.trino.jdbc.ConnectionProperties.ASSUME_LITERAL_NAMES_IN_METADATA_CALLS_FOR_NON_CONFORMING_CLIENTS; import static io.trino.jdbc.ConnectionProperties.CLIENT_INFO; import static io.trino.jdbc.ConnectionProperties.CLIENT_TAGS; import static io.trino.jdbc.ConnectionProperties.DISABLE_COMPRESSION; @@ -236,6 +237,12 @@ public boolean isCompressionDisabled() return DISABLE_COMPRESSION.getValue(properties).orElse(false); } + public boolean isAssumeLiteralNamesInMetadataCallsForNonConformingClients() + throws SQLException + { + return ASSUME_LITERAL_NAMES_IN_METADATA_CALLS_FOR_NON_CONFORMING_CLIENTS.getValue(properties).orElse(false); + } + public void setupClient(OkHttpClient.Builder builder) throws SQLException { diff --git a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java index aaa0e2ada351..924347f50f57 100644 --- a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java +++ b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java @@ -1389,6 +1389,102 @@ public void testGetColumnsMetadataCalls() .withGetColumnsCount(3000)); } + @Test + public void testAssumeLiteralMetadataCalls() + throws Exception + { + try (Connection connection = DriverManager.getConnection( + format("jdbc:trino://%s?assumeLiteralNamesInMetadataCallsForNonConformingClients=true", server.getAddress()), + "admin", + null)) { + // getTables's schema name pattern treated as literal + assertMetadataCalls( + connection, + readMetaData( + databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, "test_schema1", null, null), + list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), + countingMockConnector.getAllTables() + .filter(schemaTableName -> schemaTableName.getSchemaName().equals("test_schema1")) + .map(schemaTableName -> list(COUNTING_CATALOG, schemaTableName.getSchemaName(), schemaTableName.getTableName(), "TABLE")) + .collect(toImmutableList()), + new MetadataCallsCount() + .withListSchemasCount(0) + .withListTablesCount(1)); + + // getTables's schema and table name patterns treated as literals + assertMetadataCalls( + connection, + readMetaData( + databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, "test_schema1", "test_table1", null), + list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), + list(list(COUNTING_CATALOG, "test_schema1", "test_table1", "TABLE")), + new MetadataCallsCount() + .withGetTableHandleCount(1)); + + // no matches in getTables call as table name pattern treated as literal + assertMetadataCalls( + connection, + readMetaData( + databaseMetaData -> databaseMetaData.getTables(COUNTING_CATALOG, "test_schema_", null, null), + list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE")), + list(), + new MetadataCallsCount() + .withListTablesCount(1)); + + // getColumns's schema and table name patterns treated as literals + assertMetadataCalls( + connection, + readMetaData( + databaseMetaData -> databaseMetaData.getColumns(COUNTING_CATALOG, "test_schema1", "test_table1", null), + list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), + IntStream.range(0, 100) + .mapToObj(i -> list(COUNTING_CATALOG, "test_schema1", "test_table1", "column_" + i, "varchar")) + .collect(toImmutableList()), + new MetadataCallsCount() + .withListTablesCount(1) + .withGetColumnsCount(1)); + + // getColumns's schema, table and column name patterns treated as literals + assertMetadataCalls( + connection, + readMetaData( + databaseMetaData -> databaseMetaData.getColumns(COUNTING_CATALOG, "test_schema1", "test_table1", "column_17"), + list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), + list(list(COUNTING_CATALOG, "test_schema1", "test_table1", "column_17", "varchar")), + new MetadataCallsCount() + .withListTablesCount(1) + .withGetColumnsCount(1)); + + // no matches in getColumns call as table name pattern treated as literal + assertMetadataCalls( + connection, + readMetaData( + databaseMetaData -> databaseMetaData.getColumns(COUNTING_CATALOG, "test_schema1", "test_table_", null), + list("TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "COLUMN_NAME", "TYPE_NAME")), + list(), + new MetadataCallsCount() + .withListTablesCount(1)); + } + } + + @Test + public void testEscapeIfNecessary() + { + assertEquals(TrinoDatabaseMetaData.escapeIfNecessary(false, null), null); + assertEquals(TrinoDatabaseMetaData.escapeIfNecessary(false, "a"), "a"); + assertEquals(TrinoDatabaseMetaData.escapeIfNecessary(false, "abc_def"), "abc_def"); + assertEquals(TrinoDatabaseMetaData.escapeIfNecessary(false, "abc__de_f"), "abc__de_f"); + assertEquals(TrinoDatabaseMetaData.escapeIfNecessary(false, "abc%def"), "abc%def"); + assertEquals(TrinoDatabaseMetaData.escapeIfNecessary(false, "abc\\_def"), "abc\\_def"); + + assertEquals(TrinoDatabaseMetaData.escapeIfNecessary(true, null), null); + assertEquals(TrinoDatabaseMetaData.escapeIfNecessary(true, "a"), "a"); + assertEquals(TrinoDatabaseMetaData.escapeIfNecessary(true, "abc_def"), "abc\\_def"); + assertEquals(TrinoDatabaseMetaData.escapeIfNecessary(true, "abc__de_f"), "abc\\_\\_de\\_f"); + assertEquals(TrinoDatabaseMetaData.escapeIfNecessary(true, "abc%def"), "abc\\%def"); + assertEquals(TrinoDatabaseMetaData.escapeIfNecessary(true, "abc\\_def"), "abc\\\\\\_def"); + } + private static void assertColumnSpec(ResultSet rs, int dataType, Long precision, Long numPrecRadix, String typeName) throws SQLException { From 670cf1d703f9b1b388e3363d7c132358da3d6826 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Thu, 25 Mar 2021 21:50:44 +0100 Subject: [PATCH 006/146] Add description for max-recursion-depth config Wording copied from session property. --- .../src/main/java/io/trino/sql/analyzer/FeaturesConfig.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/FeaturesConfig.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/FeaturesConfig.java index 8f23911c8169..b61d9a716e0a 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/FeaturesConfig.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/FeaturesConfig.java @@ -906,6 +906,7 @@ public int getMaxRecursionDepth() } @Config("max-recursion-depth") + @ConfigDescription("Maximum recursion depth for recursive common table expression") public FeaturesConfig setMaxRecursionDepth(int maxRecursionDepth) { this.maxRecursionDepth = maxRecursionDepth; From 4ff5421681c6958c74332578c2c906f26df34547 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Thu, 25 Mar 2021 17:26:49 +0100 Subject: [PATCH 007/146] Support LATERAL VIEW json_tuple AS ... in Hive VIEW --- pom.xml | 2 +- .../io/trino/tests/hive/TestHiveViews.java | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0c4b6a2b9e69..3b047b57b6f2 100644 --- a/pom.xml +++ b/pom.xml @@ -61,7 +61,7 @@ 2.5.1 1.15.1 3.2.7 - 1.0.33 + 1.0.37 5.5.2 + + net.bytebuddy + byte-buddy + 1.10.22 + + net.java.dev.jna From 3d85f805276a2bb6e1e5349a099c688c8b9eb0a2 Mon Sep 17 00:00:00 2001 From: Mateusz Gajewski Date: Wed, 7 Apr 2021 18:50:58 +0200 Subject: [PATCH 057/146] Fix Hive's file_modified_time value rendering --- .../java/io/trino/plugin/hive/HivePageSource.java | 4 +++- .../java/io/trino/plugin/hive/HiveRecordCursor.java | 4 +++- .../plugin/hive/TestHiveIntegrationSmokeTest.java | 12 ++++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java index d08ad11441d1..32738cf1f9c7 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSource.java @@ -115,7 +115,9 @@ import static io.trino.spi.type.SmallintType.SMALLINT; import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; import static io.trino.spi.type.TimestampWithTimeZoneType.TIMESTAMP_TZ_MILLIS; +import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND; import static io.trino.spi.type.TinyintType.TINYINT; +import static java.lang.Math.floorDiv; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; @@ -223,7 +225,7 @@ else if (type.equals(TIMESTAMP_MILLIS)) { } else if (type.equals(TIMESTAMP_TZ_MILLIS)) { // used for $file_modified_time - prefilledValue = packDateTimeWithZone(timestampPartitionKey(columnValue, name), DateTimeZone.getDefault().getID()); + prefilledValue = packDateTimeWithZone(floorDiv(timestampPartitionKey(columnValue, name), MICROSECONDS_PER_MILLISECOND), DateTimeZone.getDefault().getID()); } else if (isShortDecimal(type)) { prefilledValue = shortDecimalPartitionKey(columnValue, (DecimalType) type, name); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveRecordCursor.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveRecordCursor.java index 2f9934a0da07..063d38f7cc60 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveRecordCursor.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveRecordCursor.java @@ -57,7 +57,9 @@ import static io.trino.spi.type.SmallintType.SMALLINT; import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; import static io.trino.spi.type.TimestampWithTimeZoneType.TIMESTAMP_TZ_MILLIS; +import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND; import static io.trino.spi.type.TinyintType.TINYINT; +import static java.lang.Math.floorDiv; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; @@ -147,7 +149,7 @@ else if (TIMESTAMP_MILLIS.equals(type)) { } else if (TIMESTAMP_TZ_MILLIS.equals(type)) { // used for $file_modified_time - longs[columnIndex] = packDateTimeWithZone(timestampPartitionKey(columnValue, name), DateTimeZone.getDefault().getID()); + longs[columnIndex] = packDateTimeWithZone(floorDiv(timestampPartitionKey(columnValue, name), MICROSECONDS_PER_MILLISECOND), DateTimeZone.getDefault().getID()); } else if (isShortDecimal(type)) { longs[columnIndex] = shortDecimalPartitionKey(columnValue, (DecimalType) type, name); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveIntegrationSmokeTest.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveIntegrationSmokeTest.java index a6755c7a3e14..23d8f79f671f 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveIntegrationSmokeTest.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveIntegrationSmokeTest.java @@ -155,6 +155,7 @@ import static java.util.stream.Collectors.joining; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.data.Offset.offset; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; @@ -4236,8 +4237,8 @@ public void testFileSizeHiddenColumn() assertUpdate("DROP TABLE test_file_size"); } - @Test - public void testFileModifiedTimeHiddenColumn() + @Test(dataProvider = "timestampPrecision") + public void testFileModifiedTimeHiddenColumn(HiveTimestampPrecision precision) { long testStartTime = Instant.now().toEpochMilli(); @@ -4267,7 +4268,10 @@ public void testFileModifiedTimeHiddenColumn() } assertEquals(getPartitions("test_file_modified_time").size(), 3); - MaterializedResult results = computeActual(format("SELECT *, \"%s\" FROM test_file_modified_time", FILE_MODIFIED_TIME_COLUMN_NAME)); + Session sessionWithTimestampPrecision = withTimestampPrecision(getSession(), precision); + MaterializedResult results = computeActual( + sessionWithTimestampPrecision, + format("SELECT *, \"%s\" FROM test_file_modified_time", FILE_MODIFIED_TIME_COLUMN_NAME)); Map fileModifiedTimeMap = new HashMap<>(); for (int i = 0; i < results.getRowCount(); i++) { MaterializedRow row = results.getMaterializedRows().get(i); @@ -4275,7 +4279,7 @@ public void testFileModifiedTimeHiddenColumn() int col1 = (int) row.getField(1); Instant fileModifiedTime = ((ZonedDateTime) row.getField(2)).toInstant(); - assertTrue(fileModifiedTime.toEpochMilli() > (testStartTime - 2_000)); + assertThat(fileModifiedTime.toEpochMilli()).isCloseTo(testStartTime, offset(2000L)); assertEquals(col0 % 3, col1); if (fileModifiedTimeMap.containsKey(col1)) { assertEquals(fileModifiedTimeMap.get(col1), fileModifiedTime); From 88918f225f981a92a36cf01aeae97f6b7ad1b27d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Kokosi=C5=84ski?= Date: Wed, 7 Apr 2021 10:08:44 +0200 Subject: [PATCH 058/146] Cleanup in StatementAnalyzer --- .../main/java/io/trino/sql/analyzer/StatementAnalyzer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java index 06013338fbc3..960ce3e9c49d 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java @@ -1149,7 +1149,7 @@ protected Scope visitQuery(Query node, Optional scope) @Override protected Scope visitUnnest(Unnest node, Optional scope) { - ImmutableMap.Builder, List> mappings = ImmutableMap., List>builder(); + ImmutableMap.Builder, List> mappings = ImmutableMap.builder(); ImmutableList.Builder outputFields = ImmutableList.builder(); for (Expression expression : node.getExpressions()) { @@ -2786,7 +2786,7 @@ private void analyzeSelectSingleColumn( } } - public void analyzeWhere(Node node, Scope scope, Expression predicate) + private void analyzeWhere(Node node, Scope scope, Expression predicate) { verifyNoAggregateWindowOrGroupingFunctions(metadata, predicate, "WHERE clause"); @@ -3207,7 +3207,7 @@ private boolean tryProcessRecursiveQuery(WithQuery withQuery, String name, Scope } Relation from = ((QuerySpecification) specification).getFrom().get(); List fromReferences = findReferences(from, withQuery.getName()); - if (fromReferences.size() == 0) { + if (fromReferences.isEmpty()) { throw semanticException(INVALID_RECURSIVE_REFERENCE, stepReferences.get(0), "recursive reference outside of FROM clause of the step relation of recursion"); } From bf85bfb58179ac287082db78fa30875540a82f3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Kokosi=C5=84ski?= Date: Wed, 7 Apr 2021 10:08:45 +0200 Subject: [PATCH 059/146] Introduce ConnectorTableSchema ConnectorTableSchema represents table schema definition, a set of information required by semantic analyzer to analyze the query. --- .../main/java/io/trino/metadata/Metadata.java | 11 ++ .../io/trino/metadata/MetadataManager.java | 11 ++ .../java/io/trino/metadata/TableSchema.java | 72 ++++++++++ .../trino/metadata/AbstractMockMetadata.java | 6 + .../trino/spi/connector/ColumnMetadata.java | 6 + .../io/trino/spi/connector/ColumnSchema.java | 135 ++++++++++++++++++ .../spi/connector/ConnectorMetadata.java | 11 ++ .../spi/connector/ConnectorTableMetadata.java | 10 ++ .../spi/connector/ConnectorTableSchema.java | 53 +++++++ .../ClassLoaderSafeConnectorMetadata.java | 9 ++ 10 files changed, 324 insertions(+) create mode 100644 core/trino-main/src/main/java/io/trino/metadata/TableSchema.java create mode 100644 core/trino-spi/src/main/java/io/trino/spi/connector/ColumnSchema.java create mode 100644 core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorTableSchema.java diff --git a/core/trino-main/src/main/java/io/trino/metadata/Metadata.java b/core/trino-main/src/main/java/io/trino/metadata/Metadata.java index f6f33ec33588..bb76cd1e54a8 100644 --- a/core/trino-main/src/main/java/io/trino/metadata/Metadata.java +++ b/core/trino-main/src/main/java/io/trino/metadata/Metadata.java @@ -114,8 +114,19 @@ public interface Metadata Optional getInfo(Session session, TableHandle handle); + /** + * Return table schema definition for the specified table handle. + * Table schema definition is a set of information + * required by semantic analyzer to analyze the query. + * @see {@link #getTableMetadata(Session, TableHandle)} + * + * @throws RuntimeException if table handle is no longer valid + */ + TableSchema getTableSchema(Session session, TableHandle tableHandle); + /** * Return the metadata for the specified table handle. + * @see {@link #getTableSchema(Session, TableHandle)} which is less expsensive. * * @throws RuntimeException if table handle is no longer valid */ diff --git a/core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java b/core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java index 9807e9d9a495..78377e0b9483 100644 --- a/core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java +++ b/core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java @@ -69,6 +69,7 @@ import io.trino.spi.connector.ConnectorTableLayoutResult; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.ConnectorTableProperties; +import io.trino.spi.connector.ConnectorTableSchema; import io.trino.spi.connector.ConnectorTransactionHandle; import io.trino.spi.connector.ConnectorViewDefinition; import io.trino.spi.connector.Constraint; @@ -508,6 +509,16 @@ public Optional getInfo(Session session, TableHandle handle) return metadata.getInfo(handle.getConnectorHandle()); } + @Override + public TableSchema getTableSchema(Session session, TableHandle tableHandle) + { + CatalogName catalogName = tableHandle.getCatalogName(); + ConnectorMetadata metadata = getMetadata(session, catalogName); + ConnectorTableSchema tableSchema = metadata.getTableSchema(session.toConnectorSession(catalogName), tableHandle.getConnectorHandle()); + + return new TableSchema(catalogName, tableSchema); + } + @Override public TableMetadata getTableMetadata(Session session, TableHandle tableHandle) { diff --git a/core/trino-main/src/main/java/io/trino/metadata/TableSchema.java b/core/trino-main/src/main/java/io/trino/metadata/TableSchema.java new file mode 100644 index 000000000000..a29498a1cbd0 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/metadata/TableSchema.java @@ -0,0 +1,72 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.metadata; + +import io.trino.connector.CatalogName; +import io.trino.spi.connector.ColumnSchema; +import io.trino.spi.connector.ConnectorTableSchema; +import io.trino.spi.connector.SchemaTableName; + +import java.util.List; + +import static com.google.common.collect.MoreCollectors.toOptional; +import static java.util.Objects.requireNonNull; + +public final class TableSchema +{ + private final CatalogName catalogName; + private final ConnectorTableSchema tableSchema; + + public TableSchema(CatalogName catalogName, ConnectorTableSchema tableSchema) + { + requireNonNull(catalogName, "catalog is null"); + requireNonNull(tableSchema, "metadata is null"); + + this.catalogName = catalogName; + this.tableSchema = tableSchema; + } + + public QualifiedObjectName getQualifiedName() + { + return new QualifiedObjectName(catalogName.getCatalogName(), tableSchema.getTable().getSchemaName(), tableSchema.getTable().getTableName()); + } + + public CatalogName getCatalogName() + { + return catalogName; + } + + public ConnectorTableSchema getTableSchema() + { + return tableSchema; + } + + public SchemaTableName getTable() + { + return tableSchema.getTable(); + } + + public List getColumns() + { + return tableSchema.getColumns(); + } + + public ColumnSchema getColumn(String name) + { + return tableSchema.getColumns().stream() + .filter(columnMetadata -> columnMetadata.getName().equals(name)) + .collect(toOptional()) + .orElseThrow(() -> new IllegalArgumentException("Invalid column name: " + name)); + } +} diff --git a/core/trino-main/src/test/java/io/trino/metadata/AbstractMockMetadata.java b/core/trino-main/src/test/java/io/trino/metadata/AbstractMockMetadata.java index 8700a21623ef..d1df73ff6e36 100644 --- a/core/trino-main/src/test/java/io/trino/metadata/AbstractMockMetadata.java +++ b/core/trino-main/src/test/java/io/trino/metadata/AbstractMockMetadata.java @@ -163,6 +163,12 @@ public Optional getInfo(Session session, TableHandle handle) throw new UnsupportedOperationException(); } + @Override + public TableSchema getTableSchema(Session session, TableHandle tableHandle) + { + throw new UnsupportedOperationException(); + } + @Override public TableMetadata getTableMetadata(Session session, TableHandle tableHandle) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/ColumnMetadata.java b/core/trino-spi/src/main/java/io/trino/spi/connector/ColumnMetadata.java index d89af0f487ab..4262a8546614 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/ColumnMetadata.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/ColumnMetadata.java @@ -135,6 +135,12 @@ public Map getProperties() return properties; } + public ColumnSchema getColumnSchema() + { + return ColumnSchema.builder(this) + .build(); + } + @Override public String toString() { diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/ColumnSchema.java b/core/trino-spi/src/main/java/io/trino/spi/connector/ColumnSchema.java new file mode 100644 index 000000000000..e3b19aa8aa1a --- /dev/null +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/ColumnSchema.java @@ -0,0 +1,135 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.spi.connector; + +import io.trino.spi.type.Type; + +import java.util.Objects; + +import static io.trino.spi.connector.SchemaUtil.checkNotEmpty; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +public final class ColumnSchema +{ + private final String name; + private final Type type; + private final boolean hidden; + + private ColumnSchema(String name, Type type, boolean hidden) + { + checkNotEmpty(name, "name"); + requireNonNull(type, "type is null"); + + this.name = name.toLowerCase(ENGLISH); + this.type = type; + this.hidden = hidden; + } + + public String getName() + { + return name; + } + + public Type getType() + { + return type; + } + + public boolean isHidden() + { + return hidden; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ColumnSchema that = (ColumnSchema) o; + return hidden == that.hidden + && name.equals(that.name) + && type.equals(that.type); + } + + @Override + public int hashCode() + { + return Objects.hash(name, type, hidden); + } + + @Override + public String toString() + { + return new StringBuilder("ColumnBasicMetadata{") + .append("name='").append(name).append('\'') + .append(", type=").append(type) + .append(", hidden=").append(hidden) + .append('}') + .toString(); + } + + public static Builder builder() + { + return new Builder(); + } + + public static Builder builder(ColumnMetadata columnMetadata) + { + return new Builder(columnMetadata); + } + + public static class Builder + { + private String name; + private Type type; + private boolean hidden; + + private Builder() {} + + private Builder(ColumnMetadata columnMetadata) + { + this.name = columnMetadata.getName(); + this.type = columnMetadata.getType(); + this.hidden = columnMetadata.isHidden(); + } + + public Builder setName(String name) + { + this.name = requireNonNull(name, "name is null"); + return this; + } + + public Builder setType(Type type) + { + this.type = requireNonNull(type, "type is null"); + return this; + } + + public Builder setHidden(boolean hidden) + { + this.hidden = hidden; + return this; + } + + public ColumnSchema build() + { + return new ColumnSchema(name, type, hidden); + } + } +} diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java index b45c94d916f7..f6bcd3b0c43c 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java @@ -161,6 +161,17 @@ default Optional getCommonPartitioningHandle(Connec return Optional.empty(); } + /** + * Return table schema definition for the specified table handle. + * This method is useful when getting full table metadata is expensive. + * + * @throws RuntimeException if table handle is no longer valid + */ + default ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle table) + { + return getTableMetadata(session, table).getTableSchema(); + } + /** * Return the metadata for the specified table handle. * diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorTableMetadata.java b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorTableMetadata.java index d35e8894a53f..cb1963095098 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorTableMetadata.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorTableMetadata.java @@ -21,6 +21,7 @@ import static java.util.Collections.emptyMap; import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toUnmodifiableList; public class ConnectorTableMetadata { @@ -71,6 +72,15 @@ public Optional getComment() return comment; } + public ConnectorTableSchema getTableSchema() + { + return new ConnectorTableSchema( + table, + columns.stream() + .map(ColumnMetadata::getColumnSchema) + .collect(toUnmodifiableList())); + } + @Override public String toString() { diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorTableSchema.java b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorTableSchema.java new file mode 100644 index 000000000000..afa9694ecb66 --- /dev/null +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorTableSchema.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.spi.connector; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public class ConnectorTableSchema +{ + private final SchemaTableName table; + private final List columns; + + public ConnectorTableSchema(SchemaTableName table, List columns) + { + requireNonNull(table, "table is null"); + requireNonNull(columns, "columns is null"); + + this.table = table; + this.columns = List.copyOf(columns); + } + + public SchemaTableName getTable() + { + return table; + } + + public List getColumns() + { + return columns; + } + + @Override + public String toString() + { + return new StringBuilder("ConnectorTableSchema{") + .append("table=").append(table) + .append(", columns=").append(columns) + .append('}') + .toString(); + } +} diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java index 6c3e5a6257aa..1bb31955d5dc 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java @@ -35,6 +35,7 @@ import io.trino.spi.connector.ConnectorTableLayoutResult; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.ConnectorTableProperties; +import io.trino.spi.connector.ConnectorTableSchema; import io.trino.spi.connector.ConnectorViewDefinition; import io.trino.spi.connector.Constraint; import io.trino.spi.connector.ConstraintApplicationResult; @@ -218,6 +219,14 @@ public Optional getSystemTable(ConnectorSession session, SchemaTabl } } + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle table) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + return delegate.getTableSchema(session, table); + } + } + @Override public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table) { From 13a73dca666df0c0346978030d00710b5e136858 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Kokosi=C5=84ski?= Date: Wed, 7 Apr 2021 10:08:46 +0200 Subject: [PATCH 060/146] Use ConnectorTableSchema in engine This covers only SELECT statement execution path. --- .../trino/sql/analyzer/StatementAnalyzer.java | 18 ++++++++++-------- .../planner/DistributedExecutionPlanner.java | 6 +++--- .../io/trino/sql/planner/InputExtractor.java | 2 +- .../planner/planprinter/TableInfoSupplier.java | 6 +++--- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java index 960ce3e9c49d..6cd88abb20c8 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java @@ -33,6 +33,7 @@ import io.trino.metadata.ResolvedFunction; import io.trino.metadata.TableHandle; import io.trino.metadata.TableMetadata; +import io.trino.metadata.TableSchema; import io.trino.security.AccessControl; import io.trino.security.AllowAllAccessControl; import io.trino.security.ViewAccessControl; @@ -41,6 +42,7 @@ import io.trino.spi.connector.CatalogSchemaName; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ColumnSchema; import io.trino.spi.connector.ConnectorMaterializedViewDefinition; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.ConnectorViewDefinition; @@ -409,20 +411,20 @@ protected Scope visitInsert(Insert insert, Optional scope) throw semanticException(NOT_SUPPORTED, insert, "Insert into table with a row filter is not supported"); } - TableMetadata tableMetadata = metadata.getTableMetadata(session, targetTableHandle.get()); + TableSchema tableSchema = metadata.getTableSchema(session, targetTableHandle.get()); - List columns = tableMetadata.getColumns().stream() + List columns = tableSchema.getColumns().stream() .filter(column -> !column.isHidden()) .collect(toImmutableList()); - for (ColumnMetadata column : columns) { + for (ColumnSchema column : columns) { if (!accessControl.getColumnMasks(session.toSecurityContext(), targetTable, column.getName(), column.getType()).isEmpty()) { throw semanticException(NOT_SUPPORTED, insert, "Insert into table with column masks is not supported"); } } List tableColumns = columns.stream() - .map(ColumnMetadata::getName) + .map(ColumnSchema::getName) .collect(toImmutableList()); // analyze target table layout, table columns should contain all partition columns @@ -461,7 +463,7 @@ protected Scope visitInsert(Insert insert, Optional scope) newTableLayout)); List tableTypes = insertColumns.stream() - .map(insertColumn -> tableMetadata.getColumn(insertColumn).getType()) + .map(insertColumn -> tableSchema.getColumn(insertColumn).getType()) .collect(toImmutableList()); List queryTypes = queryScope.getRelationType().getVisibleFields().stream() @@ -481,7 +483,7 @@ protected Scope visitInsert(Insert insert, Optional scope) targetTable, Optional.empty(), Optional.of(insertColumns.stream() - .map(insertColumn -> new Column(insertColumn, tableMetadata.getColumn(insertColumn).getType().toString())) + .map(insertColumn -> new Column(insertColumn, tableSchema.getColumn(insertColumn).getType().toString())) .collect(toImmutableList()))); return createAndAssignScope(insert, scope, Field.newUnqualified("rows", BIGINT)); @@ -1270,12 +1272,12 @@ protected Scope visitTable(Table table, Optional scope) } throw semanticException(TABLE_NOT_FOUND, table, "Table '%s' does not exist", name); } - TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle.get()); + TableSchema tableSchema = metadata.getTableSchema(session, tableHandle.get()); Map columnHandles = metadata.getColumnHandles(session, tableHandle.get()); // TODO: discover columns lazily based on where they are needed (to support connectors that can't enumerate all tables) ImmutableList.Builder fields = ImmutableList.builder(); - for (ColumnMetadata column : tableMetadata.getColumns()) { + for (ColumnSchema column : tableSchema.getColumns()) { Field field = Field.newQualified( table.getName(), Optional.of(column.getName()), diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/DistributedExecutionPlanner.java b/core/trino-main/src/main/java/io/trino/sql/planner/DistributedExecutionPlanner.java index 5f130f4ed359..0ca6bc3844d0 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/DistributedExecutionPlanner.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/DistributedExecutionPlanner.java @@ -19,8 +19,8 @@ import io.trino.Session; import io.trino.execution.TableInfo; import io.trino.metadata.Metadata; -import io.trino.metadata.TableMetadata; import io.trino.metadata.TableProperties; +import io.trino.metadata.TableSchema; import io.trino.operator.StageExecutionDescriptor; import io.trino.server.DynamicFilterService; import io.trino.spi.connector.DynamicFilter; @@ -149,9 +149,9 @@ private StageExecutionPlan doPlan(SubPlan root, Session session, ImmutableList.B private TableInfo getTableInfo(TableScanNode node, Session session) { - TableMetadata tableMetadata = metadata.getTableMetadata(session, node.getTable()); + TableSchema tableSchema = metadata.getTableSchema(session, node.getTable()); TableProperties tableProperties = metadata.getTableProperties(session, node.getTable()); - return new TableInfo(tableMetadata.getQualifiedName(), tableProperties.getPredicate()); + return new TableInfo(tableSchema.getQualifiedName(), tableProperties.getPredicate()); } private final class Visitor diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/InputExtractor.java b/core/trino-main/src/main/java/io/trino/sql/planner/InputExtractor.java index 39ea0222ad4a..8db19fcf1447 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/InputExtractor.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/InputExtractor.java @@ -63,7 +63,7 @@ private static Column createColumn(ColumnMetadata columnMetadata) private Input createInput(Session session, TableHandle table, Set columns, PlanFragmentId fragmentId, PlanNodeId planNodeId) { - SchemaTableName schemaTable = metadata.getTableMetadata(session, table).getTable(); + SchemaTableName schemaTable = metadata.getTableSchema(session, table).getTable(); Optional inputMetadata = metadata.getInfo(session, table); return new Input(table.getCatalogName().getCatalogName(), schemaTable.getSchemaName(), schemaTable.getTableName(), inputMetadata, ImmutableList.copyOf(columns), fragmentId, planNodeId); } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/TableInfoSupplier.java b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/TableInfoSupplier.java index 4d7b97a54841..5558b03e7c85 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/TableInfoSupplier.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/TableInfoSupplier.java @@ -16,8 +16,8 @@ import io.trino.Session; import io.trino.execution.TableInfo; import io.trino.metadata.Metadata; -import io.trino.metadata.TableMetadata; import io.trino.metadata.TableProperties; +import io.trino.metadata.TableSchema; import io.trino.sql.planner.plan.TableScanNode; import java.util.function.Function; @@ -39,8 +39,8 @@ public TableInfoSupplier(Metadata metadata, Session session) @Override public TableInfo apply(TableScanNode node) { - TableMetadata tableMetadata = metadata.getTableMetadata(session, node.getTable()); + TableSchema tableSchema = metadata.getTableSchema(session, node.getTable()); TableProperties tableProperties = metadata.getTableProperties(session, node.getTable()); - return new TableInfo(tableMetadata.getQualifiedName(), tableProperties.getPredicate()); + return new TableInfo(tableSchema.getQualifiedName(), tableProperties.getPredicate()); } } From 1586e600b34fa82cce5fcf00b99b2fcc7bfb326a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Kokosi=C5=84ski?= Date: Wed, 7 Apr 2021 10:08:47 +0200 Subject: [PATCH 061/146] Extract JdbcMetadata#getSchemaTableName method PhoenixMetadata was assuming that JdbcTableHandle always represents named relation which might not always be true. --- .../io/trino/plugin/jdbc/JdbcMetadata.java | 18 ++++++++++++------ .../trino/plugin/phoenix/PhoenixMetadata.java | 17 +++++++---------- .../trino/plugin/phoenix5/PhoenixMetadata.java | 17 +++++++---------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadata.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadata.java index fe0ce1314860..388fb3f021b0 100644 --- a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadata.java +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadata.java @@ -33,6 +33,7 @@ import io.trino.spi.connector.ConnectorTableHandle; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.ConnectorTableProperties; +import io.trino.spi.connector.ConnectorTableSchema; import io.trino.spi.connector.Constraint; import io.trino.spi.connector.ConstraintApplicationResult; import io.trino.spi.connector.JoinApplicationResult; @@ -521,15 +522,20 @@ public ConnectorTableMetadata getTableMetadata(ConnectorSession session, Connect { JdbcTableHandle handle = (JdbcTableHandle) table; - ImmutableList.Builder columnMetadata = ImmutableList.builder(); - for (JdbcColumnHandle column : jdbcClient.getColumns(session, handle)) { - columnMetadata.add(column.getColumnMetadata()); - } - SchemaTableName schemaTableName = handle.isNamedRelation() + return new ConnectorTableMetadata( + getSchemaTableName(handle), + jdbcClient.getColumns(session, handle).stream() + .map(JdbcColumnHandle::getColumnMetadata) + .collect(toImmutableList()), + jdbcClient.getTableProperties(session, handle)); + } + + protected static SchemaTableName getSchemaTableName(JdbcTableHandle handle) + { + return handle.isNamedRelation() ? handle.getRequiredNamedRelation().getSchemaTableName() // TODO (https://github.com/trinodb/trino/issues/6694) SchemaTableName should not be required for synthetic ConnectorTableHandle : new SchemaTableName("_generated", "_generated_query"); - return new ConnectorTableMetadata(schemaTableName, columnMetadata.build(), jdbcClient.getTableProperties(session, handle)); } @Override diff --git a/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixMetadata.java b/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixMetadata.java index 18bf5fcea8a5..1b90bc2cf959 100644 --- a/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixMetadata.java +++ b/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixMetadata.java @@ -84,18 +84,15 @@ public JdbcTableHandle getTableHandle(ConnectorSession session, SchemaTableName @Override public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table) - { - return getTableMetadata(session, table, false); - } - - private ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table, boolean rowkeyRequired) { JdbcTableHandle handle = (JdbcTableHandle) table; - List columnMetadata = phoenixClient.getColumns(session, handle).stream() - .filter(column -> rowkeyRequired || !ROWKEY.equalsIgnoreCase(column.getColumnName())) - .map(JdbcColumnHandle::getColumnMetadata) - .collect(toImmutableList()); - return new ConnectorTableMetadata(handle.getRequiredNamedRelation().getSchemaTableName(), columnMetadata, phoenixClient.getTableProperties(session, handle)); + return new ConnectorTableMetadata( + getSchemaTableName(handle), + phoenixClient.getColumns(session, handle).stream() + .filter(column -> !ROWKEY.equalsIgnoreCase(column.getColumnName())) + .map(JdbcColumnHandle::getColumnMetadata) + .collect(toImmutableList()), + phoenixClient.getTableProperties(session, handle)); } @Override diff --git a/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixMetadata.java b/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixMetadata.java index 856531a68c0a..28762cdea29c 100644 --- a/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixMetadata.java +++ b/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixMetadata.java @@ -84,18 +84,15 @@ public JdbcTableHandle getTableHandle(ConnectorSession session, SchemaTableName @Override public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table) - { - return getTableMetadata(session, table, false); - } - - private ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table, boolean rowkeyRequired) { JdbcTableHandle handle = (JdbcTableHandle) table; - List columnMetadata = phoenixClient.getColumns(session, handle).stream() - .filter(column -> rowkeyRequired || !ROWKEY.equalsIgnoreCase(column.getColumnName())) - .map(JdbcColumnHandle::getColumnMetadata) - .collect(toImmutableList()); - return new ConnectorTableMetadata(handle.getRequiredNamedRelation().getSchemaTableName(), columnMetadata, phoenixClient.getTableProperties(session, handle)); + return new ConnectorTableMetadata( + getSchemaTableName(handle), + phoenixClient.getColumns(session, handle).stream() + .filter(column -> !ROWKEY.equalsIgnoreCase(column.getColumnName())) + .map(JdbcColumnHandle::getColumnMetadata) + .collect(toImmutableList()), + phoenixClient.getTableProperties(session, handle)); } @Override From e36a7ae93d1990fa609e47c9a6fee07a977585fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Kokosi=C5=84ski?= Date: Wed, 7 Apr 2021 10:08:48 +0200 Subject: [PATCH 062/146] Avoid reading JDBC connector table properties This is achieved by implementing ConnectorMetadata::getTableSchema for JDBC connectors This covers only the SELECT statement. JDBC table properties are still accessed for INSERT statement. --- .../trino/plugin/jdbc/JdbcColumnHandle.java | 9 +++ .../io/trino/plugin/jdbc/JdbcMetadata.java | 12 +++ .../io/trino/plugin/jdbc/H2QueryRunner.java | 10 ++- .../plugin/jdbc/TestJdbcTableProperties.java | 79 +++++++++++++++++++ .../plugin/jdbc/TestingH2JdbcModule.java | 20 ++++- .../trino/plugin/phoenix/PhoenixMetadata.java | 25 +++++- .../plugin/phoenix5/PhoenixMetadata.java | 25 +++++- 7 files changed, 169 insertions(+), 11 deletions(-) create mode 100644 plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestJdbcTableProperties.java diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcColumnHandle.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcColumnHandle.java index 22b7231c1590..8f8cac58b577 100644 --- a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcColumnHandle.java +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcColumnHandle.java @@ -18,6 +18,7 @@ import com.google.common.base.Joiner; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ColumnSchema; import io.trino.spi.type.Type; import java.util.Objects; @@ -108,6 +109,14 @@ public ColumnMetadata getColumnMetadata() .build(); } + public ColumnSchema getColumnSchema() + { + return ColumnSchema.builder() + .setName(columnName) + .setType(columnType) + .build(); + } + @Override public boolean equals(Object obj) { diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadata.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadata.java index 388fb3f021b0..9c5d2e6fef35 100644 --- a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadata.java +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadata.java @@ -517,6 +517,18 @@ public ConnectorTableProperties getTableProperties(ConnectorSession session, Con return new ConnectorTableProperties(); } + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle table) + { + JdbcTableHandle handle = (JdbcTableHandle) table; + + return new ConnectorTableSchema( + getSchemaTableName(handle), + jdbcClient.getColumns(session, handle).stream() + .map(JdbcColumnHandle::getColumnSchema) + .collect(toImmutableList())); + } + @Override public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table) { diff --git a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/H2QueryRunner.java b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/H2QueryRunner.java index 485e73acb13f..b42ea26d1b19 100644 --- a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/H2QueryRunner.java +++ b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/H2QueryRunner.java @@ -14,6 +14,7 @@ package io.trino.plugin.jdbc; import com.google.common.collect.ImmutableList; +import com.google.inject.Module; import io.trino.Session; import io.trino.plugin.tpch.TpchPlugin; import io.trino.testing.DistributedQueryRunner; @@ -44,13 +45,18 @@ public static DistributedQueryRunner createH2QueryRunner(TpchTable... tables) public static DistributedQueryRunner createH2QueryRunner(Iterable> tables) throws Exception - { return createH2QueryRunner(tables, TestingH2JdbcModule.createProperties()); } public static DistributedQueryRunner createH2QueryRunner(Iterable> tables, Map properties) throws Exception + { + return createH2QueryRunner(tables, properties, new TestingH2JdbcModule()); + } + + public static DistributedQueryRunner createH2QueryRunner(Iterable> tables, Map properties, Module module) + throws Exception { DistributedQueryRunner queryRunner = null; try { @@ -61,7 +67,7 @@ public static DistributedQueryRunner createH2QueryRunner(Iterable> createSchema(properties, "tpch"); - queryRunner.installPlugin(new JdbcPlugin("base-jdbc", new TestingH2JdbcModule())); + queryRunner.installPlugin(new JdbcPlugin("base-jdbc", module)); queryRunner.createCatalog("jdbc", "base-jdbc", properties); copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), tables); diff --git a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestJdbcTableProperties.java b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestJdbcTableProperties.java new file mode 100644 index 000000000000..803f834e9990 --- /dev/null +++ b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestJdbcTableProperties.java @@ -0,0 +1,79 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.jdbc; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.trino.spi.connector.ConnectorSession; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.QueryRunner; +import io.trino.tpch.TpchTable; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static io.trino.plugin.jdbc.H2QueryRunner.createH2QueryRunner; +import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.fail; + +// Single-threaded because of shared mutable state, e.g. onGetTableProperties +@Test(singleThreaded = true) +public class TestJdbcTableProperties + extends AbstractTestQueryFramework +{ + private final Map properties = TestingH2JdbcModule.createProperties(); + private Runnable onGetTableProperties = () -> {}; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + TestingH2JdbcModule module = new TestingH2JdbcModule((config, connectionFactory) -> new TestingH2JdbcClient(config, connectionFactory) + { + @Override + public Map getTableProperties(ConnectorSession session, JdbcTableHandle tableHandle) + { + onGetTableProperties.run(); + return ImmutableMap.of(); + } + }); + return createH2QueryRunner(ImmutableList.copyOf(TpchTable.getTables()), properties, module); + } + + @BeforeTest + public void reset() + { + onGetTableProperties = () -> {}; + } + + @Test + public void testGetTablePropertiesIsNotCalledForSelect() + { + onGetTableProperties = () -> { fail("Unexpected call of: getTableProperties"); }; + assertUpdate("CREATE TABLE copy_of_nation AS SELECT * FROM nation", 25); + assertQuerySucceeds("SELECT * FROM copy_of_nation"); + assertQuerySucceeds("SELECT nationkey FROM copy_of_nation"); + } + + @Test + public void testGetTablePropertiesIsCalled() + { + AtomicInteger counter = new AtomicInteger(); + onGetTableProperties = () -> { counter.incrementAndGet(); }; + assertQuerySucceeds("SHOW CREATE TABLE nation"); + assertThat(counter.get()).isOne(); + } +} diff --git a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestingH2JdbcModule.java b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestingH2JdbcModule.java index 70bfa71b0d74..ffe2b5904d95 100644 --- a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestingH2JdbcModule.java +++ b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestingH2JdbcModule.java @@ -25,10 +25,23 @@ import java.util.concurrent.ThreadLocalRandom; import static java.lang.String.format; +import static java.util.Objects.requireNonNull; public class TestingH2JdbcModule implements Module { + private final TestingH2JdbcClientFactory testingH2JdbcClientFactory; + + public TestingH2JdbcModule() + { + this((config, connectionFactory) -> new TestingH2JdbcClient(config, connectionFactory)); + } + + public TestingH2JdbcModule(TestingH2JdbcClientFactory testingH2JdbcClientFactory) + { + this.testingH2JdbcClientFactory = requireNonNull(testingH2JdbcClientFactory, "testingH2JdbcClientFactory is null"); + } + @Override public void configure(Binder binder) {} @@ -36,7 +49,7 @@ public void configure(Binder binder) {} @ForBaseJdbc public JdbcClient provideJdbcClient(BaseJdbcConfig config, ConnectionFactory connectionFactory) { - return new TestingH2JdbcClient(config, connectionFactory); + return testingH2JdbcClientFactory.create(config, connectionFactory); } @Provides @@ -53,4 +66,9 @@ public static Map createProperties() .put("connection-url", format("jdbc:h2:mem:test%s;DB_CLOSE_DELAY=-1", System.nanoTime() + ThreadLocalRandom.current().nextLong())) .build(); } + + public interface TestingH2JdbcClientFactory + { + TestingH2JdbcClient create(BaseJdbcConfig config, ConnectionFactory connectionFactory); + } } diff --git a/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixMetadata.java b/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixMetadata.java index 1b90bc2cf959..cd09e203a156 100644 --- a/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixMetadata.java +++ b/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixMetadata.java @@ -30,6 +30,7 @@ import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.ConnectorTableHandle; import io.trino.spi.connector.ConnectorTableMetadata; +import io.trino.spi.connector.ConnectorTableSchema; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.security.TrinoPrincipal; import io.trino.spi.statistics.ComputedStatistics; @@ -82,19 +83,35 @@ public JdbcTableHandle getTableHandle(ConnectorSession session, SchemaTableName .orElse(null); } + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle table) + { + JdbcTableHandle handle = (JdbcTableHandle) table; + return new ConnectorTableSchema( + getSchemaTableName(handle), + getColumnMetadata(session, handle).stream() + .map(ColumnMetadata::getColumnSchema) + .collect(toImmutableList())); + } + @Override public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table) { JdbcTableHandle handle = (JdbcTableHandle) table; return new ConnectorTableMetadata( getSchemaTableName(handle), - phoenixClient.getColumns(session, handle).stream() - .filter(column -> !ROWKEY.equalsIgnoreCase(column.getColumnName())) - .map(JdbcColumnHandle::getColumnMetadata) - .collect(toImmutableList()), + getColumnMetadata(session, handle), phoenixClient.getTableProperties(session, handle)); } + private List getColumnMetadata(ConnectorSession session, JdbcTableHandle handle) + { + return phoenixClient.getColumns(session, handle).stream() + .filter(column -> !ROWKEY.equalsIgnoreCase(column.getColumnName())) + .map(JdbcColumnHandle::getColumnMetadata) + .collect(toImmutableList()); + } + @Override public void createSchema(ConnectorSession session, String schemaName, Map properties, TrinoPrincipal owner) { diff --git a/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixMetadata.java b/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixMetadata.java index 28762cdea29c..6087d455db96 100644 --- a/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixMetadata.java +++ b/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixMetadata.java @@ -30,6 +30,7 @@ import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.ConnectorTableHandle; import io.trino.spi.connector.ConnectorTableMetadata; +import io.trino.spi.connector.ConnectorTableSchema; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.security.TrinoPrincipal; import io.trino.spi.statistics.ComputedStatistics; @@ -82,19 +83,35 @@ public JdbcTableHandle getTableHandle(ConnectorSession session, SchemaTableName .orElse(null); } + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle table) + { + JdbcTableHandle handle = (JdbcTableHandle) table; + return new ConnectorTableSchema( + getSchemaTableName(handle), + getColumnMetadata(session, handle).stream() + .map(ColumnMetadata::getColumnSchema) + .collect(toImmutableList())); + } + @Override public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table) { JdbcTableHandle handle = (JdbcTableHandle) table; return new ConnectorTableMetadata( getSchemaTableName(handle), - phoenixClient.getColumns(session, handle).stream() - .filter(column -> !ROWKEY.equalsIgnoreCase(column.getColumnName())) - .map(JdbcColumnHandle::getColumnMetadata) - .collect(toImmutableList()), + getColumnMetadata(session, handle), phoenixClient.getTableProperties(session, handle)); } + private List getColumnMetadata(ConnectorSession session, JdbcTableHandle handle) + { + return phoenixClient.getColumns(session, handle).stream() + .filter(column -> !ROWKEY.equalsIgnoreCase(column.getColumnName())) + .map(JdbcColumnHandle::getColumnMetadata) + .collect(toImmutableList()); + } + @Override public void createSchema(ConnectorSession session, String schemaName, Map properties, TrinoPrincipal owner) { From 5954fcf32b0ef3efb3c86c9479e9051d54171654 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Wed, 7 Apr 2021 23:27:55 +0200 Subject: [PATCH 063/146] Improve message when _orc_acid_version not found or old Previously the message was suggesting Hive 3's ORC ACID tables are supported only after major compaction, which is not true. --- .../java/io/trino/plugin/hive/BackgroundHiveSplitLoader.java | 3 ++- .../io/trino/plugin/hive/TestBackgroundHiveSplitLoader.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/BackgroundHiveSplitLoader.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/BackgroundHiveSplitLoader.java index a5d02e504533..f00c5ceca8a7 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/BackgroundHiveSplitLoader.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/BackgroundHiveSplitLoader.java @@ -498,7 +498,8 @@ private ListenableFuture loadPartition(HivePartitionMetadata partition) : (directory.getCurrentDirectories().size() > 0 ? directory.getCurrentDirectories().get(0).getPath() : null); if (baseOrDeltaPath != null && AcidUtils.OrcAcidVersion.getAcidVersionFromMetaFile(baseOrDeltaPath, fs) < 2) { - throw new TrinoException(NOT_SUPPORTED, "Hive transactional tables are supported with Hive 3.0 and only after a major compaction has been run"); + throw new TrinoException(NOT_SUPPORTED, "Hive transactional tables are supported since Hive 3.0. " + + "If you have upgraded from an older version of Hive, make sure a major compaction has been run at least once after the upgrade."); } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestBackgroundHiveSplitLoader.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestBackgroundHiveSplitLoader.java index 0a074b5b3f46..cf01275aa918 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestBackgroundHiveSplitLoader.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestBackgroundHiveSplitLoader.java @@ -713,7 +713,8 @@ public void testHive2VersionedFullAcidTableFails() backgroundHiveSplitLoader.start(hiveSplitSource); assertThatThrownBy(() -> drain(hiveSplitSource)) .isInstanceOfSatisfying(TrinoException.class, e -> assertEquals(NOT_SUPPORTED.toErrorCode(), e.getErrorCode())) - .hasMessage("Hive transactional tables are supported with Hive 3.0 and only after a major compaction has been run"); + .hasMessage("Hive transactional tables are supported since Hive 3.0. " + + "If you have upgraded from an older version of Hive, make sure a major compaction has been run at least once after the upgrade."); deleteRecursively(tablePath, ALLOW_INSECURE); } From ff2afebdc0f2251f05ef5c101676c8937e152ae1 Mon Sep 17 00:00:00 2001 From: Lars Date: Tue, 6 Apr 2021 15:59:46 -0700 Subject: [PATCH 064/146] Fix LIMIT pushdown for the Phoenix connector LIMIT push down did not work since the HBase scan are created during the split phase based on the query it is transformed. With this commit splits are create from the transformed query. --- .../trino/plugin/phoenix/PhoenixClient.java | 24 +++++++++++++++---- .../plugin/phoenix/PhoenixSplitManager.java | 12 ++-------- .../trino/plugin/phoenix5/PhoenixClient.java | 24 +++++++++++++++---- .../plugin/phoenix5/PhoenixSplitManager.java | 12 ++-------- 4 files changed, 44 insertions(+), 28 deletions(-) diff --git a/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixClient.java b/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixClient.java index c8648ee054b4..74a5571d7208 100644 --- a/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixClient.java +++ b/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixClient.java @@ -243,15 +243,12 @@ protected Collection listSchemas(Connection connection) public PreparedStatement buildSql(ConnectorSession session, Connection connection, JdbcSplit split, JdbcTableHandle table, List columnHandles) throws SQLException { - PreparedQuery preparedQuery = prepareQuery( + PreparedStatement query = prepareStatement( session, connection, table, - Optional.empty(), columnHandles, - ImmutableMap.of(), Optional.of(split)); - PreparedStatement query = new QueryBuilder(this).prepareStatement(session, connection, preparedQuery); QueryPlan queryPlan = getQueryPlan((PhoenixPreparedStatement) query); ResultSet resultSet = getResultSet(((PhoenixSplit) split).getPhoenixInputSplit(), queryPlan); return new DelegatePreparedStatement(query) @@ -264,6 +261,25 @@ public ResultSet executeQuery() }; } + public PreparedStatement prepareStatement( + ConnectorSession session, + Connection connection, + JdbcTableHandle table, + List columns, + Optional split) + throws SQLException + { + PreparedQuery preparedQuery = prepareQuery( + session, + connection, + table, + Optional.empty(), + columns, + ImmutableMap.of(), + split); + return new QueryBuilder(this).prepareStatement(session, connection, preparedQuery); + } + @Override protected Optional> limitFunction() { diff --git a/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixSplitManager.java b/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixSplitManager.java index c20c7bdd71dc..9bd5b2ce6575 100644 --- a/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixSplitManager.java +++ b/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixSplitManager.java @@ -14,12 +14,9 @@ package io.trino.plugin.phoenix; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import io.airlift.log.Logger; import io.trino.plugin.jdbc.JdbcColumnHandle; import io.trino.plugin.jdbc.JdbcTableHandle; -import io.trino.plugin.jdbc.PreparedQuery; -import io.trino.plugin.jdbc.QueryBuilder; import io.trino.spi.HostAddress; import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorSession; @@ -83,17 +80,12 @@ public ConnectorSplitSource getSplits( List columns = tableHandle.getColumns() .map(columnSet -> columnSet.stream().map(JdbcColumnHandle.class::cast).collect(toList())) .orElseGet(() -> phoenixClient.getColumns(session, tableHandle)); - QueryBuilder queryBuilder = new QueryBuilder(phoenixClient); - PreparedQuery preparedQuery = queryBuilder.prepareQuery( + PhoenixPreparedStatement inputQuery = (PhoenixPreparedStatement) phoenixClient.prepareStatement( session, connection, - tableHandle.getRelationHandle(), - Optional.empty(), + tableHandle, columns, - ImmutableMap.of(), - tableHandle.getConstraint(), Optional.empty()); - PhoenixPreparedStatement inputQuery = (PhoenixPreparedStatement) queryBuilder.prepareStatement(session, connection, preparedQuery); List splits = getSplits(inputQuery).stream() .map(PhoenixInputSplit.class::cast) diff --git a/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixClient.java b/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixClient.java index e9154e25e706..97c88f038ceb 100644 --- a/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixClient.java +++ b/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixClient.java @@ -244,15 +244,12 @@ protected Collection listSchemas(Connection connection) public PreparedStatement buildSql(ConnectorSession session, Connection connection, JdbcSplit split, JdbcTableHandle table, List columnHandles) throws SQLException { - PreparedQuery preparedQuery = prepareQuery( + PreparedStatement query = prepareStatement( session, connection, table, - Optional.empty(), columnHandles, - ImmutableMap.of(), Optional.of(split)); - PreparedStatement query = new QueryBuilder(this).prepareStatement(session, connection, preparedQuery); QueryPlan queryPlan = getQueryPlan((PhoenixPreparedStatement) query); ResultSet resultSet = getResultSet(((PhoenixSplit) split).getPhoenixInputSplit(), queryPlan); return new DelegatePreparedStatement(query) @@ -265,6 +262,25 @@ public ResultSet executeQuery() }; } + public PreparedStatement prepareStatement( + ConnectorSession session, + Connection connection, + JdbcTableHandle table, + List columns, + Optional split) + throws SQLException + { + PreparedQuery preparedQuery = prepareQuery( + session, + connection, + table, + Optional.empty(), + columns, + ImmutableMap.of(), + split); + return new QueryBuilder(this).prepareStatement(session, connection, preparedQuery); + } + @Override protected Optional> limitFunction() { diff --git a/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixSplitManager.java b/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixSplitManager.java index ea04914b3d2b..878bf46ae273 100644 --- a/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixSplitManager.java +++ b/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixSplitManager.java @@ -14,12 +14,9 @@ package io.trino.plugin.phoenix5; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import io.airlift.log.Logger; import io.trino.plugin.jdbc.JdbcColumnHandle; import io.trino.plugin.jdbc.JdbcTableHandle; -import io.trino.plugin.jdbc.PreparedQuery; -import io.trino.plugin.jdbc.QueryBuilder; import io.trino.spi.HostAddress; import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorSession; @@ -83,17 +80,12 @@ public ConnectorSplitSource getSplits( List columns = tableHandle.getColumns() .map(columnSet -> columnSet.stream().map(JdbcColumnHandle.class::cast).collect(toList())) .orElseGet(() -> phoenixClient.getColumns(session, tableHandle)); - QueryBuilder queryBuilder = new QueryBuilder(phoenixClient); - PreparedQuery preparedQuery = queryBuilder.prepareQuery( + PhoenixPreparedStatement inputQuery = (PhoenixPreparedStatement) phoenixClient.prepareStatement( session, connection, - tableHandle.getRelationHandle(), - Optional.empty(), + tableHandle, columns, - ImmutableMap.of(), - tableHandle.getConstraint(), Optional.empty()); - PhoenixPreparedStatement inputQuery = (PhoenixPreparedStatement) queryBuilder.prepareStatement(session, connection, preparedQuery); List splits = getSplits(inputQuery).stream() .map(PhoenixInputSplit.class::cast) From a7d0f5423f175992cb94b2d3a71ed03bda983282 Mon Sep 17 00:00:00 2001 From: Lars Date: Tue, 6 Apr 2021 17:47:46 -0700 Subject: [PATCH 065/146] Allow topN pushdown for the Phoenix connector --- .../trino/plugin/phoenix/PhoenixClient.java | 30 +++++++++++++++++++ .../trino/plugin/phoenix5/PhoenixClient.java | 30 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixClient.java b/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixClient.java index 74a5571d7208..a7272dea7808 100644 --- a/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixClient.java +++ b/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixClient.java @@ -17,10 +17,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import io.trino.plugin.jdbc.BaseJdbcClient; +import io.trino.plugin.jdbc.BaseJdbcClient.TopNFunction; import io.trino.plugin.jdbc.ColumnMapping; import io.trino.plugin.jdbc.ConnectionFactory; import io.trino.plugin.jdbc.JdbcColumnHandle; import io.trino.plugin.jdbc.JdbcOutputTableHandle; +import io.trino.plugin.jdbc.JdbcSortItem; import io.trino.plugin.jdbc.JdbcSplit; import io.trino.plugin.jdbc.JdbcTableHandle; import io.trino.plugin.jdbc.JdbcTypeHandle; @@ -184,6 +186,7 @@ public class PhoenixClient extends BaseJdbcClient { private static final String ROWKEY = "ROWKEY"; + private static final long MAX_TOPN_LIMIT = 2000000; private final Configuration configuration; @@ -280,6 +283,33 @@ public PreparedStatement prepareStatement( return new QueryBuilder(this).prepareStatement(session, connection, preparedQuery); } + @Override + public boolean supportsTopN(ConnectorSession session, JdbcTableHandle handle, List sortOrder) + { + return true; + } + + @Override + protected Optional topNFunction() + { + return Optional.of((query, sortItems, limit) -> { + // TODO: Remove when this is fixed in Phoenix. + // Phoenix severely over-estimates the memory + // required to execute a topN query. + // https://issues.apache.org/jira/browse/PHOENIX-6436 + if (limit > MAX_TOPN_LIMIT) { + return query; + } + return TopNFunction.sqlStandard(this::quoted).apply(query, sortItems, limit); + }); + } + + @Override + public boolean isTopNLimitGuaranteed(ConnectorSession session) + { + return false; + } + @Override protected Optional> limitFunction() { diff --git a/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixClient.java b/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixClient.java index 97c88f038ceb..4049b261e178 100644 --- a/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixClient.java +++ b/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixClient.java @@ -17,10 +17,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import io.trino.plugin.jdbc.BaseJdbcClient; +import io.trino.plugin.jdbc.BaseJdbcClient.TopNFunction; import io.trino.plugin.jdbc.ColumnMapping; import io.trino.plugin.jdbc.ConnectionFactory; import io.trino.plugin.jdbc.JdbcColumnHandle; import io.trino.plugin.jdbc.JdbcOutputTableHandle; +import io.trino.plugin.jdbc.JdbcSortItem; import io.trino.plugin.jdbc.JdbcSplit; import io.trino.plugin.jdbc.JdbcTableHandle; import io.trino.plugin.jdbc.JdbcTypeHandle; @@ -185,6 +187,7 @@ public class PhoenixClient extends BaseJdbcClient { private static final String ROWKEY = "ROWKEY"; + private static final long MAX_TOPN_LIMIT = 2000000; private final Configuration configuration; @@ -281,6 +284,33 @@ public PreparedStatement prepareStatement( return new QueryBuilder(this).prepareStatement(session, connection, preparedQuery); } + @Override + public boolean supportsTopN(ConnectorSession session, JdbcTableHandle handle, List sortOrder) + { + return true; + } + + @Override + protected Optional topNFunction() + { + return Optional.of((query, sortItems, limit) -> { + // TODO: Remove when this is fixed in Phoenix. + // Phoenix severely over-estimates the memory + // required to execute a topN query. + // https://issues.apache.org/jira/browse/PHOENIX-6436 + if (limit > MAX_TOPN_LIMIT) { + return query; + } + return TopNFunction.sqlStandard(this::quoted).apply(query, sortItems, limit); + }); + } + + @Override + public boolean isTopNLimitGuaranteed(ConnectorSession session) + { + return false; + } + @Override protected Optional> limitFunction() { From da03da87be3b629dfe54eadfd459f36bae8d56bd Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Thu, 8 Apr 2021 09:22:38 +0200 Subject: [PATCH 066/146] Remove obsolete comment from PhoenixClient.isLimitGuaranteed Should have been removed in 3270b55e33d3dbd0b7b48e37f4080824a04113a1. --- .../src/main/java/io/trino/plugin/phoenix/PhoenixClient.java | 1 - .../src/main/java/io/trino/plugin/phoenix5/PhoenixClient.java | 1 - 2 files changed, 2 deletions(-) diff --git a/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixClient.java b/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixClient.java index a7272dea7808..0ab8755a3b2d 100644 --- a/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixClient.java +++ b/plugin/trino-phoenix/src/main/java/io/trino/plugin/phoenix/PhoenixClient.java @@ -319,7 +319,6 @@ protected Optional> limitFunction() @Override public boolean isLimitGuaranteed(ConnectorSession session) { - // Note that limit exceeding Integer.MAX_VALUE gets completely ignored. return false; } diff --git a/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixClient.java b/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixClient.java index 4049b261e178..1e34b86f4222 100644 --- a/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixClient.java +++ b/plugin/trino-phoenix5/src/main/java/io/trino/plugin/phoenix5/PhoenixClient.java @@ -320,7 +320,6 @@ protected Optional> limitFunction() @Override public boolean isLimitGuaranteed(ConnectorSession session) { - // Note that limit exceeding Integer.MAX_VALUE gets completely ignored. return false; } From f19bf745ae8d0f1580807d3fba50011d9bcd2b49 Mon Sep 17 00:00:00 2001 From: Ashhar Hasan Date: Sun, 4 Apr 2021 20:32:48 +0530 Subject: [PATCH 067/146] Enable TopN pushdown for char/varchar/text types in PostgreSQL --- .../plugin/jdbc/BaseJdbcConnectorTest.java | 3 +- .../plugin/postgresql/PostgreSqlClient.java | 36 ++++++++++++++++--- .../TestPostgreSqlConnectorTest.java | 3 +- .../testing/TestingConnectorBehavior.java | 1 + 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java index 031d01a0cf4c..50db9cada8ff 100644 --- a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java +++ b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java @@ -48,6 +48,7 @@ import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_EQUALITY; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_INEQUALITY; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_TOPN_PUSHDOWN; +import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_TOPN_PUSHDOWN_WITH_VARCHAR; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; @@ -214,7 +215,7 @@ public void testCaseSensitiveTopNPushdown() } // topN over varchar/char columns should only be pushed down if the remote systems's sort order matches Trino - boolean expectTopNPushdown = hasBehavior(SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_INEQUALITY); + boolean expectTopNPushdown = hasBehavior(SUPPORTS_TOPN_PUSHDOWN_WITH_VARCHAR); PlanMatchPattern topNOverTableScan = node(TopNNode.class, anyTree(node(TableScanNode.class))); try (TestTable testTable = new TestTable( diff --git a/plugin/trino-postgresql/src/main/java/io/trino/plugin/postgresql/PostgreSqlClient.java b/plugin/trino-postgresql/src/main/java/io/trino/plugin/postgresql/PostgreSqlClient.java index 32730bebc084..20f2668980dd 100644 --- a/plugin/trino-postgresql/src/main/java/io/trino/plugin/postgresql/PostgreSqlClient.java +++ b/plugin/trino-postgresql/src/main/java/io/trino/plugin/postgresql/PostgreSqlClient.java @@ -202,6 +202,7 @@ import static java.math.RoundingMode.UNNECESSARY; import static java.sql.DatabaseMetaData.columnNoNulls; import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.joining; public class PostgreSqlClient extends BaseJdbcClient @@ -686,10 +687,9 @@ public boolean supportsTopN(ConnectorSession session, JdbcTableHandle handle, Li for (JdbcSortItem sortItem : sortOrder) { Type sortItemType = sortItem.getColumn().getColumnType(); if (sortItemType instanceof CharType || sortItemType instanceof VarcharType) { - // PostgreSQL by default orders lowercase letters before uppercase, which is different from Trino - // NOTE: VarcharType also includes PostgreSQL enums - // TODO We could still push the sort down if we could inject a PostgreSQL-specific syntax for selecting a collation for given comparison. - return false; + if (!isCollatable(sortItem.getColumn())) { + return false; + } } } return true; @@ -698,7 +698,33 @@ public boolean supportsTopN(ConnectorSession session, JdbcTableHandle handle, Li @Override protected Optional topNFunction() { - return Optional.of(TopNFunction.sqlStandard(this::quoted)); + return Optional.of((query, sortItems, limit) -> { + String orderBy = sortItems.stream() + .map(sortItem -> { + String ordering = sortItem.getSortOrder().isAscending() ? "ASC" : "DESC"; + String nullsHandling = sortItem.getSortOrder().isNullsFirst() ? "NULLS FIRST" : "NULLS LAST"; + String collation = ""; + if (isCollatable(sortItem.getColumn())) { + collation = "COLLATE \"C\""; + } + return format("%s %s %s %s", quoted(sortItem.getColumn().getColumnName()), collation, ordering, nullsHandling); + }) + .collect(joining(", ")); + return format("%s ORDER BY %s LIMIT %d", query, orderBy, limit); + }); + } + + private boolean isCollatable(JdbcColumnHandle column) + { + if (column.getColumnType() instanceof CharType || column.getColumnType() instanceof VarcharType) { + String jdbcTypeName = column.getJdbcTypeHandle().getJdbcTypeName() + .orElseThrow(() -> new TrinoException(JDBC_ERROR, "Type name is missing: " + column.getJdbcTypeHandle())); + // Only char (internally named bpchar)/varchar/text are the built-in collatable types + return "bpchar".equals(jdbcTypeName) || "varchar".equals(jdbcTypeName) || "text".equals(jdbcTypeName); + } + + // non-textual types don't have the concept of collation + return false; } @Override diff --git a/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/TestPostgreSqlConnectorTest.java b/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/TestPostgreSqlConnectorTest.java index dff7e80ba6f8..161e8c274064 100644 --- a/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/TestPostgreSqlConnectorTest.java +++ b/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/TestPostgreSqlConnectorTest.java @@ -82,6 +82,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) return false; case SUPPORTS_TOPN_PUSHDOWN: + case SUPPORTS_TOPN_PUSHDOWN_WITH_VARCHAR: return true; case SUPPORTS_JOIN_PUSHDOWN: @@ -763,7 +764,7 @@ public void testLimitPushdown() // with TopN over numeric column assertThat(query("SELECT * FROM (SELECT regionkey FROM nation ORDER BY nationkey ASC LIMIT 10) LIMIT 5")).isFullyPushedDown(); // with TopN over varchar column - assertThat(query("SELECT * FROM (SELECT regionkey FROM nation ORDER BY name ASC LIMIT 10) LIMIT 5")).isNotFullyPushedDown(TopNNode.class); + assertThat(query("SELECT * FROM (SELECT regionkey FROM nation ORDER BY name ASC LIMIT 10) LIMIT 5")).isFullyPushedDown(); // LIMIT with JOIN assertThat(query(joinPushdownEnabled(getSession()), "" + diff --git a/testing/trino-testing/src/main/java/io/trino/testing/TestingConnectorBehavior.java b/testing/trino-testing/src/main/java/io/trino/testing/TestingConnectorBehavior.java index 42ba56404e37..f230c355c8fa 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/TestingConnectorBehavior.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/TestingConnectorBehavior.java @@ -26,6 +26,7 @@ public enum TestingConnectorBehavior SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_TOPN_PUSHDOWN, + SUPPORTS_TOPN_PUSHDOWN_WITH_VARCHAR(fallback -> fallback.test(SUPPORTS_TOPN_PUSHDOWN) && fallback.test(SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_INEQUALITY)), SUPPORTS_AGGREGATION_PUSHDOWN, From 217e473dcd13fcedd76df7271bbf7bc61512e3a4 Mon Sep 17 00:00:00 2001 From: Karol Sobczak Date: Fri, 2 Apr 2021 13:04:44 +0200 Subject: [PATCH 068/146] Reorder methods * Move createScopeForMaterializedView above createScopeForView * Move translateMaterializedViewColumns below usage --- .../trino/sql/analyzer/StatementAnalyzer.java | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java index 6cd88abb20c8..fbdd8ee13983 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java @@ -1421,18 +1421,18 @@ private Scope createScopeForCommonTableExpression(Table table, Optional s return createAndAssignScope(table, scope, fields); } - private Scope createScopeForView(Table table, QualifiedObjectName name, Optional scope, ConnectorViewDefinition view) + private Scope createScopeForMaterializedView(Table table, QualifiedObjectName name, Optional scope, ConnectorMaterializedViewDefinition view) { Statement statement = analysis.getStatement(); - if (statement instanceof CreateView) { - CreateView viewStatement = (CreateView) statement; + if (statement instanceof CreateMaterializedView) { + CreateMaterializedView viewStatement = (CreateMaterializedView) statement; QualifiedObjectName viewNameFromStatement = createQualifiedObjectName(session, viewStatement, viewStatement.getName()); if (viewStatement.isReplace() && viewNameFromStatement.equals(name)) { - throw semanticException(VIEW_IS_RECURSIVE, table, "Statement would create a recursive view"); + throw semanticException(VIEW_IS_RECURSIVE, table, "Statement would create a recursive materialized view"); } } if (analysis.hasTableInView(table)) { - throw semanticException(VIEW_IS_RECURSIVE, table, "View is recursive"); + throw semanticException(VIEW_IS_RECURSIVE, table, "Materialized View is recursive"); } Query query = parseView(view.getOriginalSql(), name, table); @@ -1441,13 +1441,14 @@ private Scope createScopeForView(Table table, QualifiedObjectName name, Optional RelationType descriptor = analyzeView(query, name, view.getCatalog(), view.getSchema(), view.getOwner(), table); analysis.unregisterTableForView(); - checkViewStaleness(view.getColumns(), descriptor.getVisibleFields(), name, table) - .ifPresent(explanation -> { throw semanticException(VIEW_IS_STALE, table, "View '%s' is stale or in invalid state: %s", name, explanation); }); + List viewColumns = translateMaterializedViewColumns(view.getColumns()); + checkViewStaleness(viewColumns, descriptor.getVisibleFields(), name, table) + .ifPresent(explanation -> { throw semanticException(VIEW_IS_STALE, table, "Materialized View '%s' is stale or in invalid state: %s", name, explanation); }); - // Derive the type of the view from the stored definition, not from the analysis of the underlying query. - // This is needed in case the underlying table(s) changed and the query in the view now produces types that - // are implicitly coercible to the declared view types. - List outputFields = view.getColumns().stream() + // Derive the type of the materialized view from the stored definition, not from the analysis of the underlying query. + // This is needed in case the underlying table(s) changed and the query in the materialized view now produces types that + // are implicitly coercible to the declared materialized view types. + List outputFields = viewColumns.stream() .map(column -> Field.newQualified( table.getName(), Optional.of(column.getName()), @@ -1474,18 +1475,18 @@ private List translateMaterializedViewColumn return viewColumns; } - private Scope createScopeForMaterializedView(Table table, QualifiedObjectName name, Optional scope, ConnectorMaterializedViewDefinition view) + private Scope createScopeForView(Table table, QualifiedObjectName name, Optional scope, ConnectorViewDefinition view) { Statement statement = analysis.getStatement(); - if (statement instanceof CreateMaterializedView) { - CreateMaterializedView viewStatement = (CreateMaterializedView) statement; + if (statement instanceof CreateView) { + CreateView viewStatement = (CreateView) statement; QualifiedObjectName viewNameFromStatement = createQualifiedObjectName(session, viewStatement, viewStatement.getName()); if (viewStatement.isReplace() && viewNameFromStatement.equals(name)) { - throw semanticException(VIEW_IS_RECURSIVE, table, "Statement would create a recursive materialized view"); + throw semanticException(VIEW_IS_RECURSIVE, table, "Statement would create a recursive view"); } } if (analysis.hasTableInView(table)) { - throw semanticException(VIEW_IS_RECURSIVE, table, "Materialized View is recursive"); + throw semanticException(VIEW_IS_RECURSIVE, table, "View is recursive"); } Query query = parseView(view.getOriginalSql(), name, table); @@ -1494,14 +1495,13 @@ private Scope createScopeForMaterializedView(Table table, QualifiedObjectName na RelationType descriptor = analyzeView(query, name, view.getCatalog(), view.getSchema(), view.getOwner(), table); analysis.unregisterTableForView(); - List viewColumns = translateMaterializedViewColumns(view.getColumns()); - checkViewStaleness(viewColumns, descriptor.getVisibleFields(), name, table) - .ifPresent(explanation -> { throw semanticException(VIEW_IS_STALE, table, "Materialized View '%s' is stale or in invalid state: %s", name, explanation); }); + checkViewStaleness(view.getColumns(), descriptor.getVisibleFields(), name, table) + .ifPresent(explanation -> { throw semanticException(VIEW_IS_STALE, table, "View '%s' is stale or in invalid state: %s", name, explanation); }); - // Derive the type of the materialized view from the stored definition, not from the analysis of the underlying query. - // This is needed in case the underlying table(s) changed and the query in the materialized view now produces types that - // are implicitly coercible to the declared materialized view types. - List outputFields = viewColumns.stream() + // Derive the type of the view from the stored definition, not from the analysis of the underlying query. + // This is needed in case the underlying table(s) changed and the query in the view now produces types that + // are implicitly coercible to the declared view types. + List outputFields = view.getColumns().stream() .map(column -> Field.newQualified( table.getName(), Optional.of(column.getName()), From ec5daf8aaae8e9d467b2859aec53b8909c2dd6f1 Mon Sep 17 00:00:00 2001 From: Karol Sobczak Date: Fri, 2 Apr 2021 13:22:24 +0200 Subject: [PATCH 069/146] Unify materialized view and view scope creation --- .../trino/sql/analyzer/StatementAnalyzer.java | 78 ++++++++----------- .../io/trino/testing/TestingMetadata.java | 22 ++++++ .../io/trino/sql/analyzer/TestAnalyzer.java | 25 ++++++ 3 files changed, 80 insertions(+), 45 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java index fbdd8ee13983..07fc30b832b6 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java @@ -1423,47 +1423,15 @@ private Scope createScopeForCommonTableExpression(Table table, Optional s private Scope createScopeForMaterializedView(Table table, QualifiedObjectName name, Optional scope, ConnectorMaterializedViewDefinition view) { - Statement statement = analysis.getStatement(); - if (statement instanceof CreateMaterializedView) { - CreateMaterializedView viewStatement = (CreateMaterializedView) statement; - QualifiedObjectName viewNameFromStatement = createQualifiedObjectName(session, viewStatement, viewStatement.getName()); - if (viewStatement.isReplace() && viewNameFromStatement.equals(name)) { - throw semanticException(VIEW_IS_RECURSIVE, table, "Statement would create a recursive materialized view"); - } - } - if (analysis.hasTableInView(table)) { - throw semanticException(VIEW_IS_RECURSIVE, table, "Materialized View is recursive"); - } - - Query query = parseView(view.getOriginalSql(), name, table); - analysis.registerNamedQuery(table, query); - analysis.registerTableForView(table); - RelationType descriptor = analyzeView(query, name, view.getCatalog(), view.getSchema(), view.getOwner(), table); - analysis.unregisterTableForView(); - - List viewColumns = translateMaterializedViewColumns(view.getColumns()); - checkViewStaleness(viewColumns, descriptor.getVisibleFields(), name, table) - .ifPresent(explanation -> { throw semanticException(VIEW_IS_STALE, table, "Materialized View '%s' is stale or in invalid state: %s", name, explanation); }); - - // Derive the type of the materialized view from the stored definition, not from the analysis of the underlying query. - // This is needed in case the underlying table(s) changed and the query in the materialized view now produces types that - // are implicitly coercible to the declared materialized view types. - List outputFields = viewColumns.stream() - .map(column -> Field.newQualified( - table.getName(), - Optional.of(column.getName()), - getViewColumnType(column, name, table), - false, - Optional.of(name), - Optional.of(column.getName()), - false)) - .collect(toImmutableList()); - - analysis.addRelationCoercion(table, outputFields.stream().map(Field::getType).toArray(Type[]::new)); - - analyzeFiltersAndMasks(table, name, Optional.empty(), outputFields, session.getIdentity().getUser()); - - return createAndAssignScope(table, scope, outputFields); + return createScopeForView( + table, + name, + scope, + view.getOriginalSql(), + view.getCatalog(), + view.getSchema(), + view.getOwner(), + translateMaterializedViewColumns(view.getColumns())); } private List translateMaterializedViewColumns(List materializedViewColumns) @@ -1476,6 +1444,19 @@ private List translateMaterializedViewColumn } private Scope createScopeForView(Table table, QualifiedObjectName name, Optional scope, ConnectorViewDefinition view) + { + return createScopeForView(table, name, scope, view.getOriginalSql(), view.getCatalog(), view.getSchema(), view.getOwner(), view.getColumns()); + } + + private Scope createScopeForView( + Table table, + QualifiedObjectName name, + Optional scope, + String originalSql, + Optional catalog, + Optional schema, + Optional owner, + List columns) { Statement statement = analysis.getStatement(); if (statement instanceof CreateView) { @@ -1485,23 +1466,30 @@ private Scope createScopeForView(Table table, QualifiedObjectName name, Optional throw semanticException(VIEW_IS_RECURSIVE, table, "Statement would create a recursive view"); } } + if (statement instanceof CreateMaterializedView) { + CreateMaterializedView viewStatement = (CreateMaterializedView) statement; + QualifiedObjectName viewNameFromStatement = createQualifiedObjectName(session, viewStatement, viewStatement.getName()); + if (viewStatement.isReplace() && viewNameFromStatement.equals(name)) { + throw semanticException(VIEW_IS_RECURSIVE, table, "Statement would create a recursive materialized view"); + } + } if (analysis.hasTableInView(table)) { throw semanticException(VIEW_IS_RECURSIVE, table, "View is recursive"); } - Query query = parseView(view.getOriginalSql(), name, table); + Query query = parseView(originalSql, name, table); analysis.registerNamedQuery(table, query); analysis.registerTableForView(table); - RelationType descriptor = analyzeView(query, name, view.getCatalog(), view.getSchema(), view.getOwner(), table); + RelationType descriptor = analyzeView(query, name, catalog, schema, owner, table); analysis.unregisterTableForView(); - checkViewStaleness(view.getColumns(), descriptor.getVisibleFields(), name, table) + checkViewStaleness(columns, descriptor.getVisibleFields(), name, table) .ifPresent(explanation -> { throw semanticException(VIEW_IS_STALE, table, "View '%s' is stale or in invalid state: %s", name, explanation); }); // Derive the type of the view from the stored definition, not from the analysis of the underlying query. // This is needed in case the underlying table(s) changed and the query in the view now produces types that // are implicitly coercible to the declared view types. - List outputFields = view.getColumns().stream() + List outputFields = columns.stream() .map(column -> Field.newQualified( table.getName(), Optional.of(column.getName()), diff --git a/core/trino-main/src/main/java/io/trino/testing/TestingMetadata.java b/core/trino-main/src/main/java/io/trino/testing/TestingMetadata.java index 8653db038c6f..2be2b942029a 100644 --- a/core/trino-main/src/main/java/io/trino/testing/TestingMetadata.java +++ b/core/trino-main/src/main/java/io/trino/testing/TestingMetadata.java @@ -23,6 +23,7 @@ import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.ConnectorInsertTableHandle; +import io.trino.spi.connector.ConnectorMaterializedViewDefinition; import io.trino.spi.connector.ConnectorMetadata; import io.trino.spi.connector.ConnectorNewTableLayout; import io.trino.spi.connector.ConnectorOutputMetadata; @@ -61,6 +62,7 @@ public class TestingMetadata { private final ConcurrentMap tables = new ConcurrentHashMap<>(); private final ConcurrentMap views = new ConcurrentHashMap<>(); + private final ConcurrentMap materializedViews = new ConcurrentHashMap<>(); @Override public List listSchemaNames(ConnectorSession session) @@ -218,6 +220,26 @@ public Optional getView(ConnectorSession session, Schem return Optional.ofNullable(views.get(viewName)); } + @Override + public void createMaterializedView(ConnectorSession session, SchemaTableName viewName, ConnectorMaterializedViewDefinition definition, boolean replace, boolean ignoreExisting) + { + if (replace) { + materializedViews.put(viewName, definition); + } + else if (materializedViews.putIfAbsent(viewName, definition) != null) { + if (ignoreExisting) { + return; + } + throw new TrinoException(ALREADY_EXISTS, "Materialized view already exists: " + viewName); + } + } + + @Override + public Optional getMaterializedView(ConnectorSession session, SchemaTableName viewName) + { + return Optional.ofNullable(materializedViews.get(viewName)); + } + @Override public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional layout) { diff --git a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java index 2037f1ae8c16..789e28d77fb3 100644 --- a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java +++ b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java @@ -15,6 +15,7 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import io.trino.Session; import io.trino.SystemSessionProperties; @@ -42,6 +43,7 @@ import io.trino.security.AllowAllAccessControl; import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorMaterializedViewDefinition; import io.trino.spi.connector.ConnectorMetadata; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.ConnectorTransactionHandle; @@ -2465,6 +2467,17 @@ public void testCreateRecursiveView() { assertFails("CREATE OR REPLACE VIEW v1 AS SELECT * FROM v1") .hasErrorCode(VIEW_IS_RECURSIVE); + assertFails("CREATE OR REPLACE VIEW mv1 AS SELECT * FROM mv1") + .hasErrorCode(VIEW_IS_RECURSIVE); + } + + @Test + public void testCreateMaterializedRecursiveView() + { + assertFails("CREATE OR REPLACE MATERIALIZED VIEW v1 AS SELECT * FROM v1") + .hasErrorCode(VIEW_IS_RECURSIVE); + assertFails("CREATE OR REPLACE MATERIALIZED VIEW mv1 AS SELECT * FROM mv1") + .hasErrorCode(VIEW_IS_RECURSIVE); } @Test @@ -3034,6 +3047,18 @@ public void setup() new ColumnMetadata("d", new ArrayType(DOUBLE)))), false)); + // materialized view referencing table in same schema + ConnectorMaterializedViewDefinition materializedViewData1 = new ConnectorMaterializedViewDefinition( + "select a from t1", + Optional.empty(), + Optional.of(TPCH_CATALOG), + Optional.of("s1"), + ImmutableList.of(new ConnectorMaterializedViewDefinition.Column("a", BIGINT.getTypeId())), + Optional.of("comment"), + Optional.of("user"), + ImmutableMap.of()); + inSetupTransaction(session -> metadata.createMaterializedView(session, new QualifiedObjectName(TPCH_CATALOG, "s1", "mv1"), materializedViewData1, false, true)); + // valid view referencing table in same schema ConnectorViewDefinition viewData1 = new ConnectorViewDefinition( "select a from t1", From 63199e39eed4b51b7fc10487ce2df23871ee2550 Mon Sep 17 00:00:00 2001 From: praveenkrishna Date: Wed, 31 Mar 2021 15:48:58 +0530 Subject: [PATCH 070/146] Minor cleanup in Analysis - Remove unused method --- .../src/main/java/io/trino/sql/analyzer/Analysis.java | 9 --------- .../java/io/trino/sql/analyzer/ExpressionAnalyzer.java | 1 - 2 files changed, 10 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java index 4b4253bf5c34..25e4e7131005 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java @@ -14,7 +14,6 @@ package io.trino.sql.analyzer; import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.HashMultimap; import com.google.common.collect.HashMultiset; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -122,9 +121,6 @@ public class Analysis // a map of users to the columns per table that they access private final Map>> tableColumnReferences = new LinkedHashMap<>(); - // Track referenced fields from source relation node - private final Multimap, Field> referencedFields = HashMultimap.create(); - private final Map, List> aggregates = new LinkedHashMap<>(); private final Map, List> orderByAggregates = new LinkedHashMap<>(); private final Map, GroupingSetAnalysis> groupingSets = new LinkedHashMap<>(); @@ -851,11 +847,6 @@ public void addEmptyColumnReferencesForTable(AccessControl accessControl, Identi tableColumnReferences.computeIfAbsent(accessControlInfo, k -> new LinkedHashMap<>()).computeIfAbsent(table, k -> new HashSet<>()); } - public void addReferencedFields(Multimap, Field> references) - { - referencedFields.putAll(references); - } - public Map>> getTableColumnReferences() { return tableColumnReferences; diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java index a73d722d24d5..0011ed705d0e 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java @@ -2030,7 +2030,6 @@ private static void updateAnalysis(Analysis analysis, ExpressionAnalyzer analyze analysis.addColumnReferences(analyzer.getColumnReferences()); analysis.addLambdaArgumentReferences(analyzer.getLambdaArgumentReferences()); analysis.addTableColumnReferences(accessControl, session.getIdentity(), analyzer.getTableColumnReferences()); - analysis.addReferencedFields(analyzer.getReferencedFields()); } public static ExpressionAnalyzer create( From 31f5a89326456c722c183508726b16f70358fac7 Mon Sep 17 00:00:00 2001 From: praveenkrishna Date: Wed, 24 Mar 2021 22:14:57 +0530 Subject: [PATCH 071/146] Capture column level dependency information --- .../java/io/trino/event/QueryMonitor.java | 17 +- .../io/trino/execution/CreateTableTask.java | 4 +- .../java/io/trino/sql/analyzer/Analysis.java | 87 ++++- .../sql/analyzer/ExpressionAnalyzer.java | 11 + .../java/io/trino/sql/analyzer/Output.java | 7 +- .../io/trino/sql/analyzer/OutputColumn.java | 73 +++++ .../trino/sql/analyzer/StatementAnalyzer.java | 75 ++++- .../io/trino/sql/analyzer/TestOutput.java | 14 +- .../trino/spi/eventlistener/ColumnDetail.java | 82 +++++ .../eventlistener/OutputColumnMetadata.java | 64 ++++ .../eventlistener/QueryOutputMetadata.java | 6 +- .../execution/TestEventListenerBasic.java | 300 +++++++++++++++++- 12 files changed, 689 insertions(+), 51 deletions(-) create mode 100644 core/trino-main/src/main/java/io/trino/sql/analyzer/OutputColumn.java create mode 100644 core/trino-spi/src/main/java/io/trino/spi/eventlistener/ColumnDetail.java create mode 100644 core/trino-spi/src/main/java/io/trino/spi/eventlistener/OutputColumnMetadata.java diff --git a/core/trino-main/src/main/java/io/trino/event/QueryMonitor.java b/core/trino-main/src/main/java/io/trino/event/QueryMonitor.java index b4375ba8d2b6..ae193965100e 100644 --- a/core/trino-main/src/main/java/io/trino/event/QueryMonitor.java +++ b/core/trino-main/src/main/java/io/trino/event/QueryMonitor.java @@ -44,6 +44,7 @@ import io.trino.operator.TaskStats; import io.trino.server.BasicQueryInfo; import io.trino.spi.QueryId; +import io.trino.spi.eventlistener.OutputColumnMetadata; import io.trino.spi.eventlistener.QueryCompletedEvent; import io.trino.spi.eventlistener.QueryContext; import io.trino.spi.eventlistener.QueryCreatedEvent; @@ -56,6 +57,7 @@ import io.trino.spi.eventlistener.StageCpuDistribution; import io.trino.spi.resourcegroups.QueryType; import io.trino.spi.resourcegroups.ResourceGroupId; +import io.trino.sql.analyzer.Analysis; import io.trino.sql.planner.PlanFragment; import io.trino.sql.planner.plan.PlanFragmentId; import io.trino.sql.planner.plan.PlanNode; @@ -77,6 +79,7 @@ import java.util.stream.Collectors; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableSet.toImmutableSet; import static io.trino.execution.QueryState.QUEUED; import static io.trino.execution.StageInfo.getAllStages; import static io.trino.sql.planner.planprinter.PlanPrinter.textDistributedPlan; @@ -363,15 +366,21 @@ private static QueryIOMetadata getQueryIOMetadata(QueryInfo queryInfo) .map(TableFinishInfo.class::cast) .findFirst(); + Optional> outputColumnsMetadata = queryInfo.getOutput().get().getColumns() + .map(columns -> columns.stream() + .map(column -> new OutputColumnMetadata( + column.getColumn().getName(), + column.getSourceColumns().stream() + .map(Analysis.SourceColumn::getColumnDetail) + .collect(toImmutableSet()))) + .collect(toImmutableList())); + output = Optional.of( new QueryOutputMetadata( queryInfo.getOutput().get().getCatalogName(), queryInfo.getOutput().get().getSchema(), queryInfo.getOutput().get().getTable(), - queryInfo.getOutput().get().getColumns() - .map(columns -> columns.stream() - .map(Column::getName) - .collect(toImmutableList())), + outputColumnsMetadata, tableFinishInfo.map(TableFinishInfo::getConnectorOutputMetadata), tableFinishInfo.map(TableFinishInfo::isJsonLengthLimitExceeded))); } diff --git a/core/trino-main/src/main/java/io/trino/execution/CreateTableTask.java b/core/trino-main/src/main/java/io/trino/execution/CreateTableTask.java index 6d5726eaad2e..0ea34af2dcf4 100644 --- a/core/trino-main/src/main/java/io/trino/execution/CreateTableTask.java +++ b/core/trino-main/src/main/java/io/trino/execution/CreateTableTask.java @@ -16,6 +16,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ListenableFuture; import io.trino.Session; import io.trino.connector.CatalogName; @@ -32,6 +33,7 @@ import io.trino.spi.type.Type; import io.trino.spi.type.TypeNotFoundException; import io.trino.sql.analyzer.Output; +import io.trino.sql.analyzer.OutputColumn; import io.trino.sql.tree.ColumnDefinition; import io.trino.sql.tree.CreateTable; import io.trino.sql.tree.Expression; @@ -248,7 +250,7 @@ else if (element instanceof LikeClause) { tableName.getSchemaName(), tableName.getObjectName(), Optional.of(tableMetadata.getColumns().stream() - .map(column -> new Column(column.getName(), column.getType().toString())) + .map(column -> new OutputColumn(new Column(column.getName(), column.getType().toString()), ImmutableSet.of())) .collect(toImmutableList())))); return immediateFuture(null); } diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java index 25e4e7131005..94f96381c0dd 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java @@ -13,15 +13,17 @@ */ package io.trino.sql.analyzer; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.HashMultiset; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.ListMultimap; import com.google.common.collect.Multimap; import com.google.common.collect.Multiset; import com.google.common.collect.Streams; -import io.trino.execution.Column; import io.trino.metadata.NewTableLayout; import io.trino.metadata.QualifiedObjectName; import io.trino.metadata.ResolvedFunction; @@ -32,6 +34,7 @@ import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.ConnectorTableMetadata; +import io.trino.spi.eventlistener.ColumnDetail; import io.trino.spi.eventlistener.ColumnInfo; import io.trino.spi.eventlistener.RoutineInfo; import io.trino.spi.eventlistener.TableInfo; @@ -191,6 +194,8 @@ public class Analysis // row id field for update/delete queries private final Map, FieldReference> rowIdField = new LinkedHashMap<>(); + private final Multimap originColumnDetails = ArrayListMultimap.create(); + private final Multimap, Field> fieldLineage = ArrayListMultimap.create(); public Analysis(@Nullable Statement root, Map, Expression> parameters, boolean isDescribe) { @@ -217,7 +222,7 @@ public Optional getTarget() }); } - public void setUpdateType(String updateType, QualifiedObjectName targetName, Optional targetTable, Optional> targetColumns) + public void setUpdateType(String updateType, QualifiedObjectName targetName, Optional
targetTable, Optional> targetColumns) { this.updateType = updateType; this.target = Optional.of(new UpdateTarget(targetName, targetTable, targetColumns)); @@ -957,6 +962,28 @@ public List getRoutines() .collect(toImmutableList()); } + public void addSourceColumns(Field field, Set sourceColumn) + { + originColumnDetails.putAll(field, sourceColumn); + } + + public Set getSourceColumns(Field field) + { + return ImmutableSet.copyOf(originColumnDetails.get(field)); + } + + public void addExpressionFields(Expression expression, Collection fields) + { + fieldLineage.putAll(NodeRef.of(expression), fields); + } + + public Set getExpressionSourceColumns(Expression expression) + { + return fieldLineage.get(NodeRef.of(expression)).stream() + .flatMap(field -> getSourceColumns(field).stream()) + .collect(toImmutableSet()); + } + public void setRowIdField(Table table, FieldReference field) { rowIdField.put(NodeRef.of(table), field); @@ -1479,6 +1506,56 @@ public Scope getAccessControlScope() } } + public static class SourceColumn + { + private final QualifiedObjectName tableName; + private final String columnName; + + @JsonCreator + public SourceColumn(@JsonProperty("tableName") QualifiedObjectName tableName, @JsonProperty("columnName") String columnName) + { + this.tableName = requireNonNull(tableName, "tableName is null"); + this.columnName = requireNonNull(columnName, "columnName is null"); + } + + @JsonProperty + public QualifiedObjectName getTableName() + { + return tableName; + } + + @JsonProperty + public String getColumnName() + { + return columnName; + } + + public ColumnDetail getColumnDetail() + { + return new ColumnDetail(tableName.getCatalogName(), tableName.getSchemaName(), tableName.getObjectName(), columnName); + } + + @Override + public int hashCode() + { + return Objects.hash(tableName, columnName); + } + + @Override + public boolean equals(Object obj) + { + if (obj == this) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + SourceColumn entry = (SourceColumn) obj; + return Objects.equals(tableName, entry.tableName) && + Objects.equals(columnName, entry.columnName); + } + } + private static class RoutineEntry { private final ResolvedFunction function; @@ -1505,9 +1582,9 @@ private static class UpdateTarget { private final QualifiedObjectName name; private final Optional
table; - private final Optional> columns; + private final Optional> columns; - public UpdateTarget(QualifiedObjectName name, Optional
table, Optional> columns) + public UpdateTarget(QualifiedObjectName name, Optional
table, Optional> columns) { this.name = requireNonNull(name, "name is null"); this.table = requireNonNull(table, "table is null"); @@ -1524,7 +1601,7 @@ public Optional
getTable() return table; } - public Optional> getColumns() + public Optional> getColumns() { return columns; } diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java index 0011ed705d0e..2eba8f272590 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java @@ -256,6 +256,7 @@ public class ExpressionAnalyzer private final CorrelationSupport correlationSupport; private final Function getPreanalyzedType; private final Function getResolvedWindow; + private final List sourceFields = new ArrayList<>(); public ExpressionAnalyzer( Metadata metadata, @@ -401,6 +402,11 @@ public Multimap, Field> getReferencedFields() return referencedFields; } + public List getSourceFields() + { + return sourceFields; + } + private class Visitor extends StackableAstVisitor { @@ -507,6 +513,8 @@ private Type handleResolvedField(Expression node, ResolvedField resolvedField, S tableColumnReferences.put(field.getOriginTable().get(), field.getOriginColumnName().get()); } + sourceFields.add(field); + fieldId.getRelationId() .getSourceNode() .ifPresent(source -> referencedFields.put(NodeRef.of(source), field)); @@ -1571,6 +1579,8 @@ else if (previousNode instanceof QuantifiedComparisonExpression) { scalarSubqueries.add(NodeRef.of(node)); } + sourceFields.add(queryScope.getRelationType().getFieldByIndex(0)); + Type type = getOnlyElement(queryScope.getRelationType().getVisibleFields()).getType(); return setExpressionType(node, type); } @@ -1973,6 +1983,7 @@ public static ExpressionAnalysis analyzeExpression( analyzer.analyze(expression, scope); updateAnalysis(analysis, analyzer, session, accessControl); + analysis.addExpressionFields(expression, analyzer.getSourceFields()); return new ExpressionAnalysis( analyzer.getExpressionTypes(), diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/Output.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/Output.java index 054bae97a718..abb4adcea85b 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/Output.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/Output.java @@ -16,7 +16,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; -import io.trino.execution.Column; import javax.annotation.concurrent.Immutable; @@ -32,14 +31,14 @@ public final class Output private final String catalogName; private final String schema; private final String table; - private final Optional> columns; + private final Optional> columns; @JsonCreator public Output( @JsonProperty("catalogName") String catalogName, @JsonProperty("schema") String schema, @JsonProperty("table") String table, - @JsonProperty("columns") Optional> columns) + @JsonProperty("columns") Optional> columns) { this.catalogName = requireNonNull(catalogName, "catalogName is null"); this.schema = requireNonNull(schema, "schema is null"); @@ -66,7 +65,7 @@ public String getTable() } @JsonProperty - public Optional> getColumns() + public Optional> getColumns() { return columns; } diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/OutputColumn.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/OutputColumn.java new file mode 100644 index 000000000000..7682e79be713 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/OutputColumn.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.analyzer; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableSet; +import io.trino.execution.Column; +import io.trino.sql.analyzer.Analysis.SourceColumn; + +import javax.annotation.concurrent.Immutable; + +import java.util.Objects; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +@Immutable +public final class OutputColumn +{ + private final Column column; + private final Set sourceColumns; + + @JsonCreator + public OutputColumn(@JsonProperty("column") Column column, @JsonProperty("sourceColumns") Set sourceColumns) + { + this.column = requireNonNull(column, "column is null"); + this.sourceColumns = ImmutableSet.copyOf(requireNonNull(sourceColumns, "sourceColumns is null")); + } + + @JsonProperty + public Column getColumn() + { + return column; + } + + @JsonProperty + public Set getSourceColumns() + { + return sourceColumns; + } + + @Override + public int hashCode() + { + return Objects.hash(column, sourceColumns); + } + + @Override + public boolean equals(Object obj) + { + if (obj == this) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + OutputColumn entry = (OutputColumn) obj; + return Objects.equals(column, entry.column) && + Objects.equals(sourceColumns, entry.sourceColumns); + } +} diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java index 07fc30b832b6..5f34dba6c0c1 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Streams; import io.trino.Session; import io.trino.connector.CatalogName; import io.trino.execution.Column; @@ -64,6 +65,7 @@ import io.trino.sql.analyzer.Analysis.GroupingSetAnalysis; import io.trino.sql.analyzer.Analysis.ResolvedWindow; import io.trino.sql.analyzer.Analysis.SelectExpression; +import io.trino.sql.analyzer.Analysis.SourceColumn; import io.trino.sql.analyzer.Analysis.UnnestAnalysis; import io.trino.sql.analyzer.Scope.AsteriskedIdentifierChainBasis; import io.trino.sql.parser.ParsingException; @@ -478,12 +480,20 @@ protected Scope visitInsert(Insert insert, Optional scope) Joiner.on(", ").join(queryTypes)); } + Stream columnStream = Streams.zip( + insertColumns.stream(), + tableTypes.stream() + .map(Type::toString), + Column::new); + analysis.setUpdateType( "INSERT", targetTable, Optional.empty(), - Optional.of(insertColumns.stream() - .map(insertColumn -> new Column(insertColumn, tableSchema.getColumn(insertColumn).getType().toString())) + Optional.of(Streams.zip( + columnStream, + queryScope.getRelationType().getVisibleFields().stream(), + (column, field) -> new OutputColumn(column, analysis.getSourceColumns(field))) .collect(toImmutableList()))); return createAndAssignScope(insert, scope, Field.newUnqualified("rows", BIGINT)); @@ -547,12 +557,20 @@ protected Scope visitRefreshMaterializedView(RefreshMaterializedView refreshMate "Query: [" + Joiner.on(", ").join(queryTypes) + "]"); } + Stream columns = Streams.zip( + insertColumns.stream(), + tableTypes.stream() + .map(Type::toString), + Column::new); + analysis.setUpdateType( "REFRESH MATERIALIZED VIEW", targetTable, Optional.empty(), - Optional.of(insertColumns.stream() - .map(insertColumn -> new Column(insertColumn, tableMetadata.getColumn(insertColumn).getType().toString())) + Optional.of(Streams.zip( + columns, + queryScope.getRelationType().getVisibleFields().stream(), + (column, field) -> new OutputColumn(column, analysis.getSourceColumns(field))) .collect(toImmutableList()))); return createAndAssignScope(refreshMaterializedView, scope, Field.newUnqualified("rows", BIGINT)); @@ -732,6 +750,7 @@ protected Scope visitCreateTableAsSelect(CreateTableAsSelect node, Optional columns = ImmutableList.builder(); // analyze target table columns and column aliases + ImmutableList.Builder outputColumns = ImmutableList.builder(); if (node.getColumnAliases().isPresent()) { validateColumnAliases(node.getColumnAliases().get(), queryScope.getRelationType().getVisibleFieldCount()); @@ -740,7 +759,9 @@ protected Scope visitCreateTableAsSelect(CreateTableAsSelect node, Optional new ColumnMetadata(field.getName().get(), field.getType())) .collect(toImmutableList())); + queryScope.getRelationType().getVisibleFields().stream() + .map(this::createOutputColumn) + .forEach(outputColumns::add); } // create target table metadata @@ -797,9 +821,7 @@ protected Scope visitCreateTableAsSelect(CreateTableAsSelect node, Optional new Column(column.getName(), column.getType().toString())) - .collect(toImmutableList()))); + Optional.of(outputColumns.build())); return createAndAssignScope(node, scope, Field.newUnqualified("rows", BIGINT)); } @@ -823,7 +845,7 @@ protected Scope visitCreateView(CreateView node, Optional scope) viewName, Optional.empty(), Optional.of(queryScope.getRelationType().getVisibleFields().stream() - .map(field -> new Column(field.getName().orElseThrow(), field.getType().toString())) + .map(this::createOutputColumn) .collect(toImmutableList()))); return createAndAssignScope(node, scope); @@ -1022,9 +1044,10 @@ protected Scope visitCreateMaterializedView(CreateMaterializedView node, Optiona "CREATE MATERIALIZED VIEW", viewName, Optional.empty(), - Optional.of(queryScope.getRelationType().getVisibleFields().stream() - .map(field -> new Column(field.getName().orElseThrow(), field.getType().toString())) - .collect(toImmutableList()))); + Optional.of( + queryScope.getRelationType().getVisibleFields().stream() + .map(this::createOutputColumn) + .collect(toImmutableList()))); return createAndAssignScope(node, scope); } @@ -1290,6 +1313,7 @@ protected Scope visitTable(Table table, Optional scope) ColumnHandle columnHandle = columnHandles.get(column.getName()); checkArgument(columnHandle != null, "Unknown field %s", field); analysis.setColumn(field, columnHandle); + analysis.addSourceColumns(field, ImmutableSet.of(new SourceColumn(name, column.getName()))); } if (updateKind.isPresent()) { @@ -1504,6 +1528,7 @@ private Scope createScopeForView( analyzeFiltersAndMasks(table, name, Optional.empty(), outputFields, session.getIdentity().getUser()); + outputFields.forEach(field -> analysis.addSourceColumns(field, ImmutableSet.of(new SourceColumn(name, field.getName().orElseThrow())))); return createAndAssignScope(table, scope, outputFields); } @@ -1949,7 +1974,7 @@ protected Scope visitUpdate(Update update, Optional scope) tableName, Optional.of(table), Optional.of(updatedColumns.stream() - .map(column -> new Column(column.getName(), column.getType().toString())) + .map(column -> new OutputColumn(new Column(column.getName(), column.getType().toString()), ImmutableSet.of())) .collect(toImmutableList()))); return createAndAssignScope(update, scope, Field.newUnqualified("rows", BIGINT)); @@ -2520,7 +2545,9 @@ private Scope computeAndAssignOutputScope(QuerySpecification node, Optional parentScope) return scopeBuilder; } + + private OutputColumn createOutputColumn(Field field) + { + return new OutputColumn(new Column(field.getName().orElseThrow(), field.getType().toString()), analysis.getSourceColumns(field)); + } } private Session createViewSession(Optional catalog, Optional schema, Identity identity, SqlPath path) diff --git a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestOutput.java b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestOutput.java index 89d5c29d61b9..ef110ac23894 100644 --- a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestOutput.java +++ b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestOutput.java @@ -14,8 +14,11 @@ package io.trino.sql.analyzer; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import io.airlift.json.JsonCodec; import io.trino.execution.Column; +import io.trino.metadata.QualifiedObjectName; +import io.trino.sql.analyzer.Analysis.SourceColumn; import org.testng.annotations.Test; import java.util.Optional; @@ -29,7 +32,16 @@ public class TestOutput @Test public void testRoundTrip() { - Output expected = new Output("connectorId", "schema", "table", Optional.of(ImmutableList.of(new Column("column", "type")))); + Output expected = new Output( + "connectorId", + "schema", + "table", + Optional.of( + ImmutableList.of( + new OutputColumn( + new Column("column", "type"), + ImmutableSet.of( + new SourceColumn(QualifiedObjectName.valueOf("catalog.schema.table"), "column")))))); String json = codec.toJson(expected); Output actual = codec.fromJson(json); diff --git a/core/trino-spi/src/main/java/io/trino/spi/eventlistener/ColumnDetail.java b/core/trino-spi/src/main/java/io/trino/spi/eventlistener/ColumnDetail.java new file mode 100644 index 000000000000..8e4b2be7b70b --- /dev/null +++ b/core/trino-spi/src/main/java/io/trino/spi/eventlistener/ColumnDetail.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.spi.eventlistener; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class ColumnDetail +{ + private final String catalog; + private final String schema; + private final String table; + private final String columnName; + + public ColumnDetail(String catalog, String schema, String table, String columnName) + { + this.catalog = requireNonNull(catalog, "catalog is null"); + this.schema = requireNonNull(schema, "schema is null"); + this.table = requireNonNull(table, "table is null"); + this.columnName = requireNonNull(columnName, "columnName is null"); + } + + @JsonProperty + public String getCatalog() + { + return catalog; + } + + @JsonProperty + public String getSchema() + { + return schema; + } + + @JsonProperty + public String getTable() + { + return table; + } + + @JsonProperty + public String getColumnName() + { + return columnName; + } + + @Override + public int hashCode() + { + return Objects.hash(catalog, schema, table, columnName); + } + + @Override + public boolean equals(Object obj) + { + if (obj == this) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + ColumnDetail entry = (ColumnDetail) obj; + return Objects.equals(catalog, entry.catalog) && + Objects.equals(schema, entry.schema) && + Objects.equals(table, entry.table) && + Objects.equals(columnName, entry.columnName); + } +} diff --git a/core/trino-spi/src/main/java/io/trino/spi/eventlistener/OutputColumnMetadata.java b/core/trino-spi/src/main/java/io/trino/spi/eventlistener/OutputColumnMetadata.java new file mode 100644 index 000000000000..b7616e17e86c --- /dev/null +++ b/core/trino-spi/src/main/java/io/trino/spi/eventlistener/OutputColumnMetadata.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.spi.eventlistener; + +import com.fasterxml.jackson.annotation.JsonCreator; + +import java.util.Objects; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public class OutputColumnMetadata +{ + private final String columnName; + private final Set sourceColumns; + + @JsonCreator + public OutputColumnMetadata(String columnName, Set sourceColumns) + { + this.columnName = requireNonNull(columnName, "columnName is null"); + this.sourceColumns = requireNonNull(sourceColumns, "sourceColumns is null"); + } + + public String getColumnName() + { + return columnName; + } + + public Set getSourceColumns() + { + return sourceColumns; + } + + @Override + public int hashCode() + { + return Objects.hash(columnName, sourceColumns); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + OutputColumnMetadata other = (OutputColumnMetadata) obj; + return Objects.equals(columnName, other.columnName) && + Objects.equals(sourceColumns, other.sourceColumns); + } +} diff --git a/core/trino-spi/src/main/java/io/trino/spi/eventlistener/QueryOutputMetadata.java b/core/trino-spi/src/main/java/io/trino/spi/eventlistener/QueryOutputMetadata.java index 2be1614edf1f..a50545576d7c 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/eventlistener/QueryOutputMetadata.java +++ b/core/trino-spi/src/main/java/io/trino/spi/eventlistener/QueryOutputMetadata.java @@ -25,12 +25,12 @@ public class QueryOutputMetadata private final String catalogName; private final String schema; private final String table; - private final Optional> columns; + private final Optional> columns; private final Optional connectorOutputMetadata; private final Optional jsonLengthLimitExceeded; - public QueryOutputMetadata(String catalogName, String schema, String table, Optional> columns, Optional connectorOutputMetadata, Optional jsonLengthLimitExceeded) + public QueryOutputMetadata(String catalogName, String schema, String table, Optional> columns, Optional connectorOutputMetadata, Optional jsonLengthLimitExceeded) { this.catalogName = requireNonNull(catalogName, "catalogName is null"); this.schema = requireNonNull(schema, "schema is null"); @@ -59,7 +59,7 @@ public String getTable() } @JsonProperty - public Optional> getColumns() + public Optional> getColumns() { return columns; } diff --git a/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java b/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java index da5eb629f328..88bbec20e050 100644 --- a/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java +++ b/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java @@ -31,7 +31,9 @@ import io.trino.spi.connector.ConnectorMaterializedViewDefinition.Column; import io.trino.spi.connector.ConnectorViewDefinition; import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.eventlistener.ColumnDetail; import io.trino.spi.eventlistener.ColumnInfo; +import io.trino.spi.eventlistener.OutputColumnMetadata; import io.trino.spi.eventlistener.QueryCompletedEvent; import io.trino.spi.eventlistener.QueryCreatedEvent; import io.trino.spi.eventlistener.QueryFailureInfo; @@ -406,7 +408,11 @@ public void testReferencedTablesInCreateView() assertThat(event.getIoMetadata().getOutput().get().getSchema()).isEqualTo("default"); assertThat(event.getIoMetadata().getOutput().get().getTable()).isEqualTo("test_view"); assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) - .containsExactly("nationkey", "name", "regionkey", "comment"); + .containsExactly( + new OutputColumnMetadata("nationkey", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey"))), + new OutputColumnMetadata("name", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "name"))), + new OutputColumnMetadata("regionkey", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "regionkey"))), + new OutputColumnMetadata("comment", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "comment")))); List tables = event.getMetadata().getTables(); assertThat(tables).hasSize(1); @@ -433,7 +439,11 @@ public void testReferencedTablesInCreateMaterializedView() assertThat(event.getIoMetadata().getOutput().get().getSchema()).isEqualTo("default"); assertThat(event.getIoMetadata().getOutput().get().getTable()).isEqualTo("test_view"); assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) - .containsExactly("nationkey", "name", "regionkey", "comment"); + .containsExactly( + new OutputColumnMetadata("nationkey", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey"))), + new OutputColumnMetadata("name", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "name"))), + new OutputColumnMetadata("regionkey", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "regionkey"))), + new OutputColumnMetadata("comment", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "comment")))); List tables = event.getMetadata().getTables(); assertThat(tables).hasSize(1); @@ -490,10 +500,18 @@ public void testReferencedTablesWithRowFilter() public void testReferencedTablesWithColumnMask() throws Exception { - runQueryAndWaitForEvents("SELECT * FROM mock.default.test_table_with_column_mask", 2); + runQueryAndWaitForEvents("CREATE TABLE mock.default.create_table_with_referring_mask AS SELECT * FROM mock.default.test_table_with_column_mask", 2); QueryCompletedEvent event = getOnlyElement(generatedEvents.getQueryCompletedEvents()); + assertThat(event.getIoMetadata().getOutput().get().getCatalogName()).isEqualTo("mock"); + assertThat(event.getIoMetadata().getOutput().get().getSchema()).isEqualTo("default"); + assertThat(event.getIoMetadata().getOutput().get().getTable()).isEqualTo("create_table_with_referring_mask"); + assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) + .containsExactly( + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("mock", "default", "test_table_with_column_mask", "test_varchar"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("mock", "default", "test_table_with_column_mask", "test_bigint")))); + List tables = event.getMetadata().getTables(); assertThat(tables).hasSize(2); @@ -503,7 +521,7 @@ public void testReferencedTablesWithColumnMask() assertThat(table.getTable()).isEqualTo("orders"); assertThat(table.getAuthorization()).isEqualTo("user"); assertThat(table.isDirectlyReferenced()).isFalse(); - assertThat(table.getFilters()).hasSize(0); + assertThat(table.getFilters()).isEmpty(); assertThat(table.getColumns()).hasSize(1); ColumnInfo column = table.getColumns().get(0); @@ -674,33 +692,254 @@ public void testOutputStats() } @Test - public void testOutputColumnsForCreateTableAsSelect() + public void testOutputColumnsForSelect() + throws Exception + { + assertColumnLineage( + "SELECT name as test_varchar, nationkey as test_bigint FROM nation", + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "name"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey")))); + } + + @Test + public void testOutputColumnsForSelectWithConstantExpression() + throws Exception + { + assertColumnLineage( + "SELECT 'Trino' as test_varchar, nationkey as test_bigint FROM nation", + new OutputColumnMetadata("test_varchar", ImmutableSet.of()), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey")))); + } + + @Test + public void testOutputColumnsForCreateTableAsSelectAll() throws Exception { - runQueryAndWaitForEvents("CREATE TABLE mock.default.create_new_table AS SELECT name, nationkey FROM nation", 2); + runQueryAndWaitForEvents("CREATE TABLE mock.default.create_new_table AS SELECT * FROM nation", 2); QueryCompletedEvent event = getOnlyElement(generatedEvents.getQueryCompletedEvents()); assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) - .containsExactly("name", "nationkey"); + .containsExactly( + new OutputColumnMetadata("nationkey", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey"))), + new OutputColumnMetadata("name", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "name"))), + new OutputColumnMetadata("regionkey", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "regionkey"))), + new OutputColumnMetadata("comment", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "comment")))); + } + + @Test + public void testOutputColumnsForCreateTableAsSelectAllFromView() + throws Exception + { + runQueryAndWaitForEvents("CREATE TABLE mock.default.create_new_table AS SELECT * FROM mock.default.test_view", 2); + QueryCompletedEvent event = getOnlyElement(generatedEvents.getQueryCompletedEvents()); + assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) + .containsExactly( + new OutputColumnMetadata("test_column", ImmutableSet.of(new ColumnDetail("mock", "default", "test_view", "test_column")))); + } + + @Test + public void testOutputColumnsForCreateTableAsSelectAllFromMaterializedView() + throws Exception + { + runQueryAndWaitForEvents("CREATE TABLE mock.default.create_new_table AS SELECT * FROM mock.default.test_materialized_view", 2); + QueryCompletedEvent event = getOnlyElement(generatedEvents.getQueryCompletedEvents()); + assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) + .containsExactly( + new OutputColumnMetadata("test_column", ImmutableSet.of(new ColumnDetail("mock", "default", "test_materialized_view", "test_column")))); } @Test public void testOutputColumnsForCreateTableAsSelectWithAliasedColumn() throws Exception { - runQueryAndWaitForEvents("CREATE TABLE mock.default.create_new_table(aliased_bigint, aliased_varchar) AS SELECT name, nationkey as keynation FROM nation", 2); + runQueryAndWaitForEvents("CREATE TABLE mock.default.create_new_table(aliased_bigint, aliased_varchar) AS SELECT nationkey as keynation, concat(name, comment) FROM nation", 2); QueryCompletedEvent event = getOnlyElement(generatedEvents.getQueryCompletedEvents()); assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) - .containsExactly("aliased_bigint", "aliased_varchar"); + .containsExactly( + new OutputColumnMetadata("aliased_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey"))), + new OutputColumnMetadata("aliased_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "name"), new ColumnDetail("tpch", "tiny", "nation", "comment")))); + } + + @Test + public void testOutputColumnsWithClause() + throws Exception + { + assertColumnLineage( + "WITH w AS (SELECT * FROM nation) SELECT name as test_varchar, nationkey as test_bigint FROM w", + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "name"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey")))); + } + + @Test + public void testOutputColumnsWithWhere() + throws Exception + { + assertColumnLineage( + "SELECT name as test_varchar, nationkey as test_bigint FROM nation WHERE regionkey IS NULL", + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "name"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey")))); + } + + @Test + public void testOutputColumnsWithIfExpression() + throws Exception + { + assertColumnLineage( + "SELECT if (regionkey > 100, name, comment) as test_varchar, nationkey as test_bigint FROM nation", + new OutputColumnMetadata( + "test_varchar", + ImmutableSet.of( + new ColumnDetail("tpch", "tiny", "nation", "regionkey"), + new ColumnDetail("tpch", "tiny", "nation", "name"), + new ColumnDetail("tpch", "tiny", "nation", "comment"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey")))); + } + + @Test + public void testOutputColumnsWithCaseExpression() + throws Exception + { + assertColumnLineage( + "SELECT CASE WHEN regionkey = 100 THEN name WHEN regionkey = 1000 then comment ELSE CAST(regionkey AS VARCHAR) END as test_varchar, nationkey as test_bigint FROM nation", + new OutputColumnMetadata( + "test_varchar", + ImmutableSet.of( + new ColumnDetail("tpch", "tiny", "nation", "regionkey"), + new ColumnDetail("tpch", "tiny", "nation", "name"), + new ColumnDetail("tpch", "tiny", "nation", "comment"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey")))); + } + + @Test + public void testOutputColumnsWithLimit() + throws Exception + { + assertColumnLineage( + "SELECT name as test_varchar, nationkey as test_bigint FROM nation LIMIT 100", + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "name"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey")))); + } + + @Test + public void testOutputColumnsWithOrderBy() + throws Exception + { + assertColumnLineage( + "SELECT name as test_varchar, nationkey as test_bigint FROM nation ORDER BY comment", + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "name"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey")))); + } + + @Test + public void testOutputColumnsWithAggregation() + throws Exception + { + assertColumnLineage( + "SELECT max(orderstatus) as test_varchar, sum(totalprice) as test_bigint FROM orders GROUP BY custkey", + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "orders", "orderstatus"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "orders", "totalprice")))); + } + + @Test + public void testOutputColumnsWithAggregationWithFilter() + throws Exception + { + assertColumnLineage( + "SELECT max(orderstatus) FILTER(WHERE orderdate > DATE '2000-01-01') as test_varchar, sum(totalprice) as test_bigint FROM orders GROUP BY custkey", + new OutputColumnMetadata( + "test_varchar", + ImmutableSet.of( + new ColumnDetail("tpch", "tiny", "orders", "orderstatus"), + new ColumnDetail("tpch", "tiny", "orders", "orderdate"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "orders", "totalprice")))); + } + + @Test + public void testOutputColumnsWithAggregationAndHaving() + throws Exception + { + assertColumnLineage( + "SELECT min(orderstatus) as test_varchar, sum(totalprice) as test_bigint FROM orders GROUP BY custkey HAVING min(orderdate) > DATE '2000-01-01'", + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "orders", "orderstatus"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "orders", "totalprice")))); + } + + @Test + public void testOutputColumnsWithCountAll() + throws Exception + { + assertColumnLineage( + "SELECT orderstatus as test_varchar, count(*) as test_bigint FROM orders GROUP BY orderstatus", + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "orders", "orderstatus"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of())); + } + + @Test + public void testOutputColumnsWithWindowFunction() + throws Exception + { + assertColumnLineage( + "SELECT orderstatus as test_varchar, avg(totalprice) OVER (PARTITION BY custkey ORDER BY orderdate ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS test_bigint FROM orders", + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "orders", "orderstatus"))), + new OutputColumnMetadata( + "test_bigint", + ImmutableSet.of( + new ColumnDetail("tpch", "tiny", "orders", "totalprice"), + new ColumnDetail("tpch", "tiny", "orders", "custkey"), + new ColumnDetail("tpch", "tiny", "orders", "orderdate")))); + } + + @Test + public void testOutputColumnsWithPartialWindowClause() + throws Exception + { + assertColumnLineage( + "SELECT orderstatus as test_varchar, sum(totalprice) OVER (w ORDER BY orderdate ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS test_bigint FROM orders WINDOW w AS (PARTITION BY custkey)", + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "orders", "orderstatus"))), + new OutputColumnMetadata( + "test_bigint", + ImmutableSet.of( + new ColumnDetail("tpch", "tiny", "orders", "totalprice"), + new ColumnDetail("tpch", "tiny", "orders", "orderdate")))); + } + + @Test + public void testOutputColumnsWithWindowClause() + throws Exception + { + assertColumnLineage( + "SELECT orderstatus as test_varchar, sum(totalprice) OVER w AS test_bigint FROM orders WINDOW w AS (PARTITION BY custkey ORDER BY orderdate ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)", + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "orders", "orderstatus"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "orders", "totalprice")))); + } + + @Test + public void testOutputColumnsWithUnCorrelatedQueries() + throws Exception + { + assertColumnLineage( + "SELECT orderstatus as test_varchar, (SELECT nationkey FROM nation LIMIT 1) as test_bigint FROM orders", + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "orders", "orderstatus"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey")))); + } + + @Test + public void testOutputColumnsWithCorrelatedQueries() + throws Exception + { + assertColumnLineage( + "SELECT name as test_varchar, (SELECT sum(acctbal) FROM supplier WHERE supplier.nationkey=nation.nationkey) as test_bigint FROM nation", + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "name"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "supplier", "acctbal")))); } @Test public void testOutputColumnsForInsertingSingleColumn() throws Exception { - runQueryAndWaitForEvents("INSERT INTO mock.default.table_for_output(test_bigint) SELECT nationkey FROM nation", 2); + runQueryAndWaitForEvents("INSERT INTO mock.default.table_for_output(test_bigint) SELECT nationkey + 1 as test_bigint FROM nation", 2); QueryCompletedEvent event = getOnlyElement(generatedEvents.getQueryCompletedEvents()); assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) - .containsExactly("test_bigint"); + .containsExactly(new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey")))); } @Test @@ -710,7 +949,9 @@ public void testOutputColumnsForInsertingAliasedColumn() runQueryAndWaitForEvents("INSERT INTO mock.default.table_for_output(test_varchar, test_bigint) SELECT name as aliased_name, nationkey as aliased_varchar FROM nation", 2); QueryCompletedEvent event = getOnlyElement(generatedEvents.getQueryCompletedEvents()); assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) - .containsExactly("test_varchar", "test_bigint"); + .containsExactly( + new OutputColumnMetadata("test_varchar", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "name"))), + new OutputColumnMetadata("test_bigint", ImmutableSet.of(new ColumnDetail("tpch", "tiny", "nation", "nationkey")))); } @Test @@ -720,7 +961,7 @@ public void testOutputColumnsForUpdatingAllColumns() runQueryAndWaitForEvents("UPDATE mock.default.table_for_output SET test_varchar = 'reset', test_bigint = 1", 2); QueryCompletedEvent event = getOnlyElement(generatedEvents.getQueryCompletedEvents()); assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) - .containsExactly("test_varchar", "test_bigint"); + .containsExactly(new OutputColumnMetadata("test_varchar", ImmutableSet.of()), new OutputColumnMetadata("test_bigint", ImmutableSet.of())); } @Test @@ -730,7 +971,7 @@ public void testOutputColumnsForUpdatingSingleColumn() runQueryAndWaitForEvents("UPDATE mock.default.table_for_output SET test_varchar = 're-reset' WHERE test_bigint = 1", 2); QueryCompletedEvent event = getOnlyElement(generatedEvents.getQueryCompletedEvents()); assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) - .containsExactly("test_varchar"); + .containsExactly(new OutputColumnMetadata("test_varchar", ImmutableSet.of())); } @Test @@ -742,7 +983,8 @@ public void testCreateTable() assertThat(event.getIoMetadata().getOutput().get().getCatalogName()).isEqualTo("mock"); assertThat(event.getIoMetadata().getOutput().get().getSchema()).isEqualTo("default"); assertThat(event.getIoMetadata().getOutput().get().getTable()).isEqualTo("create_simple_table"); - assertThat(event.getIoMetadata().getOutput().get().getColumns().get()).containsExactly("test_column"); + assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) + .containsExactly(new OutputColumnMetadata("test_column", ImmutableSet.of())); } @Test @@ -755,6 +997,32 @@ public void testCreateTableLike() assertThat(event.getIoMetadata().getOutput().get().getSchema()).isEqualTo("default"); assertThat(event.getIoMetadata().getOutput().get().getTable()).isEqualTo("create_simple_table"); assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) - .containsExactly("test_column", "test_varchar", "test_bigint"); + .containsExactly( + new OutputColumnMetadata("test_column", ImmutableSet.of()), + new OutputColumnMetadata("test_varchar", ImmutableSet.of()), + new OutputColumnMetadata("test_bigint", ImmutableSet.of())); + } + + private void assertColumnLineage(String baseQuery, OutputColumnMetadata... outputColumnMetadata) + throws Exception + { + runQueryAndWaitForEvents("CREATE TABLE mock.default.create_new_table AS " + baseQuery, 2); + assertColumnMetadata(outputColumnMetadata); + + runQueryAndWaitForEvents("CREATE VIEW mock.default.create_new_view AS " + baseQuery, 2); + assertColumnMetadata(outputColumnMetadata); + + runQueryAndWaitForEvents("CREATE VIEW mock.default.create_new_materialized_view AS " + baseQuery, 2); + assertColumnMetadata(outputColumnMetadata); + + runQueryAndWaitForEvents("INSERT INTO mock.default.table_for_output(test_varchar, test_bigint) " + baseQuery, 2); + assertColumnMetadata(outputColumnMetadata); + } + + private void assertColumnMetadata(OutputColumnMetadata... outputColumnMetadata) + { + QueryCompletedEvent event = getOnlyElement(generatedEvents.getQueryCompletedEvents()); + assertThat(event.getIoMetadata().getOutput().get().getColumns().get()) + .containsExactly(outputColumnMetadata); } } From 978de7b965a4d0c04f4a2879e31dfe2768221a1d Mon Sep 17 00:00:00 2001 From: David Phillips Date: Tue, 23 Mar 2021 09:19:08 -0700 Subject: [PATCH 072/146] Rename OrcPageSource to RaptorPageSource --- .../plugin/raptor/legacy/storage/OrcStorageManager.java | 6 +++--- .../storage/{OrcPageSource.java => RaptorPageSource.java} | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) rename plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/{OrcPageSource.java => RaptorPageSource.java} (99%) diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcStorageManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcStorageManager.java index ffd2851f2754..dd4955f328bc 100644 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcStorageManager.java +++ b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcStorageManager.java @@ -45,7 +45,7 @@ import io.trino.plugin.raptor.legacy.metadata.ShardInfo; import io.trino.plugin.raptor.legacy.metadata.ShardRecorder; import io.trino.plugin.raptor.legacy.storage.OrcFileRewriter.OrcFileInfo; -import io.trino.plugin.raptor.legacy.storage.OrcPageSource.ColumnAdaptation; +import io.trino.plugin.raptor.legacy.storage.RaptorPageSource.ColumnAdaptation; import io.trino.spi.NodeManager; import io.trino.spi.Page; import io.trino.spi.TrinoException; @@ -280,14 +280,14 @@ public ConnectorPageSource getPageSource( UTC, systemMemoryUsage, INITIAL_BATCH_SIZE, - OrcPageSource::handleException); + RaptorPageSource::handleException); Optional shardRewriter = Optional.empty(); if (transactionId.isPresent()) { shardRewriter = Optional.of(createShardRewriter(transactionId.getAsLong(), bucketNumber, shardUuid)); } - return new OrcPageSource(shardRewriter, recordReader, columnAdaptations, dataSource, systemMemoryUsage); + return new RaptorPageSource(shardRewriter, recordReader, columnAdaptations, dataSource, systemMemoryUsage); } catch (IOException | RuntimeException e) { closeQuietly(dataSource); diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcPageSource.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java similarity index 99% rename from plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcPageSource.java rename to plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java index fb2628e64b9e..f363f4827d32 100644 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcPageSource.java +++ b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorPageSource.java @@ -46,7 +46,7 @@ import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; -public class OrcPageSource +public class RaptorPageSource implements UpdatablePageSource { private final Optional shardRewriter; @@ -61,7 +61,7 @@ public class OrcPageSource private boolean closed; - public OrcPageSource( + public RaptorPageSource( Optional shardRewriter, OrcRecordReader recordReader, List columnAdaptations, From a6972acaed1bf283502f8ee0c30539fcf1fe97c2 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Tue, 23 Mar 2021 09:20:58 -0700 Subject: [PATCH 073/146] Rename OrcStorageManager to RaptorStorageManager --- ...Manager.java => RaptorStorageManager.java} | 12 +++--- .../legacy/storage/ShardRecoveryManager.java | 2 +- .../raptor/legacy/storage/ShardStats.java | 2 +- .../raptor/legacy/storage/StorageModule.java | 4 +- .../raptor/legacy/TestRaptorConnector.java | 4 +- ...ger.java => TestRaptorStorageManager.java} | 40 +++++++++---------- .../legacy/storage/TestShardRecovery.java | 2 +- .../organization/TestShardCompactor.java | 8 ++-- 8 files changed, 37 insertions(+), 37 deletions(-) rename plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/{OrcStorageManager.java => RaptorStorageManager.java} (98%) rename plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/{TestOrcStorageManager.java => TestRaptorStorageManager.java} (94%) diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcStorageManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java similarity index 98% rename from plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcStorageManager.java rename to plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java index dd4955f328bc..ad634e55bce6 100644 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/OrcStorageManager.java +++ b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/RaptorStorageManager.java @@ -131,7 +131,7 @@ import static java.util.stream.Collectors.toList; import static org.joda.time.DateTimeZone.UTC; -public class OrcStorageManager +public class RaptorStorageManager implements StorageManager { private static final JsonCodec SHARD_DELTA_CODEC = jsonCodec(ShardDelta.class); @@ -157,7 +157,7 @@ public class OrcStorageManager private final ExecutorService commitExecutor; @Inject - public OrcStorageManager( + public RaptorStorageManager( NodeManager nodeManager, StorageService storageService, Optional backupStore, @@ -184,7 +184,7 @@ public OrcStorageManager( config.getMinAvailableSpace()); } - public OrcStorageManager( + public RaptorStorageManager( String nodeId, StorageService storageService, Optional backupStore, @@ -319,7 +319,7 @@ public StoragePageSink createStoragePageSink(long transactionId, OptionalInt buc if (checkSpace && storageService.getAvailableBytes() < minAvailableSpace.toBytes()) { throw new TrinoException(RAPTOR_LOCAL_DISK_FULL, "Local disk is full on node " + nodeId); } - return new OrcStoragePageSink(transactionId, columnIds, columnTypes, bucketNumber); + return new RaptorStoragePageSink(transactionId, columnIds, columnTypes, bucketNumber); } private ShardRewriter createShardRewriter(long transactionId, OptionalInt bucketNumber, UUID shardUuid) @@ -594,7 +594,7 @@ private static Map columnIdIndex(List columns) return uniqueIndex(columns, column -> Long.valueOf(column.getColumnName())); } - private class OrcStoragePageSink + private class RaptorStoragePageSink implements StoragePageSink { private final long transactionId; @@ -610,7 +610,7 @@ private class OrcStoragePageSink private OrcFileWriter writer; private UUID shardUuid; - public OrcStoragePageSink(long transactionId, List columnIds, List columnTypes, OptionalInt bucketNumber) + public RaptorStoragePageSink(long transactionId, List columnIds, List columnTypes, OptionalInt bucketNumber) { this.transactionId = transactionId; this.columnIds = ImmutableList.copyOf(requireNonNull(columnIds, "columnIds is null")); diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryManager.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryManager.java index c64a5b644e45..3124ca982c18 100644 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryManager.java +++ b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardRecoveryManager.java @@ -63,7 +63,7 @@ import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_CORRUPTION; import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_RECOVERY_ERROR; -import static io.trino.plugin.raptor.legacy.storage.OrcStorageManager.xxhash64; +import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.xxhash64; import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardStats.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardStats.java index 1f2bd5403d94..4909d56e2b8c 100644 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardStats.java +++ b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/ShardStats.java @@ -39,7 +39,7 @@ import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static io.trino.orc.OrcReader.INITIAL_BATCH_SIZE; import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR; -import static io.trino.plugin.raptor.legacy.storage.OrcStorageManager.toOrcFileType; +import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.toOrcFileType; import static java.lang.Double.isInfinite; import static java.lang.Double.isNaN; import static org.joda.time.DateTimeZone.UTC; diff --git a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageModule.java b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageModule.java index f476ee266403..5c0d0c13ccbd 100644 --- a/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageModule.java +++ b/plugin/trino-raptor-legacy/src/main/java/io/trino/plugin/raptor/legacy/storage/StorageModule.java @@ -49,7 +49,7 @@ public void configure(Binder binder) binder.bind(Ticker.class).toInstance(Ticker.systemTicker()); - binder.bind(StorageManager.class).to(OrcStorageManager.class).in(Scopes.SINGLETON); + binder.bind(StorageManager.class).to(RaptorStorageManager.class).in(Scopes.SINGLETON); binder.bind(StorageService.class).to(FileStorageService.class).in(Scopes.SINGLETON); binder.bind(ShardManager.class).to(DatabaseShardManager.class).in(Scopes.SINGLETON); binder.bind(ShardRecorder.class).to(DatabaseShardRecorder.class).in(Scopes.SINGLETON); @@ -69,7 +69,7 @@ public void configure(Binder binder) newExporter(binder).export(ShardRecoveryManager.class).withGeneratedName(); newExporter(binder).export(BackupManager.class).withGeneratedName(); - newExporter(binder).export(StorageManager.class).as(generator -> generator.generatedNameOf(OrcStorageManager.class)); + newExporter(binder).export(StorageManager.class).as(generator -> generator.generatedNameOf(RaptorStorageManager.class)); newExporter(binder).export(ShardCompactionManager.class).withGeneratedName(); newExporter(binder).export(ShardOrganizer.class).withGeneratedName(); newExporter(binder).export(ShardCompactor.class).withGeneratedName(); diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnector.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnector.java index f54237b68cd6..1108b04261b6 100644 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnector.java +++ b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/TestRaptorConnector.java @@ -64,7 +64,7 @@ import static io.trino.plugin.raptor.legacy.RaptorTableProperties.TEMPORAL_COLUMN_PROPERTY; import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.storage.TestOrcStorageManager.createOrcStorageManager; +import static io.trino.plugin.raptor.legacy.storage.TestRaptorStorageManager.createRaptorStorageManager; import static io.trino.spi.transaction.IsolationLevel.READ_COMMITTED; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.spi.type.DateType.DATE; @@ -98,7 +98,7 @@ public void setup() NodeManager nodeManager = new TestingNodeManager(); NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; ShardManager shardManager = createShardManager(dbi); - StorageManager storageManager = createOrcStorageManager(dbi, dataDir); + StorageManager storageManager = createRaptorStorageManager(dbi, dataDir); StorageManagerConfig config = new StorageManagerConfig(); connector = new RaptorConnector( new LifeCycleManager(ImmutableList.of(), null), diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestOrcStorageManager.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestRaptorStorageManager.java similarity index 94% rename from plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestOrcStorageManager.java rename to plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestRaptorStorageManager.java index 7140bd13592b..22478da37b5d 100644 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestOrcStorageManager.java +++ b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestRaptorStorageManager.java @@ -83,9 +83,9 @@ import static io.trino.metadata.MetadataManager.createTestMetadataManager; import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.storage.OrcStorageManager.xxhash64; import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.createReader; import static io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.octets; +import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.xxhash64; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.spi.type.BooleanType.BOOLEAN; import static io.trino.spi.type.DateType.DATE; @@ -110,7 +110,7 @@ import static org.testng.FileAssert.assertFile; @Test(singleThreaded = true) -public class TestOrcStorageManager +public class TestRaptorStorageManager { private static final ISOChronology UTC_CHRONOLOGY = ISOChronology.getInstanceUTC(); private static final DateTime EPOCH = new DateTime(0, UTC_CHRONOLOGY); @@ -175,7 +175,7 @@ public void tearDown() public void testWriter() throws Exception { - OrcStorageManager manager = createOrcStorageManager(); + RaptorStorageManager manager = createRaptorStorageManager(); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); @@ -249,7 +249,7 @@ public void testWriter() public void testReader() throws Exception { - OrcStorageManager manager = createOrcStorageManager(); + RaptorStorageManager manager = createRaptorStorageManager(); List columnIds = ImmutableList.of(2L, 4L, 6L, 7L, 8L, 9L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DATE, BOOLEAN, DOUBLE); @@ -323,7 +323,7 @@ public void testReader() public void testRewriter() throws Exception { - OrcStorageManager manager = createOrcStorageManager(); + RaptorStorageManager manager = createRaptorStorageManager(); long transactionId = TRANSACTION_ID; List columnIds = ImmutableList.of(3L, 7L); @@ -373,7 +373,7 @@ public void testWriterRollback() assertEquals(staging.list(), new String[] {}); // create a shard in staging - OrcStorageManager manager = createOrcStorageManager(); + RaptorStorageManager manager = createRaptorStorageManager(); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); @@ -510,7 +510,7 @@ public void testShardStatsDateTimestamp() @Test public void testMaxShardRows() { - OrcStorageManager manager = createOrcStorageManager(2, DataSize.of(2, MEGABYTE)); + RaptorStorageManager manager = createRaptorStorageManager(2, DataSize.of(2, MEGABYTE)); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); @@ -536,14 +536,14 @@ public void testMaxFileSize() .build(); // Set maxFileSize to 1 byte, so adding any page makes the StoragePageSink full - OrcStorageManager manager = createOrcStorageManager(20, DataSize.ofBytes(1)); + RaptorStorageManager manager = createRaptorStorageManager(20, DataSize.ofBytes(1)); StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); sink.appendPages(pages); assertTrue(sink.isFull()); } private static ConnectorPageSource getPageSource( - OrcStorageManager manager, + RaptorStorageManager manager, List columnIds, List columnTypes, UUID uuid, @@ -558,22 +558,22 @@ private static StoragePageSink createStoragePageSink(StorageManager manager, Lis return manager.createStoragePageSink(transactionId, OptionalInt.empty(), columnIds, columnTypes, false); } - private OrcStorageManager createOrcStorageManager() + private RaptorStorageManager createRaptorStorageManager() { - return createOrcStorageManager(MAX_SHARD_ROWS, MAX_FILE_SIZE); + return createRaptorStorageManager(MAX_SHARD_ROWS, MAX_FILE_SIZE); } - private OrcStorageManager createOrcStorageManager(int maxShardRows, DataSize maxFileSize) + private RaptorStorageManager createRaptorStorageManager(int maxShardRows, DataSize maxFileSize) { - return createOrcStorageManager(storageService, backupStore, recoveryManager, shardRecorder, maxShardRows, maxFileSize); + return createRaptorStorageManager(storageService, backupStore, recoveryManager, shardRecorder, maxShardRows, maxFileSize); } - public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary) + public static RaptorStorageManager createRaptorStorageManager(IDBI dbi, File temporary) { - return createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS); + return createRaptorStorageManager(dbi, temporary, MAX_SHARD_ROWS); } - public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary, int maxShardRows) + public static RaptorStorageManager createRaptorStorageManager(IDBI dbi, File temporary, int maxShardRows) { File directory = new File(temporary, "data"); StorageService storageService = new FileStorageService(directory); @@ -592,7 +592,7 @@ public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary shardManager, MISSING_SHARD_DISCOVERY, 10); - return createOrcStorageManager( + return createRaptorStorageManager( storageService, backupStore, recoveryManager, @@ -601,7 +601,7 @@ public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary MAX_FILE_SIZE); } - public static OrcStorageManager createOrcStorageManager( + public static RaptorStorageManager createRaptorStorageManager( StorageService storageService, Optional backupStore, ShardRecoveryManager recoveryManager, @@ -609,7 +609,7 @@ public static OrcStorageManager createOrcStorageManager( int maxShardRows, DataSize maxFileSize) { - return new OrcStorageManager( + return new RaptorStorageManager( CURRENT_NODE, storageService, backupStore, @@ -669,7 +669,7 @@ private List columnStats(List columnTypes, Object[]... rows) } List columnIds = list.build(); - OrcStorageManager manager = createOrcStorageManager(); + RaptorStorageManager manager = createRaptorStorageManager(); StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); sink.appendPages(rowPagesBuilder(columnTypes).rows(rows).build()); List shards = getFutureValue(sink.commit()); diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardRecovery.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardRecovery.java index 45da969bfd7f..f7ee66281451 100644 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardRecovery.java +++ b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/TestShardRecovery.java @@ -42,7 +42,7 @@ import static io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_BACKUP_CORRUPTION; import static io.trino.plugin.raptor.legacy.metadata.SchemaDaoUtil.createTablesWithRetry; import static io.trino.plugin.raptor.legacy.metadata.TestDatabaseShardManager.createShardManager; -import static io.trino.plugin.raptor.legacy.storage.OrcStorageManager.xxhash64; +import static io.trino.plugin.raptor.legacy.storage.RaptorStorageManager.xxhash64; import static io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy; import static java.io.File.createTempFile; import static java.lang.String.format; diff --git a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardCompactor.java b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardCompactor.java index 2b505c0cb96c..a98f5f1ac430 100644 --- a/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardCompactor.java +++ b/plugin/trino-raptor-legacy/src/test/java/io/trino/plugin/raptor/legacy/storage/organization/TestShardCompactor.java @@ -21,7 +21,7 @@ import io.trino.orc.OrcReaderOptions; import io.trino.plugin.raptor.legacy.metadata.ColumnInfo; import io.trino.plugin.raptor.legacy.metadata.ShardInfo; -import io.trino.plugin.raptor.legacy.storage.OrcStorageManager; +import io.trino.plugin.raptor.legacy.storage.RaptorStorageManager; import io.trino.plugin.raptor.legacy.storage.StorageManager; import io.trino.plugin.raptor.legacy.storage.StoragePageSink; import io.trino.spi.Page; @@ -54,7 +54,7 @@ import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.airlift.concurrent.MoreFutures.getFutureValue; import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static io.trino.plugin.raptor.legacy.storage.TestOrcStorageManager.createOrcStorageManager; +import static io.trino.plugin.raptor.legacy.storage.TestRaptorStorageManager.createRaptorStorageManager; import static io.trino.spi.connector.SortOrder.ASC_NULLS_FIRST; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.spi.type.DateType.DATE; @@ -80,7 +80,7 @@ public class TestShardCompactor .withStreamBufferSize(DataSize.of(1, MEGABYTE)) .withTinyStripeThreshold(DataSize.of(1, MEGABYTE)); - private OrcStorageManager storageManager; + private RaptorStorageManager storageManager; private ShardCompactor compactor; private File temporary; private Handle dummyHandle; @@ -91,7 +91,7 @@ public void setup() temporary = createTempDir(); IDBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime() + ThreadLocalRandom.current().nextLong()); dummyHandle = dbi.open(); - storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS); + storageManager = createRaptorStorageManager(dbi, temporary, MAX_SHARD_ROWS); compactor = new ShardCompactor(storageManager, READER_OPTIONS, new TypeOperators()); } From b0638b509a0965af74731fc921a37ba849fa0334 Mon Sep 17 00:00:00 2001 From: Martin Traverso Date: Wed, 7 Apr 2021 16:35:12 -0700 Subject: [PATCH 074/146] Mark assertExpression and assertStatement as deprecated There's a much better and more flexible alternative with ParserAssert. --- .../test/java/io/trino/sql/parser/TestSqlParser.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/core/trino-parser/src/test/java/io/trino/sql/parser/TestSqlParser.java b/core/trino-parser/src/test/java/io/trino/sql/parser/TestSqlParser.java index 940760e172ce..b7a7a18d4d23 100644 --- a/core/trino-parser/src/test/java/io/trino/sql/parser/TestSqlParser.java +++ b/core/trino-parser/src/test/java/io/trino/sql/parser/TestSqlParser.java @@ -2904,18 +2904,30 @@ private static QualifiedName makeQualifiedName(String tableName) return QualifiedName.of(parts); } + /** + * @deprecated use {@link ParserAssert#statement(String)} instead + */ + @Deprecated private static void assertStatement(String query, Statement expected) { assertParsed(query, expected, SQL_PARSER.createStatement(query, new ParsingOptions())); assertFormattedSql(SQL_PARSER, expected); } + /** + * @deprecated use {@link ParserAssert#statement(String)} instead + */ + @Deprecated private static void assertInvalidStatement(String statement, String expectedErrorMessageRegex) { assertThatThrownBy(() -> SQL_PARSER.createStatement(statement, new ParsingOptions())) .isInstanceOfSatisfying(ParsingException.class, e -> assertTrue(e.getErrorMessage().matches(expectedErrorMessageRegex))); } + /** + * @deprecated use {@link ParserAssert#expression(String)} instead + */ + @Deprecated private static void assertExpression(String expression, Expression expected) { requireNonNull(expression, "expression is null"); From 76b2d0d1c377212ac2d5dd21a59391f9578d6ef5 Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Wed, 7 Apr 2021 10:48:27 -0700 Subject: [PATCH 075/146] Add multiple authenticator information --- docs/src/main/sphinx/security.rst | 1 + .../sphinx/security/authentication-types.rst | 64 +++++++++++++++++++ docs/src/main/sphinx/security/ldap.rst | 4 +- .../main/sphinx/security/password-file.rst | 11 +++- docs/src/main/sphinx/security/salesforce.rst | 11 +++- 5 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 docs/src/main/sphinx/security/authentication-types.rst diff --git a/docs/src/main/sphinx/security.rst b/docs/src/main/sphinx/security.rst index 2534421746c8..6a22bf813ae3 100644 --- a/docs/src/main/sphinx/security.rst +++ b/docs/src/main/sphinx/security.rst @@ -11,6 +11,7 @@ Cluster access security security/tls security/inspect-pem security/inspect-jks + security/authentication-types security/ldap security/password-file security/salesforce diff --git a/docs/src/main/sphinx/security/authentication-types.rst b/docs/src/main/sphinx/security/authentication-types.rst new file mode 100644 index 000000000000..f548d4d9bb66 --- /dev/null +++ b/docs/src/main/sphinx/security/authentication-types.rst @@ -0,0 +1,64 @@ +==================== +Authentication types +==================== + +Trino supports multiple authentication types to ensure all users of the system +are authenticated. Different authenticators allow user management in one or more +systems. Using :doc:`TLS ` is required for all authentications types. + +You can configure one or more authentication types with the +``http-server.authentication.type`` property. The following authentication types +and authenticators are available: + +* ``PASSSWORD`` for :doc:`password-file`, :doc:`ldap`, and :doc:`salesforce` +* ``OAUTH2`` for :doc:`oauth2` +* ``CERTIFICATE`` for certificate authentication +* ``JWT`` for Java Web Token (JWT) authentication +* ``KERBEROS`` for :doc:`Kerberos authentication ` + +Get started with a basic password authentication configuration backed by a +:doc:`password file `: + +.. code-block:: properties + + http-server.authentication.type=PASSWORD + + +Multiple authentication types +----------------------------- + +You can use multiple authentication types, separated with commas in the +configuration: + +.. code-block:: properties + + http-server.authentication.type=PASSWORD,CERTIFICATE + + +Authentication is performed in order of the entries, and first successful +authentication results in access, using the :doc:`mapped user ` +from that authentication method. + +Multiple password authenticators +-------------------------------- + +You can use multiple password authenticator types by referencing multiple +configuration files: + +.. code-block:: properties + + http-server.authentication.type=PASSWORD + password-authenticator.config-files=etc/ldap1.properties,etc/ldap2.properties,etc/password.properties + +In the preceding example, the configuration files ``ldap1.properties`` and +``ldap1.properties`` are regular :doc:`LDAP authenticator configuration files +`. The ``password.properties`` is a :doc:`password file authenticator +configuration file `. + +Relative paths to the installation directory or absolute paths can be used. + +User authentication credentials are first validated against the LDAP server from +``ldap1``, then the separate server from ``ldap2``, and finally the password +file. First successful authentication results in access, and no further +authenticators are called. + diff --git a/docs/src/main/sphinx/security/ldap.rst b/docs/src/main/sphinx/security/ldap.rst index c1404736018e..c73f96010e50 100644 --- a/docs/src/main/sphinx/security/ldap.rst +++ b/docs/src/main/sphinx/security/ldap.rst @@ -49,8 +49,8 @@ to the coordinator's ``config.properties`` file: ============================================================= ====================================================== Property Description ============================================================= ====================================================== -``http-server.authentication.type`` Enable password authentication for the Trino - coordinator. Must be set to ``PASSWORD``. +``http-server.authentication.type`` Enable the password :doc:`authentication type ` + for the Trino coordinator. Must be set to ``PASSWORD``. ``http-server.https.enabled`` Enables HTTPS access for the Trino coordinator. Should be set to ``true``. Default value is ``false``. diff --git a/docs/src/main/sphinx/security/password-file.rst b/docs/src/main/sphinx/security/password-file.rst index 0f26b3e3e6a7..e5bc7f457e56 100644 --- a/docs/src/main/sphinx/security/password-file.rst +++ b/docs/src/main/sphinx/security/password-file.rst @@ -14,8 +14,15 @@ and clients to use TLS and authenticate with a username and password. Password authenticator configuration ------------------------------------ -Enable password file authentication by creating an -``etc/password-authenticator.properties`` file on the coordinator: +To enable password file authentication, set the :doc:`password authentication +type ` in ``etc/config.properties``: + +.. code-block:: properties + + http-server.authentication.type=PASSWORD + +In addition, create a ``etc/password-authenticator.properties`` file on the +coordinator with the ``file`` authenticator name: .. code-block:: text diff --git a/docs/src/main/sphinx/security/salesforce.rst b/docs/src/main/sphinx/security/salesforce.rst index 14c61421b5e0..5041c33335e6 100644 --- a/docs/src/main/sphinx/security/salesforce.rst +++ b/docs/src/main/sphinx/security/salesforce.rst @@ -19,8 +19,15 @@ basic credentials. This can also be used to secure the :ref:`Web UI ` in ``etc/config.properties``: + +.. code-block:: properties + + http-server.authentication.type=PASSWORD + +In addition, create a ``etc/password-authenticator.properties`` file on the +coordinator with the ``salesforce`` authenticator name: .. code-block:: properties From a30733401472bc226ec10cce450b70789c16ca39 Mon Sep 17 00:00:00 2001 From: Barton Wright Date: Tue, 6 Apr 2021 10:27:02 -0400 Subject: [PATCH 076/146] Add Security overview --- docs/src/main/sphinx/security.rst | 8 ++ docs/src/main/sphinx/security/overview.rst | 149 +++++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 docs/src/main/sphinx/security/overview.rst diff --git a/docs/src/main/sphinx/security.rst b/docs/src/main/sphinx/security.rst index 6a22bf813ae3..09a73e04dea9 100644 --- a/docs/src/main/sphinx/security.rst +++ b/docs/src/main/sphinx/security.rst @@ -2,6 +2,14 @@ Security ******** +Introduction +============ + +.. toctree:: + :maxdepth: 1 + + security/overview + Cluster access security ======================= diff --git a/docs/src/main/sphinx/security/overview.rst b/docs/src/main/sphinx/security/overview.rst new file mode 100644 index 000000000000..a14f55b10440 --- /dev/null +++ b/docs/src/main/sphinx/security/overview.rst @@ -0,0 +1,149 @@ +================= +Security overview +================= + +After the initial :doc:`installation ` of your cluster, security +is the next major concern for successfully operating Trino. This overview +provides an introduction to different aspects of configuring security for your +Trino cluster. + +Aspects of configuring security +------------------------------- + +The default installation of Trino has no security features enabled. Security +can be enabled for different parts of the Trino architecture: + +* :ref:`security-client` +* :ref:`security-inside-cluster` +* :ref:`security-data-sources` + +Suggested configuration workflow +-------------------------------- + +To configure security for a new Trino cluster, follow this best practice +order of steps. Do not skip or combine steps. + +#. **Enable** :doc:`HTTPS/TLS ` + + * Work with your security team. + * Use a :ref:`load balancer or proxy ` to terminate + HTTPS, if possible. + * Use a globally trusted TLS certificate. + +#. **Enable authentication** + + * Start with :doc:`password file authentication ` to get up + and running. + * Then configure your preferred authentication provider, such as :doc:`LDAP + `. + * Avoid the complexity of Kerberos for client authentication, if possible. + +#. **Enable authorization and access control** + + * Start with :doc:`file-based rules `. + * Then configure another access control method as required. + +Configure one step at a time. Always restart the Trino server after each +change, and verify the results before proceeding. + +.. _security-client: + +Securing client access to the cluster +------------------------------------- + +Trino :doc:`clients ` include the Trino :doc:`CLI `, +the :doc:`Web UI `, the :doc:`JDBC driver +`, `Python, Go, or other clients +`_, and any applications using these tools. + +All access to the Trino cluster is managed by the coordinator. Thus, securing +access to the cluster means securing access to the coordinator. + +There are three aspects to consider: + +* :ref:`cl-access-encrypt`: protecting the integrity of client to server + communication in transit. +* :ref:`cl-access-auth`: identifying users and user name management. +* :ref:`cl-access-control`: validating each user's access rights. + +.. _cl-access-encrypt: + +Encryption +^^^^^^^^^^ + +The Trino server uses the standard :doc:`HTTPS protocol and TLS encryption +`, formerly known as SSL. + +.. _cl-access-auth: + +Authentication +^^^^^^^^^^^^^^ + +Trino supports several authentication providers. When setting up a new cluster, +start with simple password file authentication before configuring another +provider. + +* :doc:`Password file authentication ` +* :doc:`LDAP authentication ` +* :doc:`OAuth 2.0 authentication ` +* :doc:`Salesforce authentication ` +* :doc:`Kerberos authentication ` + +.. _user-name-management: + +User name management +"""""""""""""""""""" + +Trino provides ways to map the user and group names from authentication +providers to Trino user names. + +* :doc:`User mapping ` applies to all authentication systems, + and allows for JSON files to specify rules to map complex user names from + other systems (``alice@example.com``) to simple user names (``alice``). +* :doc:`File group provider ` provides a way to assign a set + of user names to a group name to ease access control. + +.. _cl-access-control: + +Authorization and access control +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Trino's :doc:`default method of access control ` +allows all operations for all authenticated users. + +To implement access control, use: + +* :doc:`File-based system access control `, where + you configure JSON files that specify fine-grained user access restrictions at + the catalog, schema, or table level. + +In addition, Trino :doc:`provides an API ` that +allows you to create a custom access control method, or to extend an existing +one. + +.. _security-inside-cluster: + +Securing inside the cluster +--------------------------- + +You can :doc:`secure the internal communication ` +between coordinator and workers inside the clusters. + +Secrets in properties files, such as passwords in catalog files, can be secured +with :doc:`secrets management `. + +.. _security-data-sources: + +Securing cluster access to data sources +--------------------------------------- + +Communication between the Trino cluster and data sources is configured for each +catalog. Each catalog uses a connector, which supports a variety of +security-related configurations. + +More information is available with the documentation for individual +:doc:`connectors `. + +:doc:`Secrets management ` can be used for the catalog properties files +content. + From 9aad2329b011061f7ab6f1100b14cb34f5ad2fe5 Mon Sep 17 00:00:00 2001 From: Martin Traverso Date: Mon, 5 Apr 2021 12:49:53 -0700 Subject: [PATCH 077/146] Add 355 release notes --- docs/src/main/sphinx/release.rst | 1 + docs/src/main/sphinx/release/release-355.md | 52 +++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 docs/src/main/sphinx/release/release-355.md diff --git a/docs/src/main/sphinx/release.rst b/docs/src/main/sphinx/release.rst index 44ed32e96bce..31374ad99e6a 100644 --- a/docs/src/main/sphinx/release.rst +++ b/docs/src/main/sphinx/release.rst @@ -10,6 +10,7 @@ Release notes .. toctree:: :maxdepth: 1 + release/release-355 release/release-354 release/release-353 release/release-352 diff --git a/docs/src/main/sphinx/release/release-355.md b/docs/src/main/sphinx/release/release-355.md new file mode 100644 index 000000000000..26d3f939ba5d --- /dev/null +++ b/docs/src/main/sphinx/release/release-355.md @@ -0,0 +1,52 @@ +# Release 355 (8 Apr 2021) + +## General + +* Report tables that are directly referenced by a query in `QueryCompletedEvent`. ({issue}`7330`) +* Report columns that are the target of `INSERT` or `UPDATE` queries in `QueryCompletedEvent`. This includes + information about which input columns they are derived from. ({issue}`7425`, {issue}`7465`) +* Rename `optimizer.plan-with-table-node-partitioning` config property to `optimizer.use-table-scan-node-partitioning`. ({issue}`7257`) +* Improve query parallelism when table bucket count is small compared to number of nodes. + This optimization is now triggered automatically when the ratio between table buckets and + possible table scan tasks exceeds or is equal to `optimizer.table-scan-node-partitioning-min-bucket-to-task-ratio`. ({issue}`7257`) +* Include information about {doc}`/admin/spill` in {doc}`/sql/explain-analyze`. ({issue}`7427`) +* Disallow inserting data into tables that have row filters. ({issue}`7346`) +* Improve performance of queries that can benefit from both {doc}`/optimizer/cost-based-optimizations` and join pushdown + by giving precedence to cost-based optimizations. ({issue}`7331`) +* Fix inconsistent behavior for {func}`to_unixtime` with values of type `timestamp(p)`. ({issue}`7450`) +* Change return type of {func}`from_unixtime` and {func}`from_unixtime_nanos` to `timestamp(p) with time zone`. ({issue}`7460`) + +## Security + +* Add support for configuring multiple password authentication plugins. ({issue}`7151`) + +## JDBC driver + +* Add `assumeLiteralNamesInMetadataCallsForNonConformingClients` parameter for use as a workaround when + applications do not properly escape schema or table names in calls to `DatabaseMetaData` methods. ({issue}`7438`) + +## ClickHouse connector + +* Support creating tables with MergeTree storage engine. ({issue}`7135`) + +## Hive connector + +* Support Hive views containing `LATERAL VIEW json_tuple(...) AS ...` syntax. ({issue}`7242`) +* Fix incorrect results when reading from a Hive view that uses array subscript operators. ({issue}`7271`) +* Fix incorrect results when querying the `$file_modified_time` hidden column. ({issue}`7511`) + +## Phoenix connector + +* Improve performance when fetching table metadata during query analysis. ({issue}`6975`) +* Improve performance of queries with `ORDER BY ... LIMIT` clause when the computation + can be pushed down to the underlying database. ({issue}`7490`) + +## SQL Server connector + +* Improve performance when fetching table metadata during query analysis. ({issue}`6975`) + +## SPI + +* Engine now uses `ConnectorMaterializedViewDefinition#storageTable` + to determine materialized view storage table. ({issue}`7319`) + From 3e15a252dcd6dfac5352e2a21dc0f3e2305008e1 Mon Sep 17 00:00:00 2001 From: Martin Traverso Date: Thu, 8 Apr 2021 16:16:05 -0700 Subject: [PATCH 078/146] [maven-release-plugin] prepare release 355 --- client/trino-cli/pom.xml | 2 +- client/trino-client/pom.xml | 2 +- client/trino-jdbc/pom.xml | 2 +- core/trino-main/pom.xml | 2 +- core/trino-parser/pom.xml | 2 +- core/trino-server-main/pom.xml | 2 +- core/trino-server-rpm/pom.xml | 2 +- core/trino-server/pom.xml | 2 +- core/trino-spi/pom.xml | 2 +- docs/pom.xml | 2 +- lib/trino-array/pom.xml | 2 +- lib/trino-geospatial-toolkit/pom.xml | 2 +- lib/trino-matching/pom.xml | 2 +- lib/trino-memory-context/pom.xml | 2 +- lib/trino-orc/pom.xml | 2 +- lib/trino-parquet/pom.xml | 2 +- lib/trino-plugin-toolkit/pom.xml | 2 +- lib/trino-rcfile/pom.xml | 2 +- lib/trino-record-decoder/pom.xml | 2 +- plugin/trino-accumulo-iterators/pom.xml | 2 +- plugin/trino-accumulo/pom.xml | 2 +- plugin/trino-atop/pom.xml | 2 +- plugin/trino-base-jdbc/pom.xml | 2 +- plugin/trino-bigquery/pom.xml | 2 +- plugin/trino-blackhole/pom.xml | 2 +- plugin/trino-cassandra/pom.xml | 2 +- plugin/trino-clickhouse/pom.xml | 2 +- plugin/trino-druid/pom.xml | 2 +- plugin/trino-elasticsearch/pom.xml | 2 +- plugin/trino-example-http/pom.xml | 2 +- plugin/trino-geospatial/pom.xml | 2 +- plugin/trino-google-sheets/pom.xml | 2 +- plugin/trino-hive-hadoop2/pom.xml | 2 +- plugin/trino-hive/pom.xml | 2 +- plugin/trino-iceberg/pom.xml | 2 +- plugin/trino-jmx/pom.xml | 2 +- plugin/trino-kafka/pom.xml | 2 +- plugin/trino-kinesis/pom.xml | 2 +- plugin/trino-kudu/pom.xml | 2 +- plugin/trino-local-file/pom.xml | 2 +- plugin/trino-memory/pom.xml | 2 +- plugin/trino-memsql/pom.xml | 2 +- plugin/trino-ml/pom.xml | 2 +- plugin/trino-mongodb/pom.xml | 2 +- plugin/trino-mysql/pom.xml | 2 +- plugin/trino-oracle/pom.xml | 2 +- plugin/trino-password-authenticators/pom.xml | 2 +- plugin/trino-phoenix/pom.xml | 2 +- plugin/trino-phoenix5/pom.xml | 2 +- plugin/trino-pinot/pom.xml | 2 +- plugin/trino-postgresql/pom.xml | 2 +- plugin/trino-prometheus/pom.xml | 2 +- plugin/trino-raptor-legacy/pom.xml | 2 +- plugin/trino-redis/pom.xml | 2 +- plugin/trino-redshift/pom.xml | 2 +- plugin/trino-resource-group-managers/pom.xml | 2 +- plugin/trino-session-property-managers/pom.xml | 2 +- plugin/trino-sqlserver/pom.xml | 2 +- plugin/trino-teradata-functions/pom.xml | 2 +- plugin/trino-thrift-api/pom.xml | 2 +- plugin/trino-thrift-testing-server/pom.xml | 2 +- plugin/trino-thrift/pom.xml | 2 +- plugin/trino-tpcds/pom.xml | 2 +- plugin/trino-tpch/pom.xml | 2 +- pom.xml | 4 ++-- service/trino-proxy/pom.xml | 2 +- service/trino-verifier/pom.xml | 2 +- testing/trino-benchmark-driver/pom.xml | 2 +- testing/trino-benchmark/pom.xml | 2 +- testing/trino-benchto-benchmarks/pom.xml | 2 +- testing/trino-product-tests-launcher/pom.xml | 2 +- testing/trino-product-tests/pom.xml | 2 +- testing/trino-server-dev/pom.xml | 2 +- testing/trino-test-jdbc-compatibility-old-driver/pom.xml | 4 ++-- testing/trino-test-jdbc-compatibility-old-server/pom.xml | 2 +- testing/trino-testing-kafka/pom.xml | 2 +- testing/trino-testing/pom.xml | 2 +- testing/trino-testng-services/pom.xml | 2 +- testing/trino-tests/pom.xml | 2 +- 79 files changed, 81 insertions(+), 81 deletions(-) diff --git a/client/trino-cli/pom.xml b/client/trino-cli/pom.xml index 85089123041b..519b6780e6bf 100644 --- a/client/trino-cli/pom.xml +++ b/client/trino-cli/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/client/trino-client/pom.xml b/client/trino-client/pom.xml index 8c4e4de10e52..aa7772fdf168 100644 --- a/client/trino-client/pom.xml +++ b/client/trino-client/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/client/trino-jdbc/pom.xml b/client/trino-jdbc/pom.xml index 74692b96692c..226f4b47dd9e 100644 --- a/client/trino-jdbc/pom.xml +++ b/client/trino-jdbc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/core/trino-main/pom.xml b/core/trino-main/pom.xml index 83baff416725..df0cfad6e9c5 100644 --- a/core/trino-main/pom.xml +++ b/core/trino-main/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/core/trino-parser/pom.xml b/core/trino-parser/pom.xml index fade86cc20d1..c4e774642dfc 100644 --- a/core/trino-parser/pom.xml +++ b/core/trino-parser/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/core/trino-server-main/pom.xml b/core/trino-server-main/pom.xml index 9b99e27f0a78..ea7502739dca 100644 --- a/core/trino-server-main/pom.xml +++ b/core/trino-server-main/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/core/trino-server-rpm/pom.xml b/core/trino-server-rpm/pom.xml index a223cea45f53..3c10246e7ca2 100644 --- a/core/trino-server-rpm/pom.xml +++ b/core/trino-server-rpm/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/core/trino-server/pom.xml b/core/trino-server/pom.xml index 9dc18f4cd3dd..68c9bea1473b 100644 --- a/core/trino-server/pom.xml +++ b/core/trino-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/core/trino-spi/pom.xml b/core/trino-spi/pom.xml index bd18c55c1550..e5c99a810744 100644 --- a/core/trino-spi/pom.xml +++ b/core/trino-spi/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/docs/pom.xml b/docs/pom.xml index 3afedf00df90..88abec2aca56 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 trino-docs diff --git a/lib/trino-array/pom.xml b/lib/trino-array/pom.xml index 21d722280b35..cd6e88705009 100644 --- a/lib/trino-array/pom.xml +++ b/lib/trino-array/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/lib/trino-geospatial-toolkit/pom.xml b/lib/trino-geospatial-toolkit/pom.xml index 8f65e96c23d6..9b3c44d5e09b 100644 --- a/lib/trino-geospatial-toolkit/pom.xml +++ b/lib/trino-geospatial-toolkit/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/lib/trino-matching/pom.xml b/lib/trino-matching/pom.xml index d5c2b8c8752d..e6af83f44d1d 100644 --- a/lib/trino-matching/pom.xml +++ b/lib/trino-matching/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/lib/trino-memory-context/pom.xml b/lib/trino-memory-context/pom.xml index 0dbd53e1d92f..4e0092fedcd9 100644 --- a/lib/trino-memory-context/pom.xml +++ b/lib/trino-memory-context/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/lib/trino-orc/pom.xml b/lib/trino-orc/pom.xml index 66509d0953b2..8d82285af6fb 100644 --- a/lib/trino-orc/pom.xml +++ b/lib/trino-orc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/lib/trino-parquet/pom.xml b/lib/trino-parquet/pom.xml index 28098fd4b104..4cadea92cbed 100644 --- a/lib/trino-parquet/pom.xml +++ b/lib/trino-parquet/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/lib/trino-plugin-toolkit/pom.xml b/lib/trino-plugin-toolkit/pom.xml index 63cacfaccc2c..f980f581ea75 100644 --- a/lib/trino-plugin-toolkit/pom.xml +++ b/lib/trino-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/lib/trino-rcfile/pom.xml b/lib/trino-rcfile/pom.xml index e974ef200c1c..fe852bde8bc4 100644 --- a/lib/trino-rcfile/pom.xml +++ b/lib/trino-rcfile/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/lib/trino-record-decoder/pom.xml b/lib/trino-record-decoder/pom.xml index 522ff3b3bf39..38c239e757b8 100644 --- a/lib/trino-record-decoder/pom.xml +++ b/lib/trino-record-decoder/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-accumulo-iterators/pom.xml b/plugin/trino-accumulo-iterators/pom.xml index 9707a3d621e6..3b7d09d7f00a 100644 --- a/plugin/trino-accumulo-iterators/pom.xml +++ b/plugin/trino-accumulo-iterators/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-accumulo/pom.xml b/plugin/trino-accumulo/pom.xml index 4c943110c70b..4c091b85fae0 100644 --- a/plugin/trino-accumulo/pom.xml +++ b/plugin/trino-accumulo/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-atop/pom.xml b/plugin/trino-atop/pom.xml index c326df81e0cf..0433fc033bde 100644 --- a/plugin/trino-atop/pom.xml +++ b/plugin/trino-atop/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-base-jdbc/pom.xml b/plugin/trino-base-jdbc/pom.xml index 2ce04f21a32b..cca3db3b2523 100644 --- a/plugin/trino-base-jdbc/pom.xml +++ b/plugin/trino-base-jdbc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-bigquery/pom.xml b/plugin/trino-bigquery/pom.xml index 3abf2cfcc99a..f415f9d2d20d 100644 --- a/plugin/trino-bigquery/pom.xml +++ b/plugin/trino-bigquery/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-blackhole/pom.xml b/plugin/trino-blackhole/pom.xml index 6d304095166b..fae7b4a952fc 100644 --- a/plugin/trino-blackhole/pom.xml +++ b/plugin/trino-blackhole/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-cassandra/pom.xml b/plugin/trino-cassandra/pom.xml index d59a28511493..124ebf1f4920 100644 --- a/plugin/trino-cassandra/pom.xml +++ b/plugin/trino-cassandra/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-clickhouse/pom.xml b/plugin/trino-clickhouse/pom.xml index c913ffa79fe4..7fa425198900 100644 --- a/plugin/trino-clickhouse/pom.xml +++ b/plugin/trino-clickhouse/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-druid/pom.xml b/plugin/trino-druid/pom.xml index f5c2133dbe29..583f07394a24 100644 --- a/plugin/trino-druid/pom.xml +++ b/plugin/trino-druid/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-elasticsearch/pom.xml b/plugin/trino-elasticsearch/pom.xml index 076d5c86bd5e..9a585e461ed6 100644 --- a/plugin/trino-elasticsearch/pom.xml +++ b/plugin/trino-elasticsearch/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-example-http/pom.xml b/plugin/trino-example-http/pom.xml index 397ef59bf1e7..eace961271ff 100644 --- a/plugin/trino-example-http/pom.xml +++ b/plugin/trino-example-http/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-geospatial/pom.xml b/plugin/trino-geospatial/pom.xml index 136a1f22882a..ae26068d9e22 100644 --- a/plugin/trino-geospatial/pom.xml +++ b/plugin/trino-geospatial/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-google-sheets/pom.xml b/plugin/trino-google-sheets/pom.xml index ea487e088f86..0d101bddfd43 100644 --- a/plugin/trino-google-sheets/pom.xml +++ b/plugin/trino-google-sheets/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-hive-hadoop2/pom.xml b/plugin/trino-hive-hadoop2/pom.xml index bfcd308f7302..f26b23a6c61e 100644 --- a/plugin/trino-hive-hadoop2/pom.xml +++ b/plugin/trino-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-hive/pom.xml b/plugin/trino-hive/pom.xml index 1073599e470d..58d60df42c93 100644 --- a/plugin/trino-hive/pom.xml +++ b/plugin/trino-hive/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-iceberg/pom.xml b/plugin/trino-iceberg/pom.xml index 5587307e6a2e..942aacc6533f 100644 --- a/plugin/trino-iceberg/pom.xml +++ b/plugin/trino-iceberg/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-jmx/pom.xml b/plugin/trino-jmx/pom.xml index b8728b143448..7e0db505e63f 100644 --- a/plugin/trino-jmx/pom.xml +++ b/plugin/trino-jmx/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-kafka/pom.xml b/plugin/trino-kafka/pom.xml index 61774692543e..2a26c705f5b8 100644 --- a/plugin/trino-kafka/pom.xml +++ b/plugin/trino-kafka/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-kinesis/pom.xml b/plugin/trino-kinesis/pom.xml index 6a64cd7edf18..e9991073d3ca 100644 --- a/plugin/trino-kinesis/pom.xml +++ b/plugin/trino-kinesis/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-kudu/pom.xml b/plugin/trino-kudu/pom.xml index 25f8d217e85a..0760482b2623 100644 --- a/plugin/trino-kudu/pom.xml +++ b/plugin/trino-kudu/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-local-file/pom.xml b/plugin/trino-local-file/pom.xml index ca1c1a7eb5b4..7b35bf332bb4 100644 --- a/plugin/trino-local-file/pom.xml +++ b/plugin/trino-local-file/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-memory/pom.xml b/plugin/trino-memory/pom.xml index b7b04c731f64..51c33adb41e4 100644 --- a/plugin/trino-memory/pom.xml +++ b/plugin/trino-memory/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-memsql/pom.xml b/plugin/trino-memsql/pom.xml index 7643c4ecc227..d11fabdff691 100644 --- a/plugin/trino-memsql/pom.xml +++ b/plugin/trino-memsql/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-ml/pom.xml b/plugin/trino-ml/pom.xml index ed8527d23142..1bd3bf95891a 100644 --- a/plugin/trino-ml/pom.xml +++ b/plugin/trino-ml/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-mongodb/pom.xml b/plugin/trino-mongodb/pom.xml index daac59775d6c..7eb0ada05fe5 100644 --- a/plugin/trino-mongodb/pom.xml +++ b/plugin/trino-mongodb/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-mysql/pom.xml b/plugin/trino-mysql/pom.xml index 463e7ef71508..21c4175983f3 100644 --- a/plugin/trino-mysql/pom.xml +++ b/plugin/trino-mysql/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-oracle/pom.xml b/plugin/trino-oracle/pom.xml index 8cb0718b7287..b930b5601ed6 100644 --- a/plugin/trino-oracle/pom.xml +++ b/plugin/trino-oracle/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-password-authenticators/pom.xml b/plugin/trino-password-authenticators/pom.xml index 152c5cd45152..0cdf9ca8ef34 100644 --- a/plugin/trino-password-authenticators/pom.xml +++ b/plugin/trino-password-authenticators/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-phoenix/pom.xml b/plugin/trino-phoenix/pom.xml index 62a8ce620ad7..1f3658c20e60 100644 --- a/plugin/trino-phoenix/pom.xml +++ b/plugin/trino-phoenix/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-phoenix5/pom.xml b/plugin/trino-phoenix5/pom.xml index b0473b9625f0..30d155769ad4 100644 --- a/plugin/trino-phoenix5/pom.xml +++ b/plugin/trino-phoenix5/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-pinot/pom.xml b/plugin/trino-pinot/pom.xml index c5d55810367e..5793cb5b6028 100755 --- a/plugin/trino-pinot/pom.xml +++ b/plugin/trino-pinot/pom.xml @@ -4,7 +4,7 @@ trino-root io.trino - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-postgresql/pom.xml b/plugin/trino-postgresql/pom.xml index 69ab20a9c10f..428e4bd97a78 100644 --- a/plugin/trino-postgresql/pom.xml +++ b/plugin/trino-postgresql/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-prometheus/pom.xml b/plugin/trino-prometheus/pom.xml index a5d58529a11c..b11d6f9586ce 100644 --- a/plugin/trino-prometheus/pom.xml +++ b/plugin/trino-prometheus/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-raptor-legacy/pom.xml b/plugin/trino-raptor-legacy/pom.xml index e5aadfecaf08..fe39fffe09cb 100644 --- a/plugin/trino-raptor-legacy/pom.xml +++ b/plugin/trino-raptor-legacy/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-redis/pom.xml b/plugin/trino-redis/pom.xml index e673f54580ef..404cf731c75e 100644 --- a/plugin/trino-redis/pom.xml +++ b/plugin/trino-redis/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-redshift/pom.xml b/plugin/trino-redshift/pom.xml index fbaa1b1bfa9e..fbe84ad978c1 100644 --- a/plugin/trino-redshift/pom.xml +++ b/plugin/trino-redshift/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-resource-group-managers/pom.xml b/plugin/trino-resource-group-managers/pom.xml index 58cf5b665149..11f7802d3c3b 100644 --- a/plugin/trino-resource-group-managers/pom.xml +++ b/plugin/trino-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-session-property-managers/pom.xml b/plugin/trino-session-property-managers/pom.xml index 01672ac36e2a..6d791f3c8d91 100644 --- a/plugin/trino-session-property-managers/pom.xml +++ b/plugin/trino-session-property-managers/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-sqlserver/pom.xml b/plugin/trino-sqlserver/pom.xml index 71d3248068be..69c7ab62f470 100644 --- a/plugin/trino-sqlserver/pom.xml +++ b/plugin/trino-sqlserver/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-teradata-functions/pom.xml b/plugin/trino-teradata-functions/pom.xml index 5b5bab774922..59dbd4a76264 100644 --- a/plugin/trino-teradata-functions/pom.xml +++ b/plugin/trino-teradata-functions/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-thrift-api/pom.xml b/plugin/trino-thrift-api/pom.xml index 1484424908a7..1f4122d6df9b 100644 --- a/plugin/trino-thrift-api/pom.xml +++ b/plugin/trino-thrift-api/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-thrift-testing-server/pom.xml b/plugin/trino-thrift-testing-server/pom.xml index 5b4fb5225d4f..c4bc9f307a8c 100644 --- a/plugin/trino-thrift-testing-server/pom.xml +++ b/plugin/trino-thrift-testing-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-thrift/pom.xml b/plugin/trino-thrift/pom.xml index f91420d2218b..2ab17db6f244 100644 --- a/plugin/trino-thrift/pom.xml +++ b/plugin/trino-thrift/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-tpcds/pom.xml b/plugin/trino-tpcds/pom.xml index 6a03b9e41e49..4567f2606469 100644 --- a/plugin/trino-tpcds/pom.xml +++ b/plugin/trino-tpcds/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/plugin/trino-tpch/pom.xml b/plugin/trino-tpch/pom.xml index 855965b1c1ac..ca20c4e472a2 100644 --- a/plugin/trino-tpch/pom.xml +++ b/plugin/trino-tpch/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/pom.xml b/pom.xml index f1594a2dfd91..ddde5d0029a9 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 trino-root Trino @@ -30,7 +30,7 @@ scm:git:git://github.com/trinodb/trino.git https://github.com/trinodb/trino - HEAD + 355 diff --git a/service/trino-proxy/pom.xml b/service/trino-proxy/pom.xml index 7f5ae27cc5af..19f66283eae6 100644 --- a/service/trino-proxy/pom.xml +++ b/service/trino-proxy/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/service/trino-verifier/pom.xml b/service/trino-verifier/pom.xml index 9bc4ad3e8e50..432130440476 100644 --- a/service/trino-verifier/pom.xml +++ b/service/trino-verifier/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/testing/trino-benchmark-driver/pom.xml b/testing/trino-benchmark-driver/pom.xml index f77a050f3934..68fb16d3038b 100644 --- a/testing/trino-benchmark-driver/pom.xml +++ b/testing/trino-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/testing/trino-benchmark/pom.xml b/testing/trino-benchmark/pom.xml index 468505af73cc..bcb3963a567c 100644 --- a/testing/trino-benchmark/pom.xml +++ b/testing/trino-benchmark/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/testing/trino-benchto-benchmarks/pom.xml b/testing/trino-benchto-benchmarks/pom.xml index 78468bedc363..fbf182f2258f 100644 --- a/testing/trino-benchto-benchmarks/pom.xml +++ b/testing/trino-benchto-benchmarks/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/testing/trino-product-tests-launcher/pom.xml b/testing/trino-product-tests-launcher/pom.xml index 1045d2d49c60..81844f3e362c 100644 --- a/testing/trino-product-tests-launcher/pom.xml +++ b/testing/trino-product-tests-launcher/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/testing/trino-product-tests/pom.xml b/testing/trino-product-tests/pom.xml index 5968f97e8a0f..5e5e3e2dba18 100644 --- a/testing/trino-product-tests/pom.xml +++ b/testing/trino-product-tests/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/testing/trino-server-dev/pom.xml b/testing/trino-server-dev/pom.xml index 418cb9ada601..05e5991f7b97 100644 --- a/testing/trino-server-dev/pom.xml +++ b/testing/trino-server-dev/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/testing/trino-test-jdbc-compatibility-old-driver/pom.xml b/testing/trino-test-jdbc-compatibility-old-driver/pom.xml index a924a1d450c6..75e4e95df4da 100644 --- a/testing/trino-test-jdbc-compatibility-old-driver/pom.xml +++ b/testing/trino-test-jdbc-compatibility-old-driver/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml @@ -14,7 +14,7 @@ ${project.parent.basedir} - 355-SNAPSHOT + 355 diff --git a/testing/trino-test-jdbc-compatibility-old-server/pom.xml b/testing/trino-test-jdbc-compatibility-old-server/pom.xml index 84068898577e..e0d5496c0aa5 100644 --- a/testing/trino-test-jdbc-compatibility-old-server/pom.xml +++ b/testing/trino-test-jdbc-compatibility-old-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/testing/trino-testing-kafka/pom.xml b/testing/trino-testing-kafka/pom.xml index 660d4a7b5707..c7bb84b6b755 100644 --- a/testing/trino-testing-kafka/pom.xml +++ b/testing/trino-testing-kafka/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/testing/trino-testing/pom.xml b/testing/trino-testing/pom.xml index 18ebb0107c14..9a772b52c7a3 100644 --- a/testing/trino-testing/pom.xml +++ b/testing/trino-testing/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/testing/trino-testng-services/pom.xml b/testing/trino-testng-services/pom.xml index f0d14acc9609..76e9d9f1c3a0 100644 --- a/testing/trino-testng-services/pom.xml +++ b/testing/trino-testng-services/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml diff --git a/testing/trino-tests/pom.xml b/testing/trino-tests/pom.xml index ee0668d6568b..4892ad4d8697 100644 --- a/testing/trino-tests/pom.xml +++ b/testing/trino-tests/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355-SNAPSHOT + 355 ../../pom.xml From 42e026154d04391f31cfd1bb10cf1cc7f09e0cbc Mon Sep 17 00:00:00 2001 From: Martin Traverso Date: Thu, 8 Apr 2021 16:16:06 -0700 Subject: [PATCH 079/146] [maven-release-plugin] prepare for next development iteration --- client/trino-cli/pom.xml | 2 +- client/trino-client/pom.xml | 2 +- client/trino-jdbc/pom.xml | 2 +- core/trino-main/pom.xml | 2 +- core/trino-parser/pom.xml | 2 +- core/trino-server-main/pom.xml | 2 +- core/trino-server-rpm/pom.xml | 2 +- core/trino-server/pom.xml | 2 +- core/trino-spi/pom.xml | 2 +- docs/pom.xml | 2 +- lib/trino-array/pom.xml | 2 +- lib/trino-geospatial-toolkit/pom.xml | 2 +- lib/trino-matching/pom.xml | 2 +- lib/trino-memory-context/pom.xml | 2 +- lib/trino-orc/pom.xml | 2 +- lib/trino-parquet/pom.xml | 2 +- lib/trino-plugin-toolkit/pom.xml | 2 +- lib/trino-rcfile/pom.xml | 2 +- lib/trino-record-decoder/pom.xml | 2 +- plugin/trino-accumulo-iterators/pom.xml | 2 +- plugin/trino-accumulo/pom.xml | 2 +- plugin/trino-atop/pom.xml | 2 +- plugin/trino-base-jdbc/pom.xml | 2 +- plugin/trino-bigquery/pom.xml | 2 +- plugin/trino-blackhole/pom.xml | 2 +- plugin/trino-cassandra/pom.xml | 2 +- plugin/trino-clickhouse/pom.xml | 2 +- plugin/trino-druid/pom.xml | 2 +- plugin/trino-elasticsearch/pom.xml | 2 +- plugin/trino-example-http/pom.xml | 2 +- plugin/trino-geospatial/pom.xml | 2 +- plugin/trino-google-sheets/pom.xml | 2 +- plugin/trino-hive-hadoop2/pom.xml | 2 +- plugin/trino-hive/pom.xml | 2 +- plugin/trino-iceberg/pom.xml | 2 +- plugin/trino-jmx/pom.xml | 2 +- plugin/trino-kafka/pom.xml | 2 +- plugin/trino-kinesis/pom.xml | 2 +- plugin/trino-kudu/pom.xml | 2 +- plugin/trino-local-file/pom.xml | 2 +- plugin/trino-memory/pom.xml | 2 +- plugin/trino-memsql/pom.xml | 2 +- plugin/trino-ml/pom.xml | 2 +- plugin/trino-mongodb/pom.xml | 2 +- plugin/trino-mysql/pom.xml | 2 +- plugin/trino-oracle/pom.xml | 2 +- plugin/trino-password-authenticators/pom.xml | 2 +- plugin/trino-phoenix/pom.xml | 2 +- plugin/trino-phoenix5/pom.xml | 2 +- plugin/trino-pinot/pom.xml | 2 +- plugin/trino-postgresql/pom.xml | 2 +- plugin/trino-prometheus/pom.xml | 2 +- plugin/trino-raptor-legacy/pom.xml | 2 +- plugin/trino-redis/pom.xml | 2 +- plugin/trino-redshift/pom.xml | 2 +- plugin/trino-resource-group-managers/pom.xml | 2 +- plugin/trino-session-property-managers/pom.xml | 2 +- plugin/trino-sqlserver/pom.xml | 2 +- plugin/trino-teradata-functions/pom.xml | 2 +- plugin/trino-thrift-api/pom.xml | 2 +- plugin/trino-thrift-testing-server/pom.xml | 2 +- plugin/trino-thrift/pom.xml | 2 +- plugin/trino-tpcds/pom.xml | 2 +- plugin/trino-tpch/pom.xml | 2 +- pom.xml | 4 ++-- service/trino-proxy/pom.xml | 2 +- service/trino-verifier/pom.xml | 2 +- testing/trino-benchmark-driver/pom.xml | 2 +- testing/trino-benchmark/pom.xml | 2 +- testing/trino-benchto-benchmarks/pom.xml | 2 +- testing/trino-product-tests-launcher/pom.xml | 2 +- testing/trino-product-tests/pom.xml | 2 +- testing/trino-server-dev/pom.xml | 2 +- testing/trino-test-jdbc-compatibility-old-driver/pom.xml | 4 ++-- testing/trino-test-jdbc-compatibility-old-server/pom.xml | 2 +- testing/trino-testing-kafka/pom.xml | 2 +- testing/trino-testing/pom.xml | 2 +- testing/trino-testng-services/pom.xml | 2 +- testing/trino-tests/pom.xml | 2 +- 79 files changed, 81 insertions(+), 81 deletions(-) diff --git a/client/trino-cli/pom.xml b/client/trino-cli/pom.xml index 519b6780e6bf..a9937a4548f1 100644 --- a/client/trino-cli/pom.xml +++ b/client/trino-cli/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/client/trino-client/pom.xml b/client/trino-client/pom.xml index aa7772fdf168..83cef7f84ab9 100644 --- a/client/trino-client/pom.xml +++ b/client/trino-client/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/client/trino-jdbc/pom.xml b/client/trino-jdbc/pom.xml index 226f4b47dd9e..2fd3bbb69af6 100644 --- a/client/trino-jdbc/pom.xml +++ b/client/trino-jdbc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/core/trino-main/pom.xml b/core/trino-main/pom.xml index df0cfad6e9c5..521dee25ed1b 100644 --- a/core/trino-main/pom.xml +++ b/core/trino-main/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/core/trino-parser/pom.xml b/core/trino-parser/pom.xml index c4e774642dfc..14b61b3b61a3 100644 --- a/core/trino-parser/pom.xml +++ b/core/trino-parser/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/core/trino-server-main/pom.xml b/core/trino-server-main/pom.xml index ea7502739dca..2e0f58ec6f4d 100644 --- a/core/trino-server-main/pom.xml +++ b/core/trino-server-main/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/core/trino-server-rpm/pom.xml b/core/trino-server-rpm/pom.xml index 3c10246e7ca2..ac39c734f37d 100644 --- a/core/trino-server-rpm/pom.xml +++ b/core/trino-server-rpm/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/core/trino-server/pom.xml b/core/trino-server/pom.xml index 68c9bea1473b..c083fb9e1635 100644 --- a/core/trino-server/pom.xml +++ b/core/trino-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/core/trino-spi/pom.xml b/core/trino-spi/pom.xml index e5c99a810744..5e9e540d09e2 100644 --- a/core/trino-spi/pom.xml +++ b/core/trino-spi/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/docs/pom.xml b/docs/pom.xml index 88abec2aca56..ac9902938b32 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT trino-docs diff --git a/lib/trino-array/pom.xml b/lib/trino-array/pom.xml index cd6e88705009..3bf6c9b2a67a 100644 --- a/lib/trino-array/pom.xml +++ b/lib/trino-array/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/lib/trino-geospatial-toolkit/pom.xml b/lib/trino-geospatial-toolkit/pom.xml index 9b3c44d5e09b..c002e8349fa8 100644 --- a/lib/trino-geospatial-toolkit/pom.xml +++ b/lib/trino-geospatial-toolkit/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/lib/trino-matching/pom.xml b/lib/trino-matching/pom.xml index e6af83f44d1d..97d63f9ee261 100644 --- a/lib/trino-matching/pom.xml +++ b/lib/trino-matching/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/lib/trino-memory-context/pom.xml b/lib/trino-memory-context/pom.xml index 4e0092fedcd9..119c7c51724a 100644 --- a/lib/trino-memory-context/pom.xml +++ b/lib/trino-memory-context/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/lib/trino-orc/pom.xml b/lib/trino-orc/pom.xml index 8d82285af6fb..1467a966633e 100644 --- a/lib/trino-orc/pom.xml +++ b/lib/trino-orc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/lib/trino-parquet/pom.xml b/lib/trino-parquet/pom.xml index 4cadea92cbed..e3fe9f3fb49d 100644 --- a/lib/trino-parquet/pom.xml +++ b/lib/trino-parquet/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/lib/trino-plugin-toolkit/pom.xml b/lib/trino-plugin-toolkit/pom.xml index f980f581ea75..b6bf9533aa2f 100644 --- a/lib/trino-plugin-toolkit/pom.xml +++ b/lib/trino-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/lib/trino-rcfile/pom.xml b/lib/trino-rcfile/pom.xml index fe852bde8bc4..5a5331ac4121 100644 --- a/lib/trino-rcfile/pom.xml +++ b/lib/trino-rcfile/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/lib/trino-record-decoder/pom.xml b/lib/trino-record-decoder/pom.xml index 38c239e757b8..8d0926b0d4fa 100644 --- a/lib/trino-record-decoder/pom.xml +++ b/lib/trino-record-decoder/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-accumulo-iterators/pom.xml b/plugin/trino-accumulo-iterators/pom.xml index 3b7d09d7f00a..56f725e86739 100644 --- a/plugin/trino-accumulo-iterators/pom.xml +++ b/plugin/trino-accumulo-iterators/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-accumulo/pom.xml b/plugin/trino-accumulo/pom.xml index 4c091b85fae0..a7261dc6ce44 100644 --- a/plugin/trino-accumulo/pom.xml +++ b/plugin/trino-accumulo/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-atop/pom.xml b/plugin/trino-atop/pom.xml index 0433fc033bde..ab85e6475d99 100644 --- a/plugin/trino-atop/pom.xml +++ b/plugin/trino-atop/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-base-jdbc/pom.xml b/plugin/trino-base-jdbc/pom.xml index cca3db3b2523..7a3d593613c6 100644 --- a/plugin/trino-base-jdbc/pom.xml +++ b/plugin/trino-base-jdbc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-bigquery/pom.xml b/plugin/trino-bigquery/pom.xml index f415f9d2d20d..2bf44208b3be 100644 --- a/plugin/trino-bigquery/pom.xml +++ b/plugin/trino-bigquery/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-blackhole/pom.xml b/plugin/trino-blackhole/pom.xml index fae7b4a952fc..5fad955e198d 100644 --- a/plugin/trino-blackhole/pom.xml +++ b/plugin/trino-blackhole/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-cassandra/pom.xml b/plugin/trino-cassandra/pom.xml index 124ebf1f4920..63a6e51628a1 100644 --- a/plugin/trino-cassandra/pom.xml +++ b/plugin/trino-cassandra/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-clickhouse/pom.xml b/plugin/trino-clickhouse/pom.xml index 7fa425198900..1b498e0d41d9 100644 --- a/plugin/trino-clickhouse/pom.xml +++ b/plugin/trino-clickhouse/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-druid/pom.xml b/plugin/trino-druid/pom.xml index 583f07394a24..e38bda244f4b 100644 --- a/plugin/trino-druid/pom.xml +++ b/plugin/trino-druid/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-elasticsearch/pom.xml b/plugin/trino-elasticsearch/pom.xml index 9a585e461ed6..6bdfae5fa6e1 100644 --- a/plugin/trino-elasticsearch/pom.xml +++ b/plugin/trino-elasticsearch/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-example-http/pom.xml b/plugin/trino-example-http/pom.xml index eace961271ff..ae20d60f2309 100644 --- a/plugin/trino-example-http/pom.xml +++ b/plugin/trino-example-http/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-geospatial/pom.xml b/plugin/trino-geospatial/pom.xml index ae26068d9e22..d9d6f60cf0a1 100644 --- a/plugin/trino-geospatial/pom.xml +++ b/plugin/trino-geospatial/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-google-sheets/pom.xml b/plugin/trino-google-sheets/pom.xml index 0d101bddfd43..0aec5ceb3fca 100644 --- a/plugin/trino-google-sheets/pom.xml +++ b/plugin/trino-google-sheets/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-hive-hadoop2/pom.xml b/plugin/trino-hive-hadoop2/pom.xml index f26b23a6c61e..16e99dc6c048 100644 --- a/plugin/trino-hive-hadoop2/pom.xml +++ b/plugin/trino-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-hive/pom.xml b/plugin/trino-hive/pom.xml index 58d60df42c93..7a79f8ddc382 100644 --- a/plugin/trino-hive/pom.xml +++ b/plugin/trino-hive/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-iceberg/pom.xml b/plugin/trino-iceberg/pom.xml index 942aacc6533f..1f8513591b6e 100644 --- a/plugin/trino-iceberg/pom.xml +++ b/plugin/trino-iceberg/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-jmx/pom.xml b/plugin/trino-jmx/pom.xml index 7e0db505e63f..2a15879d17bc 100644 --- a/plugin/trino-jmx/pom.xml +++ b/plugin/trino-jmx/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-kafka/pom.xml b/plugin/trino-kafka/pom.xml index 2a26c705f5b8..2df458dcaa0c 100644 --- a/plugin/trino-kafka/pom.xml +++ b/plugin/trino-kafka/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-kinesis/pom.xml b/plugin/trino-kinesis/pom.xml index e9991073d3ca..c3400ef5cab1 100644 --- a/plugin/trino-kinesis/pom.xml +++ b/plugin/trino-kinesis/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-kudu/pom.xml b/plugin/trino-kudu/pom.xml index 0760482b2623..e7ac7c3d3197 100644 --- a/plugin/trino-kudu/pom.xml +++ b/plugin/trino-kudu/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-local-file/pom.xml b/plugin/trino-local-file/pom.xml index 7b35bf332bb4..97e75a7cddb3 100644 --- a/plugin/trino-local-file/pom.xml +++ b/plugin/trino-local-file/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-memory/pom.xml b/plugin/trino-memory/pom.xml index 51c33adb41e4..3750295a7c00 100644 --- a/plugin/trino-memory/pom.xml +++ b/plugin/trino-memory/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-memsql/pom.xml b/plugin/trino-memsql/pom.xml index d11fabdff691..e75be50df81e 100644 --- a/plugin/trino-memsql/pom.xml +++ b/plugin/trino-memsql/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-ml/pom.xml b/plugin/trino-ml/pom.xml index 1bd3bf95891a..f412149570e8 100644 --- a/plugin/trino-ml/pom.xml +++ b/plugin/trino-ml/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-mongodb/pom.xml b/plugin/trino-mongodb/pom.xml index 7eb0ada05fe5..f7b6466514f1 100644 --- a/plugin/trino-mongodb/pom.xml +++ b/plugin/trino-mongodb/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-mysql/pom.xml b/plugin/trino-mysql/pom.xml index 21c4175983f3..af55465b9d74 100644 --- a/plugin/trino-mysql/pom.xml +++ b/plugin/trino-mysql/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-oracle/pom.xml b/plugin/trino-oracle/pom.xml index b930b5601ed6..f0d62ae0c4f9 100644 --- a/plugin/trino-oracle/pom.xml +++ b/plugin/trino-oracle/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-password-authenticators/pom.xml b/plugin/trino-password-authenticators/pom.xml index 0cdf9ca8ef34..5f41100593fc 100644 --- a/plugin/trino-password-authenticators/pom.xml +++ b/plugin/trino-password-authenticators/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-phoenix/pom.xml b/plugin/trino-phoenix/pom.xml index 1f3658c20e60..7c9eeaaf77d7 100644 --- a/plugin/trino-phoenix/pom.xml +++ b/plugin/trino-phoenix/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-phoenix5/pom.xml b/plugin/trino-phoenix5/pom.xml index 30d155769ad4..95ab3b328a4f 100644 --- a/plugin/trino-phoenix5/pom.xml +++ b/plugin/trino-phoenix5/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-pinot/pom.xml b/plugin/trino-pinot/pom.xml index 5793cb5b6028..2a1c702b2146 100755 --- a/plugin/trino-pinot/pom.xml +++ b/plugin/trino-pinot/pom.xml @@ -4,7 +4,7 @@ trino-root io.trino - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-postgresql/pom.xml b/plugin/trino-postgresql/pom.xml index 428e4bd97a78..a5e6091bfa78 100644 --- a/plugin/trino-postgresql/pom.xml +++ b/plugin/trino-postgresql/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-prometheus/pom.xml b/plugin/trino-prometheus/pom.xml index b11d6f9586ce..8a19dd06cb9a 100644 --- a/plugin/trino-prometheus/pom.xml +++ b/plugin/trino-prometheus/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-raptor-legacy/pom.xml b/plugin/trino-raptor-legacy/pom.xml index fe39fffe09cb..7e8dbc0b8b08 100644 --- a/plugin/trino-raptor-legacy/pom.xml +++ b/plugin/trino-raptor-legacy/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-redis/pom.xml b/plugin/trino-redis/pom.xml index 404cf731c75e..82fddf14d896 100644 --- a/plugin/trino-redis/pom.xml +++ b/plugin/trino-redis/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-redshift/pom.xml b/plugin/trino-redshift/pom.xml index fbe84ad978c1..c4abf93fd9fa 100644 --- a/plugin/trino-redshift/pom.xml +++ b/plugin/trino-redshift/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-resource-group-managers/pom.xml b/plugin/trino-resource-group-managers/pom.xml index 11f7802d3c3b..7ca367dd671e 100644 --- a/plugin/trino-resource-group-managers/pom.xml +++ b/plugin/trino-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-session-property-managers/pom.xml b/plugin/trino-session-property-managers/pom.xml index 6d791f3c8d91..459b2d5e27ca 100644 --- a/plugin/trino-session-property-managers/pom.xml +++ b/plugin/trino-session-property-managers/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-sqlserver/pom.xml b/plugin/trino-sqlserver/pom.xml index 69c7ab62f470..1cc12b60908c 100644 --- a/plugin/trino-sqlserver/pom.xml +++ b/plugin/trino-sqlserver/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-teradata-functions/pom.xml b/plugin/trino-teradata-functions/pom.xml index 59dbd4a76264..a59e5a32d6bb 100644 --- a/plugin/trino-teradata-functions/pom.xml +++ b/plugin/trino-teradata-functions/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-thrift-api/pom.xml b/plugin/trino-thrift-api/pom.xml index 1f4122d6df9b..a9a8e40a6c36 100644 --- a/plugin/trino-thrift-api/pom.xml +++ b/plugin/trino-thrift-api/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-thrift-testing-server/pom.xml b/plugin/trino-thrift-testing-server/pom.xml index c4bc9f307a8c..31ae52d277b8 100644 --- a/plugin/trino-thrift-testing-server/pom.xml +++ b/plugin/trino-thrift-testing-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-thrift/pom.xml b/plugin/trino-thrift/pom.xml index 2ab17db6f244..1809ac40e2ba 100644 --- a/plugin/trino-thrift/pom.xml +++ b/plugin/trino-thrift/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-tpcds/pom.xml b/plugin/trino-tpcds/pom.xml index 4567f2606469..c3b98501b02b 100644 --- a/plugin/trino-tpcds/pom.xml +++ b/plugin/trino-tpcds/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-tpch/pom.xml b/plugin/trino-tpch/pom.xml index ca20c4e472a2..5c24cca6083f 100644 --- a/plugin/trino-tpch/pom.xml +++ b/plugin/trino-tpch/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index ddde5d0029a9..3abc18c1d3a8 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT trino-root Trino @@ -30,7 +30,7 @@ scm:git:git://github.com/trinodb/trino.git https://github.com/trinodb/trino - 355 + HEAD diff --git a/service/trino-proxy/pom.xml b/service/trino-proxy/pom.xml index 19f66283eae6..09367392d540 100644 --- a/service/trino-proxy/pom.xml +++ b/service/trino-proxy/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/service/trino-verifier/pom.xml b/service/trino-verifier/pom.xml index 432130440476..faa62b471de6 100644 --- a/service/trino-verifier/pom.xml +++ b/service/trino-verifier/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/testing/trino-benchmark-driver/pom.xml b/testing/trino-benchmark-driver/pom.xml index 68fb16d3038b..e4a37a9251a0 100644 --- a/testing/trino-benchmark-driver/pom.xml +++ b/testing/trino-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/testing/trino-benchmark/pom.xml b/testing/trino-benchmark/pom.xml index bcb3963a567c..8b6d71316057 100644 --- a/testing/trino-benchmark/pom.xml +++ b/testing/trino-benchmark/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/testing/trino-benchto-benchmarks/pom.xml b/testing/trino-benchto-benchmarks/pom.xml index fbf182f2258f..8eb5945f09ef 100644 --- a/testing/trino-benchto-benchmarks/pom.xml +++ b/testing/trino-benchto-benchmarks/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/testing/trino-product-tests-launcher/pom.xml b/testing/trino-product-tests-launcher/pom.xml index 81844f3e362c..3d526270c0db 100644 --- a/testing/trino-product-tests-launcher/pom.xml +++ b/testing/trino-product-tests-launcher/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/testing/trino-product-tests/pom.xml b/testing/trino-product-tests/pom.xml index 5e5e3e2dba18..5d73528c3a27 100644 --- a/testing/trino-product-tests/pom.xml +++ b/testing/trino-product-tests/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/testing/trino-server-dev/pom.xml b/testing/trino-server-dev/pom.xml index 05e5991f7b97..22890b67d29a 100644 --- a/testing/trino-server-dev/pom.xml +++ b/testing/trino-server-dev/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/testing/trino-test-jdbc-compatibility-old-driver/pom.xml b/testing/trino-test-jdbc-compatibility-old-driver/pom.xml index 75e4e95df4da..100eeab69fad 100644 --- a/testing/trino-test-jdbc-compatibility-old-driver/pom.xml +++ b/testing/trino-test-jdbc-compatibility-old-driver/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml @@ -14,7 +14,7 @@ ${project.parent.basedir} - 355 + 356-SNAPSHOT diff --git a/testing/trino-test-jdbc-compatibility-old-server/pom.xml b/testing/trino-test-jdbc-compatibility-old-server/pom.xml index e0d5496c0aa5..3fccb1a3447b 100644 --- a/testing/trino-test-jdbc-compatibility-old-server/pom.xml +++ b/testing/trino-test-jdbc-compatibility-old-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/testing/trino-testing-kafka/pom.xml b/testing/trino-testing-kafka/pom.xml index c7bb84b6b755..eba645ba9902 100644 --- a/testing/trino-testing-kafka/pom.xml +++ b/testing/trino-testing-kafka/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/testing/trino-testing/pom.xml b/testing/trino-testing/pom.xml index 9a772b52c7a3..340536fb6f7c 100644 --- a/testing/trino-testing/pom.xml +++ b/testing/trino-testing/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/testing/trino-testng-services/pom.xml b/testing/trino-testng-services/pom.xml index 76e9d9f1c3a0..8ae6f4494f29 100644 --- a/testing/trino-testng-services/pom.xml +++ b/testing/trino-testng-services/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml diff --git a/testing/trino-tests/pom.xml b/testing/trino-tests/pom.xml index 4892ad4d8697..ecfea56d3de6 100644 --- a/testing/trino-tests/pom.xml +++ b/testing/trino-tests/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 355 + 356-SNAPSHOT ../../pom.xml From ca97f6156e7feb5fd30ede810209f5f0719ef246 Mon Sep 17 00:00:00 2001 From: Rose Williams Date: Thu, 8 Apr 2021 14:37:12 -0400 Subject: [PATCH 080/146] Add that dynamic filtering is default --- docs/src/main/sphinx/admin/dynamic-filtering.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/src/main/sphinx/admin/dynamic-filtering.rst b/docs/src/main/sphinx/admin/dynamic-filtering.rst index b364506fa673..679be13ce3a2 100644 --- a/docs/src/main/sphinx/admin/dynamic-filtering.rst +++ b/docs/src/main/sphinx/admin/dynamic-filtering.rst @@ -23,6 +23,11 @@ from the processed dimension table on the right side of join. In the case of bro the runtime predicates generated from this collection are pushed into the local table scan on the left side of the join running on the same worker. +Dynamic filtering is enabled by default using the ``enable-dynamic-filtering`` +configuration property. To disable dynamic filtering, set the configuration +property to ``false``. Alternatively, use the session property +``enable_dynamic_filtering``. + Additionally, these runtime predicates are communicated to the coordinator over the network so that dynamic filtering can also be performed on the coordinator during enumeration of table scan splits. From fe5335b0baf1b32fa189d7201d4df4ca03da0504 Mon Sep 17 00:00:00 2001 From: Szymon Homa Date: Tue, 16 Mar 2021 22:57:18 +0100 Subject: [PATCH 081/146] Extract class MockRedirectHandler --- .../auth/external/MockRedirectHandler.java | 34 +++++++++++++++++++ .../external/TestExternalAuthentication.java | 18 ---------- 2 files changed, 34 insertions(+), 18 deletions(-) create mode 100644 client/trino-client/src/test/java/io/trino/client/auth/external/MockRedirectHandler.java diff --git a/client/trino-client/src/test/java/io/trino/client/auth/external/MockRedirectHandler.java b/client/trino-client/src/test/java/io/trino/client/auth/external/MockRedirectHandler.java new file mode 100644 index 000000000000..7aa3bebc06bb --- /dev/null +++ b/client/trino-client/src/test/java/io/trino/client/auth/external/MockRedirectHandler.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.client.auth.external; + +import java.net.URI; + +public class MockRedirectHandler + implements RedirectHandler +{ + private URI redirectedTo; + + @Override + public void redirectTo(URI uri) + throws RedirectException + { + redirectedTo = uri; + } + + public URI redirectedTo() + { + return redirectedTo; + } +} diff --git a/client/trino-client/src/test/java/io/trino/client/auth/external/TestExternalAuthentication.java b/client/trino-client/src/test/java/io/trino/client/auth/external/TestExternalAuthentication.java index 279d1966b4c7..dce341d20392 100644 --- a/client/trino-client/src/test/java/io/trino/client/auth/external/TestExternalAuthentication.java +++ b/client/trino-client/src/test/java/io/trino/client/auth/external/TestExternalAuthentication.java @@ -125,22 +125,4 @@ public void testObtainTokenWhenNoRedirectUriHasBeenProvided() assertThat(redirectHandler.redirectedTo()).isNull(); assertThat(token).map(Token::token).hasValue(AUTH_TOKEN); } - - private static class MockRedirectHandler - implements RedirectHandler - { - private URI redirectedTo; - - @Override - public void redirectTo(URI uri) - throws RedirectException - { - redirectedTo = uri; - } - - public URI redirectedTo() - { - return redirectedTo; - } - } } From ceb2d1b8238c4b6949abf2e7f506aab83bf1ea02 Mon Sep 17 00:00:00 2001 From: Szymon Homa Date: Tue, 16 Mar 2021 23:30:27 +0100 Subject: [PATCH 082/146] Add cached token option to jdbc externalAuthentication This change allows sharing external authentication tokens between different Connections. Each time when a new token is required, first Connection that needs it, will handle obtaining a new token when all the other Connections wait for this operation to finish. Token is kept in memmory, guarded by ReadWriteLock. To enable token cache use externalAuthenticationTokenCache=MEMORY Default value for externalAuthenticationTokenCache is NONE. --- .../main/java/io/trino/cli/QueryRunner.java | 2 + client/trino-client/pom.xml | 6 + .../auth/external/ExternalAuthenticator.java | 32 +-- .../client/auth/external/KnownToken.java | 34 +++ .../client/auth/external/LocalKnownToken.java | 46 ++++ .../auth/external/MemoryCachedKnownToken.java | 83 +++++++ .../auth/external/MockRedirectHandler.java | 25 +++ .../client/auth/external/MockTokenPoller.java | 12 +- .../external/TestExternalAuthenticator.java | 208 +++++++++++++++++- .../io/trino/jdbc/ConnectionProperties.java | 11 + .../java/io/trino/jdbc/KnownTokenCache.java | 36 +++ .../java/io/trino/jdbc/TrinoDriverUri.java | 5 +- 12 files changed, 475 insertions(+), 25 deletions(-) create mode 100644 client/trino-client/src/main/java/io/trino/client/auth/external/KnownToken.java create mode 100644 client/trino-client/src/main/java/io/trino/client/auth/external/LocalKnownToken.java create mode 100644 client/trino-client/src/main/java/io/trino/client/auth/external/MemoryCachedKnownToken.java create mode 100644 client/trino-jdbc/src/main/java/io/trino/jdbc/KnownTokenCache.java diff --git a/client/trino-cli/src/main/java/io/trino/cli/QueryRunner.java b/client/trino-cli/src/main/java/io/trino/cli/QueryRunner.java index 680118dc63bc..8160fab605a3 100644 --- a/client/trino-cli/src/main/java/io/trino/cli/QueryRunner.java +++ b/client/trino-cli/src/main/java/io/trino/cli/QueryRunner.java @@ -20,6 +20,7 @@ import io.trino.client.StatementClient; import io.trino.client.auth.external.ExternalAuthenticator; import io.trino.client.auth.external.HttpTokenPoller; +import io.trino.client.auth.external.KnownToken; import io.trino.client.auth.external.RedirectHandler; import io.trino.client.auth.external.TokenPoller; import okhttp3.OkHttpClient; @@ -193,6 +194,7 @@ private static void setupExternalAuth( ExternalAuthenticator authenticator = new ExternalAuthenticator( redirectHandler, poller, + KnownToken.local(), Duration.ofMinutes(10)); builder.authenticator(authenticator); diff --git a/client/trino-client/pom.xml b/client/trino-client/pom.xml index 83cef7f84ab9..62e612ab4b5e 100644 --- a/client/trino-client/pom.xml +++ b/client/trino-client/pom.xml @@ -81,6 +81,12 @@ test + + io.airlift + concurrent + test + + com.squareup.okhttp3 mockwebserver diff --git a/client/trino-client/src/main/java/io/trino/client/auth/external/ExternalAuthenticator.java b/client/trino-client/src/main/java/io/trino/client/auth/external/ExternalAuthenticator.java index 2269cf87c2ac..e91ff6572dcd 100644 --- a/client/trino-client/src/main/java/io/trino/client/auth/external/ExternalAuthenticator.java +++ b/client/trino-client/src/main/java/io/trino/client/auth/external/ExternalAuthenticator.java @@ -44,12 +44,13 @@ public class ExternalAuthenticator private final TokenPoller tokenPoller; private final RedirectHandler redirectHandler; private final Duration timeout; - private Token knownToken; + private final KnownToken knownToken; - public ExternalAuthenticator(RedirectHandler redirect, TokenPoller tokenPoller, Duration timeout) + public ExternalAuthenticator(RedirectHandler redirect, TokenPoller tokenPoller, KnownToken knownToken, Duration timeout) { this.tokenPoller = requireNonNull(tokenPoller, "tokenPoller is null"); this.redirectHandler = requireNonNull(redirect, "redirect is null"); + this.knownToken = requireNonNull(knownToken, "knownToken is null"); this.timeout = requireNonNull(timeout, "timeout is null"); } @@ -57,28 +58,27 @@ public ExternalAuthenticator(RedirectHandler redirect, TokenPoller tokenPoller, @Override public Request authenticate(Route route, Response response) { - knownToken = null; - - Optional authentication = toAuthentication(response); - if (!authentication.isPresent()) { - return null; - } + knownToken.setupToken(() -> { + Optional authentication = toAuthentication(response); + if (!authentication.isPresent()) { + return Optional.empty(); + } - Optional token = authentication.get().obtainToken(timeout, redirectHandler, tokenPoller); - if (!token.isPresent()) { - return null; - } + return authentication.get().obtainToken(timeout, redirectHandler, tokenPoller); + }); - knownToken = token.get(); - return withBearerToken(response.request(), knownToken); + return knownToken.getToken() + .map(token -> withBearerToken(response.request(), token)) + .orElse(null); } @Override public Response intercept(Chain chain) throws IOException { - if (knownToken != null) { - return chain.proceed(withBearerToken(chain.request(), knownToken)); + Optional token = knownToken.getToken(); + if (token.isPresent()) { + return chain.proceed(withBearerToken(chain.request(), token.get())); } return chain.proceed(chain.request()); diff --git a/client/trino-client/src/main/java/io/trino/client/auth/external/KnownToken.java b/client/trino-client/src/main/java/io/trino/client/auth/external/KnownToken.java new file mode 100644 index 000000000000..240af9075763 --- /dev/null +++ b/client/trino-client/src/main/java/io/trino/client/auth/external/KnownToken.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.client.auth.external; + +import java.util.Optional; +import java.util.function.Supplier; + +public interface KnownToken +{ + Optional getToken(); + + void setupToken(Supplier> tokenSource); + + static KnownToken local() + { + return new LocalKnownToken(); + } + + static KnownToken memoryCached() + { + return MemoryCachedKnownToken.INSTANCE; + } +} diff --git a/client/trino-client/src/main/java/io/trino/client/auth/external/LocalKnownToken.java b/client/trino-client/src/main/java/io/trino/client/auth/external/LocalKnownToken.java new file mode 100644 index 000000000000..40b984a0764b --- /dev/null +++ b/client/trino-client/src/main/java/io/trino/client/auth/external/LocalKnownToken.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.client.auth.external; + +import javax.annotation.concurrent.NotThreadSafe; + +import java.util.Optional; +import java.util.function.Supplier; + +import static java.util.Objects.requireNonNull; + +/** + * LocalKnownToken class keeps the token on its field + * and it's designed to use it in fully serialized manner. + */ +@NotThreadSafe +class LocalKnownToken + implements KnownToken +{ + private Optional knownToken = Optional.empty(); + + @Override + public Optional getToken() + { + return knownToken; + } + + @Override + public void setupToken(Supplier> tokenSource) + { + requireNonNull(tokenSource, "tokenSource is null"); + + knownToken = tokenSource.get(); + } +} diff --git a/client/trino-client/src/main/java/io/trino/client/auth/external/MemoryCachedKnownToken.java b/client/trino-client/src/main/java/io/trino/client/auth/external/MemoryCachedKnownToken.java new file mode 100644 index 000000000000..e8513e4bd87f --- /dev/null +++ b/client/trino-client/src/main/java/io/trino/client/auth/external/MemoryCachedKnownToken.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.client.auth.external; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Optional; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Supplier; + +/** + * This KnownToken instance forces all Connections to reuse same token. + * Every time an existing token is considered to be invalid each Connection + * will try to obtain a new token, but only the first one will actually do the job, + * where every other connection will be waiting on readLock + * until obtaining new token finishes. + *

+ * In general the game is to reuse same token and obtain it only once, no matter how + * many Connections will be actively using it. It's very important as obtaining the new token + * will take minutes, as it mostly requires user thinking time. + */ +@ThreadSafe +class MemoryCachedKnownToken + implements KnownToken +{ + public static final MemoryCachedKnownToken INSTANCE = new MemoryCachedKnownToken(); + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final Lock readLock = lock.readLock(); + private final Lock writeLock = lock.writeLock(); + private Optional knownToken = Optional.empty(); + + private MemoryCachedKnownToken() + { + } + + @Override + public Optional getToken() + { + try { + readLock.lockInterruptibly(); + return knownToken; + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + finally { + readLock.unlock(); + } + } + + @Override + public void setupToken(Supplier> tokenSource) + { + // Try to lock and generate new token. If some other thread (Connection) has + // already obtained writeLock and is generating new token, then skipp this + // to block on getToken() + if (writeLock.tryLock()) { + try { + // Clear knownToken before obtaining new token, as it might fail leaving old invalid token. + knownToken = Optional.empty(); + knownToken = tokenSource.get(); + } + finally { + writeLock.unlock(); + } + } + } +} diff --git a/client/trino-client/src/test/java/io/trino/client/auth/external/MockRedirectHandler.java b/client/trino-client/src/test/java/io/trino/client/auth/external/MockRedirectHandler.java index 7aa3bebc06bb..cf671ea391c9 100644 --- a/client/trino-client/src/test/java/io/trino/client/auth/external/MockRedirectHandler.java +++ b/client/trino-client/src/test/java/io/trino/client/auth/external/MockRedirectHandler.java @@ -14,21 +14,46 @@ package io.trino.client.auth.external; import java.net.URI; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; public class MockRedirectHandler implements RedirectHandler { private URI redirectedTo; + private AtomicInteger redirectionCount = new AtomicInteger(0); + private Duration redirectTime; @Override public void redirectTo(URI uri) throws RedirectException { redirectedTo = uri; + redirectionCount.incrementAndGet(); + try { + if (redirectTime != null) { + Thread.sleep(redirectTime.toMillis()); + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } } public URI redirectedTo() { return redirectedTo; } + + public int getRedirectionCount() + { + return redirectionCount.get(); + } + + public MockRedirectHandler sleepOnRedirect(Duration redirectTime) + { + this.redirectTime = redirectTime; + return this; + } } diff --git a/client/trino-client/src/test/java/io/trino/client/auth/external/MockTokenPoller.java b/client/trino-client/src/test/java/io/trino/client/auth/external/MockTokenPoller.java index fae34ea9c031..d07205f38636 100644 --- a/client/trino-client/src/test/java/io/trino/client/auth/external/MockTokenPoller.java +++ b/client/trino-client/src/test/java/io/trino/client/auth/external/MockTokenPoller.java @@ -17,21 +17,21 @@ import java.net.URI; import java.time.Duration; -import java.util.ArrayDeque; -import java.util.HashMap; import java.util.Map; -import java.util.Queue; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingDeque; public final class MockTokenPoller implements TokenPoller { - private final Map> results = new HashMap<>(); + private final Map> results = new ConcurrentHashMap<>(); public MockTokenPoller withResult(URI tokenUri, TokenPollResult result) { results.compute(tokenUri, (uri, queue) -> { if (queue == null) { - return new ArrayDeque<>(ImmutableList.of(result)); + return new LinkedBlockingDeque<>(ImmutableList.of(result)); } queue.add(result); return queue; @@ -42,7 +42,7 @@ public MockTokenPoller withResult(URI tokenUri, TokenPollResult result) @Override public TokenPollResult pollForToken(URI tokenUri, Duration ignored) { - Queue queue = results.get(tokenUri); + BlockingDeque queue = results.get(tokenUri); if (queue == null) { throw new IllegalArgumentException("Unknown token URI: " + tokenUri); } diff --git a/client/trino-client/src/test/java/io/trino/client/auth/external/TestExternalAuthenticator.java b/client/trino-client/src/test/java/io/trino/client/auth/external/TestExternalAuthenticator.java index 729ffe6fa023..c9ea6fc73199 100644 --- a/client/trino-client/src/test/java/io/trino/client/auth/external/TestExternalAuthenticator.java +++ b/client/trino-client/src/test/java/io/trino/client/auth/external/TestExternalAuthenticator.java @@ -13,31 +13,55 @@ */ package io.trino.client.auth.external; +import com.google.common.collect.ImmutableList; import io.trino.client.ClientException; import okhttp3.HttpUrl; import okhttp3.Protocol; import okhttp3.Request; import okhttp3.Response; +import org.assertj.core.api.ListAssert; +import org.assertj.core.api.ThrowableAssert; +import org.testng.annotations.AfterClass; import org.testng.annotations.Test; import java.net.URI; import java.net.URISyntaxException; import java.time.Duration; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.stream.Stream; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.net.HttpHeaders.AUTHORIZATION; import static com.google.common.net.HttpHeaders.WWW_AUTHENTICATE; +import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.trino.client.auth.external.ExternalAuthenticator.TOKEN_URI_FIELD; import static io.trino.client.auth.external.ExternalAuthenticator.toAuthentication; import static io.trino.client.auth.external.TokenPollResult.successful; import static java.lang.String.format; import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED; import static java.net.URI.create; +import static java.util.concurrent.Executors.newCachedThreadPool; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestExternalAuthenticator { + private static final ExecutorService executor = newCachedThreadPool(daemonThreadsNamed(TestExternalAuthenticator.class.getName() + "-%d")); + + @AfterClass(alwaysRun = true) + public void shutDownThreadPool() + { + executor.shutdownNow(); + } + @Test public void testChallengeWithOnlyTokenServerUri() { @@ -110,7 +134,7 @@ public void testAuthentication() { MockTokenPoller tokenPoller = new MockTokenPoller() .withResult(URI.create("http://token.uri"), successful(new Token("valid-token"))); - ExternalAuthenticator authenticator = new ExternalAuthenticator(uri -> {}, tokenPoller, Duration.ofSeconds(1)); + ExternalAuthenticator authenticator = new ExternalAuthenticator(uri -> {}, tokenPoller, KnownToken.local(), Duration.ofSeconds(1)); Request authenticated = authenticator.authenticate(null, getUnauthorizedResponse("Bearer x_token_server=\"http://token.uri\"")); @@ -125,7 +149,7 @@ public void testReAuthenticationAfterRejectingToken() MockTokenPoller tokenPoller = new MockTokenPoller() .withResult(URI.create("http://token.uri"), successful(new Token("first-token"))) .withResult(URI.create("http://token.uri"), successful(new Token("second-token"))); - ExternalAuthenticator authenticator = new ExternalAuthenticator(uri -> {}, tokenPoller, Duration.ofSeconds(1)); + ExternalAuthenticator authenticator = new ExternalAuthenticator(uri -> {}, tokenPoller, KnownToken.local(), Duration.ofSeconds(1)); Request request = authenticator.authenticate(null, getUnauthorizedResponse("Bearer x_token_server=\"http://token.uri\"")); Request reAuthenticated = authenticator.authenticate(null, getUnauthorizedResponse("Bearer x_token_server=\"http://token.uri\"", request)); @@ -134,6 +158,140 @@ public void testReAuthenticationAfterRejectingToken() .containsExactly("Bearer second-token"); } + @Test(timeOut = 2000) + public void testAuthenticationFromMultipleThreadsWithLocallyStoredToken() + { + MockTokenPoller tokenPoller = new MockTokenPoller() + .withResult(URI.create("http://token.uri"), successful(new Token("valid-token-1"))) + .withResult(URI.create("http://token.uri"), successful(new Token("valid-token-2"))) + .withResult(URI.create("http://token.uri"), successful(new Token("valid-token-3"))) + .withResult(URI.create("http://token.uri"), successful(new Token("valid-token-4"))); + MockRedirectHandler redirectHandler = new MockRedirectHandler(); + + ExternalAuthenticator authenticator = new ExternalAuthenticator(redirectHandler, tokenPoller, KnownToken.local(), Duration.ofSeconds(1)); + List> requests = times( + 4, + () -> authenticator.authenticate(null, getUnauthorizedResponse("Bearer x_token_server=\"http://token.uri\", x_redirect_server=\"http://redirect.uri\""))) + .map(executor::submit) + .collect(toImmutableList()); + + ConcurrentRequestAssertion assertion = new ConcurrentRequestAssertion(requests); + assertion.requests() + .extracting(Request::headers) + .extracting(headers -> headers.get(AUTHORIZATION)) + .contains("Bearer valid-token-1", "Bearer valid-token-2", "Bearer valid-token-3", "Bearer valid-token-4"); + assertion.assertThatNoExceptionsHasBeenThrown(); + assertThat(redirectHandler.getRedirectionCount()).isEqualTo(4); + } + + @Test(timeOut = 2000) + public void testAuthenticationFromMultipleThreadsWithCachedToken() + { + ExecutorService executor = newCachedThreadPool(daemonThreadsNamed(this.getClass().getName() + "%n")); + MockTokenPoller tokenPoller = new MockTokenPoller() + .withResult(URI.create("http://token.uri"), successful(new Token("valid-token"))); + MockRedirectHandler redirectHandler = new MockRedirectHandler() + .sleepOnRedirect(Duration.ofMillis(10)); + + ExternalAuthenticator authenticator = new ExternalAuthenticator(redirectHandler, tokenPoller, KnownToken.memoryCached(), Duration.ofSeconds(1)); + List> requests = times( + 4, + () -> authenticator.authenticate(null, getUnauthorizedResponse("Bearer x_token_server=\"http://token.uri\", x_redirect_server=\"http://redirect.uri\""))) + .map(executor::submit) + .collect(toImmutableList()); + + ConcurrentRequestAssertion assertion = new ConcurrentRequestAssertion(requests); + assertion.requests() + .extracting(Request::headers) + .extracting(headers -> headers.get(AUTHORIZATION)) + .containsOnly("Bearer valid-token"); + assertion.assertThatNoExceptionsHasBeenThrown(); + assertThat(redirectHandler.getRedirectionCount()).isEqualTo(1); + } + + @Test(timeOut = 2000) + public void testAuthenticationFromMultipleThreadsWithCachedTokenAfterAuthenticateFails() + { + MockTokenPoller tokenPoller = new MockTokenPoller() + .withResult(URI.create("http://token.uri"), TokenPollResult.successful(new Token("first-token"))) + .withResult(URI.create("http://token.uri"), TokenPollResult.failed("external authentication error")); + MockRedirectHandler redirectHandler = new MockRedirectHandler() + .sleepOnRedirect(Duration.ofMillis(10)); + + ExternalAuthenticator authenticator = new ExternalAuthenticator(redirectHandler, tokenPoller, KnownToken.memoryCached(), Duration.ofSeconds(1)); + Request firstRequest = authenticator.authenticate(null, getUnauthorizedResponse("Bearer x_token_server=\"http://token.uri\", x_redirect_server=\"http://redirect.uri\"")); + + List> requests = times( + 4, + () -> authenticator.authenticate(null, getUnauthorizedResponse("Bearer x_token_server=\"http://token.uri\", x_redirect_server=\"http://redirect.uri\"", firstRequest))) + .map(executor::submit) + .collect(toImmutableList()); + + ConcurrentRequestAssertion assertion = new ConcurrentRequestAssertion(requests); + assertion.requests().containsExactly(null, null, null); + assertion.firstException().hasMessage("external authentication error") + .isInstanceOf(ClientException.class); + + assertThat(redirectHandler.getRedirectionCount()).isEqualTo(2); + } + + @Test(timeOut = 2000) + public void testAuthenticationFromMultipleThreadsWithCachedTokenAfterAuthenticateTimesOut() + { + MockRedirectHandler redirectHandler = new MockRedirectHandler() + .sleepOnRedirect(Duration.ofMillis(5)); + + ExternalAuthenticator authenticator = new ExternalAuthenticator(redirectHandler, (uri, duration) -> TokenPollResult.pending(uri), KnownToken.memoryCached(), Duration.ofMillis(1)); + List> requests = times( + 4, + () -> authenticator.authenticate(null, getUnauthorizedResponse("Bearer x_token_server=\"http://token.uri\", x_redirect_server=\"http://redirect.uri\""))) + .map(executor::submit) + .collect(toImmutableList()); + + ConcurrentRequestAssertion assertion = new ConcurrentRequestAssertion(requests); + assertion.requests() + .containsExactly(null, null, null, null); + assertion.assertThatNoExceptionsHasBeenThrown(); + assertThat(redirectHandler.getRedirectionCount()).isEqualTo(1); + } + + @Test(timeOut = 2000) + public void testAuthenticationFromMultipleThreadsWithCachedTokenAfterAuthenticateIsInterrupted() + throws Exception + { + ExecutorService interruptableThreadPool = newCachedThreadPool(daemonThreadsNamed(this.getClass().getName() + "-interruptable-%d")); + MockRedirectHandler redirectHandler = new MockRedirectHandler() + .sleepOnRedirect(Duration.ofMinutes(1)); + + ExternalAuthenticator authenticator = new ExternalAuthenticator(redirectHandler, (uri, duration) -> TokenPollResult.pending(uri), KnownToken.memoryCached(), Duration.ofMillis(1)); + Future interruptedAuthentication = interruptableThreadPool.submit( + () -> authenticator.authenticate(null, getUnauthorizedResponse("Bearer x_token_server=\"http://token.uri\", x_redirect_server=\"http://redirect.uri\""))); + Thread.sleep(100); //It's here to make sure that authentication will start before the other threads. + List> requests = times( + 2, + () -> authenticator.authenticate(null, getUnauthorizedResponse("Bearer x_token_server=\"http://token.uri\", x_redirect_server=\"http://redirect.uri\""))) + .map(executor::submit) + .collect(toImmutableList()); + + Thread.sleep(100); + interruptableThreadPool.shutdownNow(); + + ConcurrentRequestAssertion assertion = new ConcurrentRequestAssertion(ImmutableList.>builder() + .addAll(requests) + .add(interruptedAuthentication) + .build()); + assertion.requests().containsExactly(null, null); + assertion.firstException().hasRootCauseInstanceOf(InterruptedException.class); + + assertThat(redirectHandler.getRedirectionCount()).isEqualTo(1); + } + + private static Stream> times(int times, Callable request) + { + return Stream.generate(() -> request) + .limit(times); + } + private static Optional buildAuthentication(String challengeHeader) { return toAuthentication(getUnauthorizedResponse(challengeHeader)); @@ -157,4 +315,50 @@ private static Response getUnauthorizedResponse(String challengeHeader, Request .header(WWW_AUTHENTICATE, challengeHeader) .build(); } + + static class ConcurrentRequestAssertion + { + private final List exceptions = new ArrayList<>(); + private final List requests = new ArrayList<>(); + + public ConcurrentRequestAssertion(List> requests) + { + for (Future request : requests) { + try { + this.requests.add(request.get()); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + catch (CancellationException ex) { + exceptions.add(ex); + } + catch (ExecutionException ex) { + checkState(ex.getCause() != null, "Missing cause on ExecutionException " + ex.getMessage()); + + exceptions.add(ex.getCause()); + } + } + } + + ThrowableAssert firstException() + { + return exceptions.stream() + .findFirst() + .map(ThrowableAssert::new) + .orElseGet(() -> new ThrowableAssert(() -> null)); + } + + void assertThatNoExceptionsHasBeenThrown() + { + assertThat(exceptions) + .isEmpty(); + } + + ListAssert requests() + { + return assertThat(requests); + } + } } diff --git a/client/trino-jdbc/src/main/java/io/trino/jdbc/ConnectionProperties.java b/client/trino-jdbc/src/main/java/io/trino/jdbc/ConnectionProperties.java index 0c1f3bea085d..5618e47bd1b4 100644 --- a/client/trino-jdbc/src/main/java/io/trino/jdbc/ConnectionProperties.java +++ b/client/trino-jdbc/src/main/java/io/trino/jdbc/ConnectionProperties.java @@ -74,6 +74,7 @@ enum SslVerificationMode public static final ConnectionProperty ACCESS_TOKEN = new AccessToken(); public static final ConnectionProperty EXTERNAL_AUTHENTICATION = new ExternalAuthentication(); public static final ConnectionProperty EXTERNAL_AUTHENTICATION_TIMEOUT = new ExternalAuthenticationTimeout(); + public static final ConnectionProperty EXTERNAL_AUTHENTICATION_TOKEN_CACHE = new ExternalAuthenticationTokenCache(); public static final ConnectionProperty> EXTRA_CREDENTIALS = new ExtraCredentials(); public static final ConnectionProperty CLIENT_INFO = new ClientInfo(); public static final ConnectionProperty CLIENT_TAGS = new ClientTags(); @@ -115,6 +116,7 @@ enum SslVerificationMode .add(SOURCE) .add(EXTERNAL_AUTHENTICATION) .add(EXTERNAL_AUTHENTICATION_TIMEOUT) + .add(EXTERNAL_AUTHENTICATION_TOKEN_CACHE) .build(); private static final Map> KEY_LOOKUP = unmodifiableMap(ALL_PROPERTIES.stream() @@ -472,6 +474,15 @@ public ExternalAuthenticationTimeout() } } + private static class ExternalAuthenticationTokenCache + extends AbstractConnectionProperty + { + public ExternalAuthenticationTokenCache() + { + super("externalAuthenticationTokenCache", Optional.of(KnownTokenCache.NONE.name()), NOT_REQUIRED, ALLOWED, KnownTokenCache::valueOf); + } + } + private static class ExtraCredentials extends AbstractConnectionProperty> { diff --git a/client/trino-jdbc/src/main/java/io/trino/jdbc/KnownTokenCache.java b/client/trino-jdbc/src/main/java/io/trino/jdbc/KnownTokenCache.java new file mode 100644 index 000000000000..6c3dde57d8c7 --- /dev/null +++ b/client/trino-jdbc/src/main/java/io/trino/jdbc/KnownTokenCache.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.jdbc; + +import io.trino.client.auth.external.KnownToken; + +public enum KnownTokenCache +{ + NONE { + @Override + KnownToken create() + { + return KnownToken.local(); + } + }, + MEMORY { + @Override + KnownToken create() + { + return KnownToken.memoryCached(); + } + }; + + abstract KnownToken create(); +} diff --git a/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoDriverUri.java b/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoDriverUri.java index 23bf7440a46a..b4c8e6121d64 100644 --- a/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoDriverUri.java +++ b/client/trino-jdbc/src/main/java/io/trino/jdbc/TrinoDriverUri.java @@ -59,6 +59,7 @@ import static io.trino.jdbc.ConnectionProperties.DISABLE_COMPRESSION; import static io.trino.jdbc.ConnectionProperties.EXTERNAL_AUTHENTICATION; import static io.trino.jdbc.ConnectionProperties.EXTERNAL_AUTHENTICATION_TIMEOUT; +import static io.trino.jdbc.ConnectionProperties.EXTERNAL_AUTHENTICATION_TOKEN_CACHE; import static io.trino.jdbc.ConnectionProperties.EXTRA_CREDENTIALS; import static io.trino.jdbc.ConnectionProperties.HTTP_PROXY; import static io.trino.jdbc.ConnectionProperties.KERBEROS_CONFIG_PATH; @@ -317,7 +318,9 @@ public void setupClient(OkHttpClient.Builder builder) .map(value -> Duration.ofMillis(value.toMillis())) .orElse(Duration.ofMinutes(2)); - ExternalAuthenticator authenticator = new ExternalAuthenticator(REDIRECT_HANDLER.get(), poller, timeout); + KnownTokenCache knownTokenCache = EXTERNAL_AUTHENTICATION_TOKEN_CACHE.getValue(properties).get(); + + ExternalAuthenticator authenticator = new ExternalAuthenticator(REDIRECT_HANDLER.get(), poller, knownTokenCache.create(), timeout); builder.authenticator(authenticator); builder.addInterceptor(authenticator); From ed98f2470edb75ad96c0c8698560c860547f068d Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Thu, 8 Apr 2021 14:48:19 -0700 Subject: [PATCH 083/146] Add missing functions by name --- docs/src/main/sphinx/functions/list.rst | 41 ++++++++++++++++++++----- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/docs/src/main/sphinx/functions/list.rst b/docs/src/main/sphinx/functions/list.rst index cf2af1b99862..22edc5f55c23 100644 --- a/docs/src/main/sphinx/functions/list.rst +++ b/docs/src/main/sphinx/functions/list.rst @@ -65,15 +65,19 @@ B - :func:`bing_tile_at` - :func:`bing_tile_coordinates` - :func:`bing_tile_polygon` +- ``bing_tile_quadkey`` - :func:`bing_tile_zoom_level` - :func:`bing_tiles_around` - :func:`bit_count` - :func:`bitwise_and` +- :func:`bitwise_and_agg` +- ``bitwise_left_shift`` - :func:`bitwise_not` - :func:`bitwise_or` -- :func:`bitwise_xor` -- :func:`bitwise_and_agg` - :func:`bitwise_or_agg` +- ``bitwise_right_shift`` +- ``bitwise_right_shift_arithmetic`` +- :func:`bitwise_xor` - :func:`bool_and` - :func:`bool_or` @@ -86,8 +90,8 @@ C - :func:`cbrt` - :func:`ceil` - :func:`ceiling` -- :func:`checksum` - :func:`char2hexint` +- :func:`checksum` - :func:`chr` - :func:`classify` - :ref:`coalesce ` @@ -105,8 +109,8 @@ C - :func:`cosine_similarity` - :func:`count` - :func:`count_if` -- :func:`covar_samp` - :func:`covar_pop` +- :func:`covar_samp` - :func:`crc32` - :func:`cume_dist` - :data:`current_date` @@ -140,6 +144,7 @@ E - :func:`e` - :func:`element_at` - :func:`empty_approx_set` +- ``evaluate_classifier_predictions`` - :func:`every` - :func:`extract` - :func:`exp` @@ -153,18 +158,22 @@ F - :func:`flatten` - :func:`floor` - :func:`format` +- ``format_datetime`` - :func:`from_base` - :func:`from_base64` - :func:`from_base64url` - :func:`from_big_endian_32` - :func:`from_big_endian_64` - :func:`from_encoded_polyline` +- ``from_geojson_geometry`` - :func:`from_hex` - :func:`from_ieee754_32` - :func:`from_ieee754_64` - :func:`from_iso8601_date` - :func:`from_iso8601_timestamp` +- ``from_iso8601_timestamp_nanos`` - :func:`from_unixtime` +- ``from_unixtime_nanos`` - :func:`from_utf8` G @@ -183,12 +192,14 @@ H - - :func:`hamming_distance` +- ``hash_counts`` - :func:`histogram` - :func:`hmac_md5` - :func:`hmac_sha1` - :func:`hmac_sha256` - :func:`hmac_sha512` - :func:`hour` +- ``human_readable_seconds`` I - @@ -196,6 +207,7 @@ I - :ref:`if ` - :func:`index` - :func:`infinity` +- ``intersection_cardinality`` - :func:`inverse_beta_cdf` - :func:`inverse_normal_cdf` - :func:`is_finite` @@ -210,6 +222,7 @@ I J - +- ``jaccard_index`` - :func:`json_array_contains` - :func:`json_array_get` - :func:`json_array_length` @@ -249,10 +262,12 @@ L - :func:`lower` - :func:`lpad` - :func:`ltrim` +- ``luhn_check`` M - +- ``make_set_digest`` - :func:`map` - :func:`map_agg` - :func:`map_concat` @@ -260,12 +275,14 @@ M - :func:`map_filter` - :func:`map_from_entries` - :func:`map_keys` +- ``map_union`` - :func:`map_values` - :func:`map_zip_with` - :func:`max` - :func:`max_by` - :func:`md5` - :func:`merge` +- ``merge_set_digest`` - :func:`millisecond` - :func:`min` - :func:`min_by` @@ -274,6 +291,7 @@ M - :func:`month` - :func:`multimap_agg` - :func:`multimap_from_entries` +- ``murmur3`` N - @@ -294,6 +312,8 @@ N O - +- ``objectid`` +- ``objectid_timestamp`` - :ref:`OR ` P @@ -355,9 +375,11 @@ S - :func:`sign` - :func:`simplify_geometry` - :func:`sin` -- :func:`slice` - :func:`skewness` +- :func:`slice` - :ref:`SOME ` +- ``spatial_partitioning`` +- ``spatial_partitions`` - :func:`split` - :func:`split_part` - :func:`split_to_map` @@ -385,8 +407,8 @@ S - :func:`ST_Equals` - :func:`ST_ExteriorRing` - :func:`ST_Geometries` -- :func:`ST_GeometryN` - :func:`ST_GeometryFromText` +- :func:`ST_GeometryN` - :func:`ST_GeometryType` - :func:`ST_GeomFromBinary` - :func:`ST_InteriorRingN` @@ -395,14 +417,15 @@ S - :func:`ST_Intersects` - :func:`ST_IsClosed` - :func:`ST_IsEmpty` -- :func:`ST_IsSimple` - :func:`ST_IsRing` +- :func:`ST_IsSimple` - :func:`ST_IsValid` - :func:`ST_Length` - :func:`ST_LineFromText` - :func:`ST_LineString` - :func:`ST_MultiPoint` - :func:`ST_NumGeometries` +- ``ST_NumInteriorRing`` - :func:`ST_NumPoints` - :func:`ST_Overlaps` - :func:`ST_Point` @@ -435,15 +458,18 @@ T - :func:`tan` - :func:`tanh` +- ``tdigest_agg`` - :func:`timezone_hour` - :func:`timezone_minute` - :func:`to_base` - :func:`to_base64` +- ``to_base64url`` - :func:`to_big_endian_32` - :func:`to_big_endian_64` - :func:`to_char` - :func:`to_date` - :func:`to_encoded_polyline` +- ``to_geojson_geometry`` - :func:`to_geometry` - :func:`to_hex` - :func:`to_ieee754_32` @@ -474,6 +500,7 @@ U - :func:`url_extract_host` - :func:`url_extract_parameter` - :func:`url_extract_path` +- ``url_extract_protocol`` - :func:`url_extract_port` - :func:`url_extract_query` - :func:`uuid` From b39168f3ac833d90783264bd142753358cfa5dd2 Mon Sep 17 00:00:00 2001 From: Ashhar Hasan Date: Thu, 1 Apr 2021 13:49:50 +0530 Subject: [PATCH 084/146] Add method to list all objects matching a given prefix All existing methods in TrinoS3Filesystem list objects rooted at the provided path. This makes listing inefficient for flat structures where we require multiple API calls to S3 and end up filtering on the client side. A new method is added which lists all objects whose absolute path matches a given prefix. This makes searching for all objects matching a provided prefix efficient. All other methods retain their existing behavior in this commit. --- .../plugin/hive/s3/TrinoS3FileSystem.java | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystem.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystem.java index a577c9ed754f..2b57d09ba69b 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystem.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/s3/TrinoS3FileSystem.java @@ -344,14 +344,21 @@ public RemoteIterator listFiles(Path path, boolean recursive) { // Either a single level or full listing, depending on the recursive flag, no "directories" // included in either path - return new S3ObjectsV2RemoteIterator(listPrefix(path, OptionalInt.empty(), recursive ? ListingMode.RECURSIVE_FILES_ONLY : ListingMode.SHALLOW_FILES_ONLY)); + return new S3ObjectsV2RemoteIterator(listPath(path, OptionalInt.empty(), recursive ? ListingMode.RECURSIVE_FILES_ONLY : ListingMode.SHALLOW_FILES_ONLY)); + } + + public RemoteIterator listFilesByPrefix(Path prefix, boolean recursive) + { + // Either a single level or full listing, depending on the recursive flag, no "directories" + // included in either path + return new S3ObjectsV2RemoteIterator(listPrefix(keyFromPath(prefix), OptionalInt.empty(), recursive ? ListingMode.RECURSIVE_FILES_ONLY : ListingMode.SHALLOW_FILES_ONLY)); } @Override public RemoteIterator listLocatedStatus(Path path) { STATS.newListLocatedStatusCall(); - return new S3ObjectsV2RemoteIterator(listPrefix(path, OptionalInt.empty(), ListingMode.SHALLOW_ALL)); + return new S3ObjectsV2RemoteIterator(listPath(path, OptionalInt.empty(), ListingMode.SHALLOW_ALL)); } private static final class S3ObjectsV2RemoteIterator @@ -405,7 +412,7 @@ public FileStatus getFileStatus(Path path) if (metadata == null) { // check if this path is a directory - Iterator iterator = listPrefix(path, OptionalInt.of(1), ListingMode.SHALLOW_ALL); + Iterator iterator = listPath(path, OptionalInt.of(1), ListingMode.SHALLOW_ALL); if (iterator.hasNext()) { return new FileStatus(0, true, 1, 0, 0, qualifiedPath(path)); } @@ -605,16 +612,27 @@ public boolean isFilesOnly() } } - private Iterator listPrefix(Path path, OptionalInt initialMaxKeys, ListingMode mode) + /** + * List all objects rooted at the provided path. + */ + private Iterator listPath(Path path, OptionalInt initialMaxKeys, ListingMode mode) { String key = keyFromPath(path); if (!key.isEmpty()) { key += PATH_SEPARATOR; } + return listPrefix(key, initialMaxKeys, mode); + } + + /** + * List all objects whose absolute path matches the provided prefix. + */ + private Iterator listPrefix(String prefix, OptionalInt initialMaxKeys, ListingMode mode) + { ListObjectsV2Request request = new ListObjectsV2Request() .withBucketName(getBucketName(uri)) - .withPrefix(key) + .withPrefix(prefix) .withDelimiter(mode == ListingMode.RECURSIVE_FILES_ONLY ? null : PATH_SEPARATOR) .withMaxKeys(initialMaxKeys.isPresent() ? initialMaxKeys.getAsInt() : null) .withRequesterPays(requesterPaysEnabled); From 5f3381f22d052b23dd7e418aa53a3f2085a603b8 Mon Sep 17 00:00:00 2001 From: "Jacob I. Komissar" Date: Mon, 5 Apr 2021 14:11:02 -0400 Subject: [PATCH 085/146] Replace use of isPresent with map and orElseGet in ParquetPageSource --- .../trino/plugin/hive/parquet/ParquetPageSource.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java index 8ab0162bd4a1..8c82036e23ff 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java @@ -92,13 +92,10 @@ public Page getNextPage() Block[] blocks = new Block[fields.size()]; for (int fieldId = 0; fieldId < blocks.length; fieldId++) { - Optional field = fields.get(fieldId); - if (field.isPresent()) { - blocks[fieldId] = new LazyBlock(batchSize, new ParquetBlockLoader(field.get())); - } - else { - blocks[fieldId] = RunLengthEncodedBlock.create(types.get(fieldId), null, batchSize); - } + Type type = types.get(fieldId); + blocks[fieldId] = fields.get(fieldId) + .map(field -> new LazyBlock(batchSize, new ParquetBlockLoader(field))) + .orElseGet(() -> RunLengthEncodedBlock.create(type, null, batchSize)); } return new Page(batchSize, blocks); } From 1f3955a67e8862c729e472c9bc9689d0d23dda7b Mon Sep 17 00:00:00 2001 From: "Jacob I. Komissar" Date: Tue, 6 Apr 2021 13:24:54 -0400 Subject: [PATCH 086/146] Add a doc comment and error message in ParquetPageSource --- .../io/trino/plugin/hive/parquet/ParquetPageSource.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java index 8c82036e23ff..c1dddc0571cb 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java @@ -142,6 +142,10 @@ public void close() private final class ParquetBlockLoader implements LazyBlockLoader { + /** + * Stores batch ID at instantiation time. Loading fails if the ID + * changes before {@link #load()} is called. + */ private final int expectedBatchId = batchId; private final Field field; private boolean loaded; @@ -155,7 +159,7 @@ public ParquetBlockLoader(Field field) public final Block load() { checkState(!loaded, "Already loaded"); - checkState(batchId == expectedBatchId); + checkState(batchId == expectedBatchId, "Inconsistent state; wrong batch"); Block block; String parquetDataSourceId = parquetReader.getDataSource().getId().toString(); From 92ee3345657e19d33ed3a28d3495206595508616 Mon Sep 17 00:00:00 2001 From: Matthias Strobl Date: Mon, 14 Sep 2020 14:27:07 +0200 Subject: [PATCH 087/146] Add support for optional Kafka SSL Co-authored-by: Kenzyme Le Co-authored-by: Matthias Strobl --- docs/src/main/sphinx/connector/kafka.rst | 76 +++++++ plugin/trino-kafka/pom.xml | 5 + .../plugin/kafka/KafkaClientsModule.java | 60 ++++++ .../io/trino/plugin/kafka/KafkaConfig.java | 16 ++ .../plugin/kafka/KafkaConnectorFactory.java | 2 + .../io/trino/plugin/kafka/KafkaPlugin.java | 15 +- .../plugin/kafka/KafkaProducerFactory.java | 1 - .../kafka/PlainTextKafkaAdminFactory.java | 5 + .../kafka/PlainTextKafkaConsumerFactory.java | 5 + .../kafka/PlainTextKafkaProducerFactory.java | 5 + .../plugin/kafka/SslKafkaAdminFactory.java | 51 +++++ .../plugin/kafka/SslKafkaConsumerFactory.java | 50 +++++ .../plugin/kafka/SslKafkaProducerFactory.java | 50 +++++ .../plugin/kafka/security/ForKafkaSsl.java | 31 +++ .../KafkaEndpointIdentificationAlgorithm.java | 41 ++++ .../KafkaKeystoreTruststoreType.java} | 19 +- .../kafka/security/KafkaSecurityModule.java | 42 ++++ .../plugin/kafka/security/KafkaSslConfig.java | 187 ++++++++++++++++++ .../SecurityProtocol.java} | 16 +- .../SslSecurityModule.java} | 9 +- .../trino/plugin/kafka/TestKafkaConfig.java | 8 +- .../trino/plugin/kafka/TestKafkaPlugin.java | 111 +++++++++++ .../plugin/kafka/TestKafkaSslConfig.java | 176 +++++++++++++++++ .../launcher/env/EnvironmentModule.java | 2 + .../product/launcher/env/common/Kafka.java | 13 +- .../product/launcher/env/common/KafkaSsl.java | 77 ++++++++ .../env/environment/MultinodeKafkaSsl.java | 77 ++++++++ .../suite/suites/Suite6NonGeneric.java | 4 +- .../multinode-kafka-ssl/kafka.properties | 28 +++ .../kafka_schema_registry.properties | 14 ++ .../multinode-kafka-ssl/secrets/.gitignore | 12 ++ .../secrets/broker1_keystore_creds | 1 + .../secrets/broker1_sslkey_creds | 1 + .../secrets/broker1_truststore_creds | 1 + .../secrets/client_keystore_creds | 1 + .../secrets/client_sslkey_creds | 1 + .../secrets/client_truststore_creds | 1 + .../secrets/create-certs.sh | 50 +++++ .../secrets/kafka.broker1.keystore | Bin 0 -> 4745 bytes .../secrets/kafka.broker1.truststore | Bin 0 -> 1250 bytes .../secrets/kafka.client.keystore | Bin 0 -> 4743 bytes .../secrets/kafka.client.truststore | Bin 0 -> 1250 bytes .../tempto-configuration.yaml | 17 ++ 43 files changed, 1243 insertions(+), 38 deletions(-) create mode 100644 plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaClientsModule.java create mode 100644 plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/SslKafkaAdminFactory.java create mode 100644 plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/SslKafkaConsumerFactory.java create mode 100644 plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/SslKafkaProducerFactory.java create mode 100644 plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/ForKafkaSsl.java create mode 100644 plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaEndpointIdentificationAlgorithm.java rename plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/{KafkaConsumerModule.java => security/KafkaKeystoreTruststoreType.java} (62%) create mode 100644 plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSecurityModule.java create mode 100644 plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSslConfig.java rename plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/{KafkaProducerModule.java => security/SecurityProtocol.java} (60%) rename plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/{KafkaAdminModule.java => security/SslSecurityModule.java} (78%) create mode 100644 plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaSslConfig.java create mode 100644 testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/KafkaSsl.java create mode 100644 testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/MultinodeKafkaSsl.java create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/kafka.properties create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/kafka_schema_registry.properties create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/.gitignore create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/broker1_keystore_creds create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/broker1_sslkey_creds create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/broker1_truststore_creds create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/client_keystore_creds create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/client_sslkey_creds create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/client_truststore_creds create mode 100755 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/create-certs.sh create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/kafka.broker1.keystore create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/kafka.broker1.truststore create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/kafka.client.keystore create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/kafka.client.truststore create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/tempto-configuration.yaml diff --git a/docs/src/main/sphinx/connector/kafka.rst b/docs/src/main/sphinx/connector/kafka.rst index b0232e4f41e9..f8ed1d674fff 100644 --- a/docs/src/main/sphinx/connector/kafka.rst +++ b/docs/src/main/sphinx/connector/kafka.rst @@ -67,6 +67,15 @@ Property Name Description ``kafka.hide-internal-columns`` Controls whether internal columns are part of the table schema or not ``kafka.messages-per-split`` Number of messages that are processed by each Trino split, defaults to 100000 ``kafka.timestamp-upper-bound-force-push-down-enabled`` Controls if upper bound timestamp push down is enabled for topics using ``CreateTime`` mode +``kafka.security-protocol`` Security protocol for connection to Kafka cluster, defaults to ``PLAINTEXT`` +``kafka.ssl.keystore.location`` Location of the keystore file +``kafka.ssl.keystore.password`` Password for the keystore file +``kafka.ssl.keystore.type`` File format of the keystore file, defaults to ``JKS`` +``kafka.ssl.truststore.location`` Location of the truststore file +``kafka.ssl.truststore.password`` Password for the truststore file +``kafka.ssl.truststore.type`` File format of the truststore file, defaults to ``JKS`` +``kafka.ssl.key.password`` Password for the private key in the keystore file +``kafka.ssl.endpoint-identification-algorithm`` Endpoint identification algorithm used by clients to validate server host name, defaults to ``https`` ========================================================== ============================================================================== In addition, you need to configure :ref:`table schema and schema registry usage @@ -125,6 +134,73 @@ show up in ``DESCRIBE `` or ``SELECT *``. This property is optional; the default is ``true``. +``kafka.security-protocol`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Protocol used to communicate with brokers. +Valid values are: PLAINTEXT, SSL. + +This property is optional; default is ``PLAINTEXT``. + +``kafka.ssl.keystore.location`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Location of the keystore file used for connection to Kafka cluster. + +This property is optional. + +``kafka.ssl.keystore.password`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Password for the keystore file used for connection to Kafka cluster. + +This property is optional, but required when ``kafka.ssl.keystore.location`` is given. + +``kafka.ssl.keystore.type`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +File format of the keystore file. +Valid values are: JKS, PKCS12. + +This property is optional; default is ``JKS``. + +``kafka.ssl.truststore.location`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Location of the truststore file used for connection to Kafka cluster. + +This property is optional. + +``kafka.ssl.truststore.password`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Password for the truststore file used for connection to Kafka cluster. + +This property is optional, but required when ``kafka.ssl.truststore.location`` is given. + +``kafka.ssl.truststore.type`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +File format of the truststore file. +Valid values are: JKS, PKCS12. + +This property is optional; default is ``JKS``. + +``kafka.ssl.key.password`` +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Password for the private key in the keystore file used for connection to Kafka cluster. + +This property is optional. This is required for clients only if two-way authentication is configured i.e. ``ssl.client.auth=required``. + +``kafka.ssl.endpoint-identification-algorithm`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The endpoint identification algorithm used by clients to validate server host name for connection to Kafka cluster. +Kafka uses ``https`` as default. Use ``disabled`` to disable server host name validation. + +This property is optional; default is ``https``. + Internal columns ---------------- diff --git a/plugin/trino-kafka/pom.xml b/plugin/trino-kafka/pom.xml index 2df458dcaa0c..98fa0644af25 100644 --- a/plugin/trino-kafka/pom.xml +++ b/plugin/trino-kafka/pom.xml @@ -84,6 +84,11 @@ kafka-schema-registry-client + + javax.annotation + javax.annotation-api + + javax.inject javax.inject diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaClientsModule.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaClientsModule.java new file mode 100644 index 000000000000..9b8c0c8411ef --- /dev/null +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaClientsModule.java @@ -0,0 +1,60 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kafka; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.plugin.kafka.security.ForKafkaSsl; +import io.trino.plugin.kafka.security.SecurityProtocol; + +import static io.airlift.configuration.ConditionalModule.installModuleIf; + +public class KafkaClientsModule + extends AbstractConfigurationAwareModule +{ + @Override + protected void setup(Binder binder) + { + installClientModule(SecurityProtocol.PLAINTEXT, KafkaClientsModule::configurePlainText); + installClientModule(SecurityProtocol.SSL, KafkaClientsModule::configureSsl); + } + + private void installClientModule(SecurityProtocol securityProtocol, Module module) + { + install(installModuleIf( + KafkaConfig.class, + config -> config.getSecurityProtocol().equals(securityProtocol), + module)); + } + + private static void configurePlainText(Binder binder) + { + binder.bind(KafkaConsumerFactory.class).to(PlainTextKafkaConsumerFactory.class).in(Scopes.SINGLETON); + binder.bind(KafkaProducerFactory.class).to(PlainTextKafkaProducerFactory.class).in(Scopes.SINGLETON); + binder.bind(KafkaAdminFactory.class).to(PlainTextKafkaAdminFactory.class).in(Scopes.SINGLETON); + } + + private static void configureSsl(Binder binder) + { + binder.bind(KafkaConsumerFactory.class).annotatedWith(ForKafkaSsl.class).to(PlainTextKafkaConsumerFactory.class).in(Scopes.SINGLETON); + binder.bind(KafkaProducerFactory.class).annotatedWith(ForKafkaSsl.class).to(PlainTextKafkaProducerFactory.class).in(Scopes.SINGLETON); + binder.bind(KafkaAdminFactory.class).annotatedWith(ForKafkaSsl.class).to(PlainTextKafkaAdminFactory.class).in(Scopes.SINGLETON); + + binder.bind(KafkaConsumerFactory.class).to(SslKafkaConsumerFactory.class).in(Scopes.SINGLETON); + binder.bind(KafkaProducerFactory.class).to(SslKafkaProducerFactory.class).in(Scopes.SINGLETON); + binder.bind(KafkaAdminFactory.class).to(SslKafkaAdminFactory.class).in(Scopes.SINGLETON); + } +} diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConfig.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConfig.java index 78e24db15428..b348403e5d76 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConfig.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConfig.java @@ -21,6 +21,7 @@ import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; import io.trino.plugin.kafka.schema.file.FileTableDescriptionSupplier; +import io.trino.plugin.kafka.security.SecurityProtocol; import io.trino.spi.HostAddress; import javax.validation.constraints.Min; @@ -31,6 +32,7 @@ import java.util.stream.StreamSupport; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static io.trino.plugin.kafka.security.SecurityProtocol.PLAINTEXT; @DefunctConfig("kafka.connect-timeout") public class KafkaConfig @@ -44,6 +46,7 @@ public class KafkaConfig private int messagesPerSplit = 100_000; private boolean timestampUpperBoundPushDownEnabled; private String tableDescriptionSupplier = FileTableDescriptionSupplier.NAME; + private SecurityProtocol securityProtocol = PLAINTEXT; @Size(min = 1) public Set getNodes() @@ -152,4 +155,17 @@ public KafkaConfig setTimestampUpperBoundPushDownEnabled(boolean timestampUpperB this.timestampUpperBoundPushDownEnabled = timestampUpperBoundPushDownEnabled; return this; } + + @Config("kafka.security-protocol") + @ConfigDescription("Security protocol used for Kafka connection") + public KafkaConfig setSecurityProtocol(SecurityProtocol securityProtocol) + { + this.securityProtocol = securityProtocol; + return this; + } + + public SecurityProtocol getSecurityProtocol() + { + return securityProtocol; + } } diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorFactory.java index ddecd4810555..85d747270032 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorFactory.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorFactory.java @@ -17,6 +17,7 @@ import com.google.inject.Module; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; +import io.trino.plugin.kafka.security.KafkaSecurityModule; import io.trino.spi.NodeManager; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; @@ -59,6 +60,7 @@ public Connector create(String catalogName, Map config, Connecto Bootstrap app = new Bootstrap( new JsonModule(), new KafkaConnectorModule(), + new KafkaSecurityModule(), extension, binder -> { binder.bind(ClassLoader.class).toInstance(KafkaConnectorFactory.class.getClassLoader()); diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaPlugin.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaPlugin.java index c7487c33c020..1eeaa143c770 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaPlugin.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaPlugin.java @@ -14,7 +14,10 @@ package io.trino.plugin.kafka; import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; import com.google.inject.Module; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.plugin.kafka.security.KafkaSecurityModule; import io.trino.spi.Plugin; import io.trino.spi.connector.ConnectorFactory; @@ -23,10 +26,14 @@ public class KafkaPlugin implements Plugin { - public static final Module DEFAULT_EXTENSION = binder -> { - binder.install(new KafkaConsumerModule()); - binder.install(new KafkaProducerModule()); - binder.install(new KafkaAdminModule()); + public static final Module DEFAULT_EXTENSION = new AbstractConfigurationAwareModule() + { + @Override + protected void setup(Binder binder) + { + install(new KafkaClientsModule()); + install(new KafkaSecurityModule()); + } }; private final Module extension; diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaProducerFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaProducerFactory.java index ad049a125845..ee0287dedffe 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaProducerFactory.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaProducerFactory.java @@ -11,7 +11,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package io.trino.plugin.kafka; import io.trino.spi.connector.ConnectorSession; diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaAdminFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaAdminFactory.java index d775e7130a48..b34f911d129b 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaAdminFactory.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaAdminFactory.java @@ -14,6 +14,7 @@ package io.trino.plugin.kafka; +import io.trino.plugin.kafka.security.SecurityProtocol; import io.trino.spi.HostAddress; import io.trino.spi.connector.ConnectorSession; @@ -24,12 +25,14 @@ import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; +import static org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG; public class PlainTextKafkaAdminFactory implements KafkaAdminFactory { private final Set nodes; + private final SecurityProtocol securityProtocol; @Inject public PlainTextKafkaAdminFactory(KafkaConfig kafkaConfig) @@ -37,6 +40,7 @@ public PlainTextKafkaAdminFactory(KafkaConfig kafkaConfig) requireNonNull(kafkaConfig, "kafkaConfig is null"); nodes = kafkaConfig.getNodes(); + securityProtocol = kafkaConfig.getSecurityProtocol(); } @Override @@ -46,6 +50,7 @@ public Properties configure(ConnectorSession session) properties.setProperty(BOOTSTRAP_SERVERS_CONFIG, nodes.stream() .map(HostAddress::toString) .collect(joining(","))); + properties.setProperty(SECURITY_PROTOCOL_CONFIG, securityProtocol.name()); return properties; } } diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaConsumerFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaConsumerFactory.java index b4b0e1786593..232e7725524d 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaConsumerFactory.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaConsumerFactory.java @@ -14,6 +14,7 @@ package io.trino.plugin.kafka; import io.airlift.units.DataSize; +import io.trino.plugin.kafka.security.SecurityProtocol; import io.trino.spi.HostAddress; import io.trino.spi.connector.ConnectorSession; import org.apache.kafka.common.serialization.ByteArrayDeserializer; @@ -25,6 +26,7 @@ import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; +import static org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; @@ -36,6 +38,7 @@ public class PlainTextKafkaConsumerFactory { private final Set nodes; private final DataSize kafkaBufferSize; + private final SecurityProtocol securityProtocol; @Inject public PlainTextKafkaConsumerFactory(KafkaConfig kafkaConfig) @@ -44,6 +47,7 @@ public PlainTextKafkaConsumerFactory(KafkaConfig kafkaConfig) nodes = kafkaConfig.getNodes(); kafkaBufferSize = kafkaConfig.getKafkaBufferSize(); + securityProtocol = kafkaConfig.getSecurityProtocol(); } @Override @@ -57,6 +61,7 @@ public Properties configure(ConnectorSession session) properties.setProperty(VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); properties.setProperty(RECEIVE_BUFFER_CONFIG, Long.toString(kafkaBufferSize.toBytes())); properties.setProperty(ENABLE_AUTO_COMMIT_CONFIG, Boolean.toString(false)); + properties.setProperty(SECURITY_PROTOCOL_CONFIG, securityProtocol.name()); return properties; } } diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaProducerFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaProducerFactory.java index cd45bdbdd69c..0412fb65f0be 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaProducerFactory.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaProducerFactory.java @@ -13,6 +13,7 @@ */ package io.trino.plugin.kafka; +import io.trino.plugin.kafka.security.SecurityProtocol; import io.trino.spi.HostAddress; import io.trino.spi.connector.ConnectorSession; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -24,6 +25,7 @@ import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; +import static org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.ACKS_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; @@ -34,6 +36,7 @@ public class PlainTextKafkaProducerFactory implements KafkaProducerFactory { private final Set nodes; + private final SecurityProtocol securityProtocol; @Inject public PlainTextKafkaProducerFactory(KafkaConfig kafkaConfig) @@ -41,6 +44,7 @@ public PlainTextKafkaProducerFactory(KafkaConfig kafkaConfig) requireNonNull(kafkaConfig, "kafkaConfig is null"); nodes = kafkaConfig.getNodes(); + securityProtocol = kafkaConfig.getSecurityProtocol(); } @Override @@ -54,6 +58,7 @@ public Properties configure(ConnectorSession session) properties.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); properties.setProperty(ACKS_CONFIG, "all"); properties.setProperty(LINGER_MS_CONFIG, Long.toString(5)); + properties.setProperty(SECURITY_PROTOCOL_CONFIG, securityProtocol.name()); return properties; } } diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/SslKafkaAdminFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/SslKafkaAdminFactory.java new file mode 100644 index 000000000000..a5e005d21d5f --- /dev/null +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/SslKafkaAdminFactory.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.kafka; + +import com.google.common.collect.ImmutableMap; +import io.trino.plugin.kafka.security.ForKafkaSsl; +import io.trino.plugin.kafka.security.KafkaSslConfig; +import io.trino.spi.connector.ConnectorSession; + +import javax.inject.Inject; + +import java.util.Properties; + +import static java.util.Objects.requireNonNull; + +public class SslKafkaAdminFactory + implements KafkaAdminFactory +{ + private final ImmutableMap map; + private final KafkaAdminFactory delegate; + + @Inject + public SslKafkaAdminFactory(@ForKafkaSsl KafkaAdminFactory delegate, KafkaSslConfig sslConfig) + { + this.delegate = requireNonNull(delegate, "delegate is null"); + requireNonNull(sslConfig, "sslConfig is null"); + + map = ImmutableMap.copyOf(sslConfig.getKafkaClientProperties()); + } + + @Override + public Properties configure(ConnectorSession session) + { + Properties properties = new Properties(); + properties.putAll(delegate.configure(session)); + properties.putAll(map); + return properties; + } +} diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/SslKafkaConsumerFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/SslKafkaConsumerFactory.java new file mode 100644 index 000000000000..a18058f6b61f --- /dev/null +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/SslKafkaConsumerFactory.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kafka; + +import com.google.common.collect.ImmutableMap; +import io.trino.plugin.kafka.security.ForKafkaSsl; +import io.trino.plugin.kafka.security.KafkaSslConfig; +import io.trino.spi.connector.ConnectorSession; + +import javax.inject.Inject; + +import java.util.Properties; + +import static java.util.Objects.requireNonNull; + +public class SslKafkaConsumerFactory + implements KafkaConsumerFactory +{ + private final ImmutableMap map; + private final KafkaConsumerFactory delegate; + + @Inject + public SslKafkaConsumerFactory(@ForKafkaSsl KafkaConsumerFactory delegate, KafkaSslConfig sslConfig) + { + this.delegate = requireNonNull(delegate, "delegate is null"); + requireNonNull(sslConfig, "sslConfig is null"); + + map = ImmutableMap.copyOf(sslConfig.getKafkaClientProperties()); + } + + @Override + public Properties configure(ConnectorSession session) + { + Properties properties = new Properties(); + properties.putAll(delegate.configure(session)); + properties.putAll(map); + return properties; + } +} diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/SslKafkaProducerFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/SslKafkaProducerFactory.java new file mode 100644 index 000000000000..0495e2f096f0 --- /dev/null +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/SslKafkaProducerFactory.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kafka; + +import com.google.common.collect.ImmutableMap; +import io.trino.plugin.kafka.security.ForKafkaSsl; +import io.trino.plugin.kafka.security.KafkaSslConfig; +import io.trino.spi.connector.ConnectorSession; + +import javax.inject.Inject; + +import java.util.Properties; + +import static java.util.Objects.requireNonNull; + +public class SslKafkaProducerFactory + implements KafkaProducerFactory +{ + private final ImmutableMap map; + private final KafkaProducerFactory delegate; + + @Inject + public SslKafkaProducerFactory(@ForKafkaSsl KafkaProducerFactory delegate, KafkaSslConfig sslConfig) + { + this.delegate = requireNonNull(delegate, "delegate is null"); + requireNonNull(sslConfig, "sslConfig is null"); + + map = ImmutableMap.copyOf(sslConfig.getKafkaClientProperties()); + } + + @Override + public Properties configure(ConnectorSession session) + { + Properties properties = new Properties(); + properties.putAll(delegate.configure(session)); + properties.putAll(map); + return properties; + } +} diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/ForKafkaSsl.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/ForKafkaSsl.java new file mode 100644 index 000000000000..4f4b95414e8b --- /dev/null +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/ForKafkaSsl.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kafka.security; + +import javax.inject.Qualifier; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({FIELD, PARAMETER, METHOD}) +@Qualifier +public @interface ForKafkaSsl +{ +} diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaEndpointIdentificationAlgorithm.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaEndpointIdentificationAlgorithm.java new file mode 100644 index 000000000000..74b2aab83a84 --- /dev/null +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaEndpointIdentificationAlgorithm.java @@ -0,0 +1,41 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kafka.security; + +import java.util.Optional; + +import static java.util.Locale.ENGLISH; + +public enum KafkaEndpointIdentificationAlgorithm +{ + HTTPS("https"), + DISABLED(""); + + private final String value; + + KafkaEndpointIdentificationAlgorithm(String value) + { + this.value = value; + } + + public static Optional fromString(String value) + { + return Optional.of(KafkaEndpointIdentificationAlgorithm.valueOf(value.toUpperCase(ENGLISH))); + } + + public String getValue() + { + return value; + } +} diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConsumerModule.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaKeystoreTruststoreType.java similarity index 62% rename from plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConsumerModule.java rename to plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaKeystoreTruststoreType.java index 0fc43c41215e..eecaea983357 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConsumerModule.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaKeystoreTruststoreType.java @@ -11,18 +11,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.kafka; +package io.trino.plugin.kafka.security; -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Scopes; +import java.util.Optional; -public class KafkaConsumerModule - implements Module +import static java.util.Locale.ENGLISH; + +public enum KafkaKeystoreTruststoreType { - @Override - public void configure(Binder binder) + JKS, + PKCS12; + + public static Optional fromString(String value) { - binder.bind(KafkaConsumerFactory.class).to(PlainTextKafkaConsumerFactory.class).in(Scopes.SINGLETON); + return Optional.of(KafkaKeystoreTruststoreType.valueOf(value.toUpperCase(ENGLISH))); } } diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSecurityModule.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSecurityModule.java new file mode 100644 index 000000000000..01707c412280 --- /dev/null +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSecurityModule.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kafka.security; + +import com.google.inject.Binder; +import com.google.inject.Module; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.plugin.kafka.KafkaConfig; + +import static io.airlift.configuration.ConditionalModule.installModuleIf; +import static io.airlift.configuration.ConfigurationModule.installModules; + +public class KafkaSecurityModule + extends AbstractConfigurationAwareModule +{ + @Override + protected void setup(Binder binder) + { + bindSecurityModule( + SecurityProtocol.SSL, + installModules(new SslSecurityModule())); + } + + private void bindSecurityModule(SecurityProtocol securityProtocol, Module module) + { + install(installModuleIf( + KafkaConfig.class, + config -> config.getSecurityProtocol().equals(securityProtocol), + module)); + } +} diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSslConfig.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSslConfig.java new file mode 100644 index 000000000000..af379ea624ce --- /dev/null +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSslConfig.java @@ -0,0 +1,187 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kafka.security; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.inject.ConfigurationException; +import com.google.inject.spi.Message; +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import io.airlift.configuration.ConfigSecuritySensitive; +import io.airlift.configuration.validation.FileExists; + +import javax.annotation.PostConstruct; + +import java.util.Map; +import java.util.Optional; + +import static io.trino.plugin.kafka.security.KafkaEndpointIdentificationAlgorithm.HTTPS; +import static io.trino.plugin.kafka.security.KafkaKeystoreTruststoreType.JKS; +import static org.apache.kafka.common.config.SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_KEYSTORE_TYPE_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_KEY_PASSWORD_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG; + +/** + * {@KafkaSslConfig} manages Kafka SSL authentication and encryption between clients and brokers. + */ +public class KafkaSslConfig +{ + private String keystoreLocation; + private String keystorePassword; + private KafkaKeystoreTruststoreType keystoreType = JKS; + private String truststoreLocation; + private String truststorePassword; + private KafkaKeystoreTruststoreType truststoreType = JKS; + private String keyPassword; + private KafkaEndpointIdentificationAlgorithm endpointIdentificationAlgorithm = HTTPS; + + public Optional<@FileExists String> getKeystoreLocation() + { + return Optional.ofNullable(keystoreLocation); + } + + @Config("kafka.ssl.keystore.location") + @ConfigDescription("The location of the key store file. This can be used for two-way authentication for client") + public KafkaSslConfig setKeystoreLocation(String keystoreLocation) + { + this.keystoreLocation = keystoreLocation; + return this; + } + + public Optional getKeystorePassword() + { + return Optional.ofNullable(keystorePassword); + } + + @Config("kafka.ssl.keystore.password") + @ConfigDescription("The store password for the key store file") + @ConfigSecuritySensitive + public KafkaSslConfig setKeystorePassword(String keystorePassword) + { + this.keystorePassword = keystorePassword; + return this; + } + + public Optional getKeystoreType() + { + return Optional.ofNullable(keystoreType); + } + + @Config("kafka.ssl.keystore.type") + @ConfigDescription("The file format of the key store file") + public KafkaSslConfig setKeystoreType(KafkaKeystoreTruststoreType keystoreType) + { + this.keystoreType = keystoreType; + return this; + } + + public Optional<@FileExists String> getTruststoreLocation() + { + return Optional.ofNullable(truststoreLocation); + } + + @Config("kafka.ssl.truststore.location") + @ConfigDescription("The location of the trust store file") + public KafkaSslConfig setTruststoreLocation(String truststoreLocation) + { + this.truststoreLocation = truststoreLocation; + return this; + } + + public Optional getTruststorePassword() + { + return Optional.ofNullable(truststorePassword); + } + + @Config("kafka.ssl.truststore.password") + @ConfigDescription("The password for the trust store file") + @ConfigSecuritySensitive + public KafkaSslConfig setTruststorePassword(String truststorePassword) + { + this.truststorePassword = truststorePassword; + return this; + } + + public Optional getTruststoreType() + { + return Optional.ofNullable(truststoreType); + } + + @Config("kafka.ssl.truststore.type") + @ConfigDescription("The file format of the trust store file") + public KafkaSslConfig setTruststoreType(KafkaKeystoreTruststoreType truststoreType) + { + this.truststoreType = truststoreType; + return this; + } + + public Optional getKeyPassword() + { + return Optional.ofNullable(keyPassword); + } + + @Config("kafka.ssl.key.password") + @ConfigDescription("The password of the private key in the key store file") + @ConfigSecuritySensitive + public KafkaSslConfig setKeyPassword(String keyPassword) + { + this.keyPassword = keyPassword; + return this; + } + + public Optional getEndpointIdentificationAlgorithm() + { + return Optional.ofNullable(endpointIdentificationAlgorithm); + } + + @Config("kafka.ssl.endpoint-identification-algorithm") + @ConfigDescription("The endpoint identification algorithm to validate server hostname using server certificate") + public KafkaSslConfig setEndpointIdentificationAlgorithm(KafkaEndpointIdentificationAlgorithm endpointIdentificationAlgorithm) + { + this.endpointIdentificationAlgorithm = endpointIdentificationAlgorithm; + return this; + } + + public Map getKafkaClientProperties() + { + ImmutableMap.Builder properties = ImmutableMap.builder(); + getKeystoreLocation().ifPresent(v -> properties.put(SSL_KEYSTORE_LOCATION_CONFIG, v)); + getKeystorePassword().ifPresent(v -> properties.put(SSL_KEYSTORE_PASSWORD_CONFIG, v)); + getKeystoreType().ifPresent(v -> properties.put(SSL_KEYSTORE_TYPE_CONFIG, v.name())); + getTruststoreLocation().ifPresent(v -> properties.put(SSL_TRUSTSTORE_LOCATION_CONFIG, v)); + getTruststorePassword().ifPresent(v -> properties.put(SSL_TRUSTSTORE_PASSWORD_CONFIG, v)); + getTruststoreType().ifPresent(v -> properties.put(SSL_TRUSTSTORE_TYPE_CONFIG, v.name())); + getKeyPassword().ifPresent(v -> properties.put(SSL_KEY_PASSWORD_CONFIG, v)); + getEndpointIdentificationAlgorithm().ifPresent(v -> properties.put(SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, v.getValue())); + + return properties.build(); + } + + @PostConstruct + public void validate() + { + if (getKeystoreLocation().isPresent() && getKeystorePassword().isEmpty()) { + throw new ConfigurationException(ImmutableList.of(new Message("kafka.ssl.keystore.password must set when kafka.ssl.keystore.location is given"))); + } + if (getTruststoreLocation().isPresent() && getTruststorePassword().isEmpty()) { + throw new ConfigurationException(ImmutableList.of(new Message("kafka.ssl.truststore.password must set when kafka.ssl.truststore.location is given"))); + } + } +} diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaProducerModule.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/SecurityProtocol.java similarity index 60% rename from plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaProducerModule.java rename to plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/SecurityProtocol.java index af1954cc3dfd..db4dad45a62c 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaProducerModule.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/SecurityProtocol.java @@ -11,18 +11,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.kafka; +package io.trino.plugin.kafka.security; -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Scopes; - -public class KafkaProducerModule - implements Module +public enum SecurityProtocol { - @Override - public void configure(Binder binder) - { - binder.bind(KafkaProducerFactory.class).to(PlainTextKafkaProducerFactory.class).in(Scopes.SINGLETON); - } + PLAINTEXT, + SSL } diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaAdminModule.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/SslSecurityModule.java similarity index 78% rename from plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaAdminModule.java rename to plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/SslSecurityModule.java index b4bc52bb21e5..3d070c132d8d 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaAdminModule.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/SslSecurityModule.java @@ -11,18 +11,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.kafka; +package io.trino.plugin.kafka.security; import com.google.inject.Binder; import com.google.inject.Module; -import com.google.inject.Scopes; -public class KafkaAdminModule +import static io.airlift.configuration.ConfigBinder.configBinder; + +public class SslSecurityModule implements Module { @Override public void configure(Binder binder) { - binder.bind(KafkaAdminFactory.class).to(PlainTextKafkaAdminFactory.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(KafkaSslConfig.class); } } diff --git a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaConfig.java b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaConfig.java index ffa1e0ec83ed..ea95d14570f1 100644 --- a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaConfig.java +++ b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaConfig.java @@ -15,6 +15,7 @@ import com.google.common.collect.ImmutableMap; import io.trino.plugin.kafka.schema.file.FileTableDescriptionSupplier; +import io.trino.plugin.kafka.security.SecurityProtocol; import org.testng.annotations.Test; import java.util.Map; @@ -35,7 +36,8 @@ public void testDefaults() .setTableDescriptionSupplier(FileTableDescriptionSupplier.NAME) .setHideInternalColumns(true) .setMessagesPerSplit(100_000) - .setTimestampUpperBoundPushDownEnabled(false)); + .setTimestampUpperBoundPushDownEnabled(false) + .setSecurityProtocol(SecurityProtocol.PLAINTEXT)); } @Test @@ -49,6 +51,7 @@ public void testExplicitPropertyMappings() .put("kafka.hide-internal-columns", "false") .put("kafka.messages-per-split", "1") .put("kafka.timestamp-upper-bound-force-push-down-enabled", "true") + .put("kafka.security-protocol", "SSL") .build(); KafkaConfig expected = new KafkaConfig() @@ -58,7 +61,8 @@ public void testExplicitPropertyMappings() .setKafkaBufferSize("1MB") .setHideInternalColumns(false) .setMessagesPerSplit(1) - .setTimestampUpperBoundPushDownEnabled(true); + .setTimestampUpperBoundPushDownEnabled(true) + .setSecurityProtocol(SecurityProtocol.SSL); assertFullMapping(properties, expected); } diff --git a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaPlugin.java b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaPlugin.java index bf453b946944..18dc2b8b24ec 100644 --- a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaPlugin.java +++ b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaPlugin.java @@ -19,8 +19,15 @@ import io.trino.testing.TestingConnectorContext; import org.testng.annotations.Test; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + import static com.google.common.collect.Iterables.getOnlyElement; import static io.airlift.testing.Assertions.assertInstanceOf; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.testng.Assert.assertNotNull; public class TestKafkaPlugin @@ -43,4 +50,108 @@ public void testSpinup() assertNotNull(connector); connector.shutdown(); } + + @Test + public void testSslSpinup() + throws IOException + { + KafkaPlugin plugin = new KafkaPlugin(); + + ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories()); + assertInstanceOf(factory, KafkaConnectorFactory.class); + + String secret = "confluent"; + Path keystorePath = Files.createTempFile("keystore", ".jks"); + Path truststorePath = Files.createTempFile("truststore", ".jks"); + + writeToFile(keystorePath, secret); + writeToFile(truststorePath, secret); + + Connector connector = factory.create( + "test-connector", + ImmutableMap.builder() + .put("kafka.table-names", "test") + .put("kafka.nodes", "localhost:9092") + .put("kafka.security-protocol", "SSL") + .put("kafka.ssl.keystore.type", "JKS") + .put("kafka.ssl.keystore.location", keystorePath.toString()) + .put("kafka.ssl.keystore.password", "keystore-password") + .put("kafka.ssl.key.password", "key-password") + .put("kafka.ssl.truststore.type", "JKS") + .put("kafka.ssl.truststore.location", truststorePath.toString()) + .put("kafka.ssl.truststore.password", "truststore-password") + .put("kafka.ssl.endpoint-identification-algorithm", "https") + .build(), + new TestingConnectorContext()); + assertNotNull(connector); + connector.shutdown(); + } + + @Test + public void testSslKeystoreMissingFileSpindown() + throws IOException + { + KafkaPlugin plugin = new KafkaPlugin(); + + ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories()); + assertInstanceOf(factory, KafkaConnectorFactory.class); + + Path truststorePath = Files.createTempFile("test", ".jks"); + + assertThatThrownBy(() -> factory.create( + "test-connector", + ImmutableMap.builder() + .put("kafka.table-names", "test") + .put("kafka.nodes", "localhost:9092") + .put("kafka.security-protocol", "SSL") + .put("kafka.ssl.keystore.type", "JKS") + .put("kafka.ssl.keystore.location", "/not/a/real/path") + .put("kafka.ssl.keystore.password", "keystore-password") + .put("kafka.ssl.key.password", "key-password") + .put("kafka.ssl.truststore.type", "JKS") + .put("kafka.ssl.truststore.location", truststorePath.toString()) + .put("kafka.ssl.truststore.password", "truststore-password") + .put("kafka.ssl.endpoint-identification-algorithm", "https") + .build(), + new TestingConnectorContext())) + .hasMessageContaining("Error: Invalid configuration property kafka.ssl.keystore.location: file does not exist: /not/a/real/path"); + } + + @Test + public void testSslTruststoreMissingFileSpindown() + throws IOException + { + KafkaPlugin plugin = new KafkaPlugin(); + + ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories()); + assertInstanceOf(factory, KafkaConnectorFactory.class); + + Path keystorePath = Files.createTempFile("test", ".jks"); + + assertThatThrownBy(() -> factory.create( + "test-connector", + ImmutableMap.builder() + .put("kafka.table-names", "test") + .put("kafka.nodes", "localhost:9092") + .put("kafka.security-protocol", "SSL") + .put("kafka.ssl.keystore.type", "JKS") + .put("kafka.ssl.keystore.location", keystorePath.toString()) + .put("kafka.ssl.keystore.password", "keystore-password") + .put("kafka.ssl.key.password", "key-password") + .put("kafka.ssl.truststore.type", "JKS") + .put("kafka.ssl.truststore.location", "/not/a/real/path") + .put("kafka.ssl.truststore.password", "truststore-password") + .put("kafka.ssl.endpoint-identification-algorithm", "https") + .build(), + new TestingConnectorContext())) + .hasMessageContaining("Error: Invalid configuration property kafka.ssl.truststore.location: file does not exist: /not/a/real/path"); + } + + private void writeToFile(Path filepath, String content) + throws IOException + { + try (FileWriter writer = new FileWriter(filepath.toFile(), StandardCharsets.UTF_8)) { + writer.write(content); + } + } } diff --git a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaSslConfig.java b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaSslConfig.java new file mode 100644 index 000000000000..d8cf8e912dda --- /dev/null +++ b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaSslConfig.java @@ -0,0 +1,176 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kafka; + +import com.google.common.collect.ImmutableMap; +import com.google.inject.ConfigurationException; +import io.trino.plugin.kafka.security.KafkaEndpointIdentificationAlgorithm; +import io.trino.plugin.kafka.security.KafkaSslConfig; +import org.testng.annotations.Test; + +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.trino.plugin.kafka.security.KafkaEndpointIdentificationAlgorithm.DISABLED; +import static io.trino.plugin.kafka.security.KafkaEndpointIdentificationAlgorithm.HTTPS; +import static io.trino.plugin.kafka.security.KafkaKeystoreTruststoreType.JKS; +import static io.trino.plugin.kafka.security.KafkaKeystoreTruststoreType.PKCS12; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.kafka.common.config.SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_KEYSTORE_TYPE_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_KEY_PASSWORD_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG; +import static org.apache.kafka.common.config.SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestKafkaSslConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(KafkaSslConfig.class) + .setKeystoreLocation(null) + .setKeystorePassword(null) + .setKeystoreType(JKS) + .setTruststoreLocation(null) + .setTruststorePassword(null) + .setTruststoreType(JKS) + .setKeyPassword(null) + .setEndpointIdentificationAlgorithm(HTTPS)); + } + + @Test + public void testExplicitPropertyMappings() + throws IOException + { + String secret = "confluent"; + Path keystorePath = Files.createTempFile("keystore", ".p12"); + Path truststorePath = Files.createTempFile("truststore", ".p12"); + + writeToFile(keystorePath, secret); + writeToFile(truststorePath, secret); + + Map properties = new ImmutableMap.Builder() + .put("kafka.ssl.keystore.location", keystorePath.toString()) + .put("kafka.ssl.keystore.password", "keystore-password") + .put("kafka.ssl.keystore.type", "PKCS12") + .put("kafka.ssl.truststore.location", truststorePath.toString()) + .put("kafka.ssl.truststore.password", "truststore-password") + .put("kafka.ssl.truststore.type", "PKCS12") + .put("kafka.ssl.key.password", "key-password") + .put("kafka.ssl.endpoint-identification-algorithm", "disabled") + .build(); + KafkaSslConfig expected = new KafkaSslConfig() + .setKeystoreLocation(keystorePath.toString()) + .setKeystorePassword("keystore-password") + .setKeystoreType(PKCS12) + .setTruststoreLocation(truststorePath.toString()) + .setTruststorePassword("truststore-password") + .setTruststoreType(PKCS12) + .setKeyPassword("key-password") + .setEndpointIdentificationAlgorithm(DISABLED); + + assertFullMapping(properties, expected); + } + + @Test + public void testAllConfigPropertiesAreContained() + { + KafkaSslConfig config = new KafkaSslConfig() + .setKeystoreLocation("/some/path/to/keystore") + .setKeystorePassword("superSavePasswordForKeystore") + .setKeystoreType(JKS) + .setTruststoreLocation("/some/path/to/truststore") + .setTruststorePassword("superSavePasswordForTruststore") + .setTruststoreType(JKS) + .setKeyPassword("aSslKeyPassword") + .setEndpointIdentificationAlgorithm(HTTPS); + + Map securityProperties = config.getKafkaClientProperties(); + // Since security related properties are all passed to the underlying kafka-clients library, + // the property names must match those expected by kafka-clients + assertThat(securityProperties) + .containsExactlyInAnyOrderEntriesOf(ImmutableMap.copyOf(Map.of( + SSL_KEYSTORE_LOCATION_CONFIG, "/some/path/to/keystore", + SSL_KEYSTORE_PASSWORD_CONFIG, "superSavePasswordForKeystore", + SSL_KEYSTORE_TYPE_CONFIG, JKS.name(), + SSL_TRUSTSTORE_LOCATION_CONFIG, "/some/path/to/truststore", + SSL_TRUSTSTORE_PASSWORD_CONFIG, "superSavePasswordForTruststore", + SSL_TRUSTSTORE_TYPE_CONFIG, JKS.name(), + SSL_KEY_PASSWORD_CONFIG, "aSslKeyPassword", + SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, HTTPS.getValue()))); + } + + @Test + public void testDisabledEndpointIdentificationAlgorithm() + { + KafkaSslConfig config = new KafkaSslConfig(); + if (KafkaEndpointIdentificationAlgorithm.fromString("disabled").isPresent()) { + config.setEndpointIdentificationAlgorithm(KafkaEndpointIdentificationAlgorithm.fromString("disabled").get()); + } + Map securityProperties = config.getKafkaClientProperties(); + assertThat(securityProperties).containsKey(SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG); + assertThat(securityProperties.get(SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG)).isEqualTo(""); + } + + @Test + public void testFailOnMissingKeystorePasswordWithKeystoreLocationSet() + throws Exception + { + String secret = "confluent"; + Path keystorePath = Files.createTempFile("keystore", ".p12"); + + writeToFile(keystorePath, secret); + + KafkaSslConfig config = new KafkaSslConfig(); + config.setKeystoreLocation(keystorePath.toString()); + assertThatThrownBy(config::validate) + .isInstanceOf(ConfigurationException.class) + .hasMessageContaining("kafka.ssl.keystore.password must set when kafka.ssl.keystore.location is given"); + } + + @Test + public void testFailOnMissingTruststorePasswordWithTruststoreLocationSet() + throws Exception + { + String secret = "confluent"; + Path truststorePath = Files.createTempFile("truststore", ".p12"); + + writeToFile(truststorePath, secret); + + KafkaSslConfig config = new KafkaSslConfig(); + config.setTruststoreLocation(truststorePath.toString()); + assertThatThrownBy(config::validate) + .isInstanceOf(ConfigurationException.class) + .hasMessageContaining("kafka.ssl.truststore.password must set when kafka.ssl.truststore.location is given"); + } + + private void writeToFile(Path filepath, String content) + throws IOException + { + try (FileWriter writer = new FileWriter(filepath.toFile(), UTF_8)) { + writer.write(content); + } + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java index 51d750c4523d..b8c42a1c1a00 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java @@ -23,6 +23,7 @@ import io.trino.tests.product.launcher.env.common.HadoopKerberosKms; import io.trino.tests.product.launcher.env.common.HydraIdentityProvider; import io.trino.tests.product.launcher.env.common.Kafka; +import io.trino.tests.product.launcher.env.common.KafkaSsl; import io.trino.tests.product.launcher.env.common.SeleniumChrome; import io.trino.tests.product.launcher.env.common.Standard; import io.trino.tests.product.launcher.env.common.StandardMultinode; @@ -64,6 +65,7 @@ public void configure(Binder binder) binder.bind(HadoopKerberosKms.class).in(SINGLETON); binder.bind(HydraIdentityProvider.class).in(SINGLETON); binder.bind(Kafka.class).in(SINGLETON); + binder.bind(KafkaSsl.class).in(SINGLETON); binder.bind(SeleniumChrome.class).in(SINGLETON); binder.bind(Standard.class).in(SINGLETON); binder.bind(StandardMultinode.class).in(SINGLETON); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Kafka.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Kafka.java index 55ff9b48888e..11b0ca722b3e 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Kafka.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Kafka.java @@ -30,6 +30,9 @@ public class Kafka { private static final String CONFLUENT_VERSION = "5.5.2"; private static final int SCHEMA_REGISTRY_PORT = 8081; + static final String KAFKA = "kafka"; + static final String SCHEMA_REGISTRY = "schema-registry"; + static final String ZOOKEEPER = "zookeeper"; private final PortBinder portBinder; @@ -43,14 +46,14 @@ public Kafka(PortBinder portBinder) public void extendEnvironment(Environment.Builder builder) { builder.addContainers(createZookeeper(), createKafka(), createSchemaRegistry()) - .containerDependsOn("kafka", "zookeeper") - .containerDependsOn("schema-registry", "kafka"); + .containerDependsOn(KAFKA, ZOOKEEPER) + .containerDependsOn(SCHEMA_REGISTRY, KAFKA); } @SuppressWarnings("resource") private DockerContainer createZookeeper() { - DockerContainer container = new DockerContainer("confluentinc/cp-zookeeper:" + CONFLUENT_VERSION, "zookeeper") + DockerContainer container = new DockerContainer("confluentinc/cp-zookeeper:" + CONFLUENT_VERSION, ZOOKEEPER) .withEnv("ZOOKEEPER_CLIENT_PORT", "2181") .withEnv("ZOOKEEPER_TICK_TIME", "2000") .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) @@ -64,7 +67,7 @@ private DockerContainer createZookeeper() @SuppressWarnings("resource") private DockerContainer createKafka() { - DockerContainer container = new DockerContainer("confluentinc/cp-kafka:" + CONFLUENT_VERSION, "kafka") + DockerContainer container = new DockerContainer("confluentinc/cp-kafka:" + CONFLUENT_VERSION, KAFKA) .withEnv("KAFKA_BROKER_ID", "1") .withEnv("KAFKA_ZOOKEEPER_CONNECT", "zookeeper:2181") .withEnv("KAFKA_ADVERTISED_LISTENERS", "PLAINTEXT://kafka:9092") @@ -81,7 +84,7 @@ private DockerContainer createKafka() @SuppressWarnings("resource") private DockerContainer createSchemaRegistry() { - DockerContainer container = new DockerContainer("confluentinc/cp-schema-registry:" + CONFLUENT_VERSION, "schema-registry") + DockerContainer container = new DockerContainer("confluentinc/cp-schema-registry:" + CONFLUENT_VERSION, SCHEMA_REGISTRY) .withEnv("SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS", "PLAINTEXT://kafka:9092") .withEnv("SCHEMA_REGISTRY_HOST_NAME", "0.0.0.0") .withEnv("SCHEMA_REGISTRY_LISTENERS", "http://0.0.0.0:" + SCHEMA_REGISTRY_PORT) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/KafkaSsl.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/KafkaSsl.java new file mode 100644 index 000000000000..f15f6bf9edf8 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/KafkaSsl.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.tests.product.launcher.env.common; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.env.Environment; +import org.testcontainers.containers.BindMode; + +import javax.inject.Inject; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public class KafkaSsl + implements EnvironmentExtender +{ + private final Kafka kafka; + + @Inject + public KafkaSsl(Kafka kafka) + { + this.kafka = requireNonNull(kafka, "kafka is null"); + } + + @Override + public void extendEnvironment(Environment.Builder builder) + { + builder.configureContainer(Kafka.KAFKA, container -> container.withEnv("KAFKA_ADVERTISED_LISTENERS", "SSL://kafka:9092") + .withEnv("KAFKA_SSL_KEYSTORE_FILENAME", "kafka.broker1.keystore") + .withEnv("KAFKA_SSL_KEYSTORE_CREDENTIALS", "broker1_keystore_creds") + .withEnv("KAFKA_SSL_KEYSTORE_TYPE", "JKS") + .withEnv("KAFKA_SSL_KEY_CREDENTIALS", "broker1_sslkey_creds") + .withEnv("KAFKA_SSL_TRUSTSTORE_FILENAME", "kafka.broker1.truststore") + .withEnv("KAFKA_SSL_TRUSTSTORE_CREDENTIALS", "broker1_truststore_creds") + .withEnv("KAFKA_SSL_TRUSTSTORE_TYPE", "JKS") + .withEnv("KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM", "https") + .withEnv("KAFKA_SSL_CLIENT_AUTH", "required") + .withEnv("KAFKA_SECURITY_INTER_BROKER_PROTOCOL", "SSL") + .withClasspathResourceMapping("docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets", "/etc/kafka/secrets", BindMode.READ_ONLY)); + builder.configureContainer(Kafka.SCHEMA_REGISTRY, container -> container.withEnv("SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS", "SSL://kafka:9092") + .withEnv("SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL", "SSL") + .withEnv("SCHEMA_REGISTRY_KAFKASTORE_SSL_KEYSTORE_LOCATION", "/var/private/ssl/kafka.client.keystore") + .withEnv("SCHEMA_REGISTRY_KAFKASTORE_SSL_KEYSTORE_PASSWORD", "confluent") + .withEnv("SCHEMA_REGISTRY_KAFKASTORE_SSL_KEYSTORE_TYPE", "JKS") + .withEnv("SCHEMA_REGISTRY_KAFKASTORE_SSL_TRUSTSTORE_LOCATION", "/var/private/ssl/kafka.client.truststore") + .withEnv("SCHEMA_REGISTRY_KAFKASTORE_SSL_TRUSTSTORE_PASSWORD", "confluent") + .withEnv("SCHEMA_REGISTRY_KAFKASTORE_SSL_TRUSTSTORE_TYPE", "JKS") + .withEnv("SCHEMA_REGISTRY_KAFKASTORE_SSL_KEY_PASSWORD", "confluent") + .withEnv("SCHEMA_REGISTRY_KAFKASTORE_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM", "https") + .withEnv("SCHEMA_REGISTRY_KAFKASTORE_SSL_CLIENT_AUTH", "requested") + .withEnv("SCHEMA_REGISTRY_SSL_KEYSTORE_LOCATION", "/var/private/ssl/kafka.client.keystore") + .withEnv("SCHEMA_REGISTRY_SSL_KEYSTORE_PASSWORD", "confluent") + .withEnv("SCHEMA_REGISTRY_SSL_KEYSTORE_TYPE", "JKS") + .withEnv("SCHEMA_REGISTRY_SSL_KEY_PASSWORD", "confluent") + .withEnv("SCHEMA_REGISTRY_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM", "https") + .withClasspathResourceMapping("docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets", "/var/private/ssl", BindMode.READ_ONLY)); + } + + @Override + public List getDependencies() + { + return ImmutableList.of(kafka); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/MultinodeKafkaSsl.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/MultinodeKafkaSsl.java new file mode 100644 index 000000000000..f681a73c90cf --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/MultinodeKafkaSsl.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.tests.product.launcher.env.environment; + +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.docker.DockerFiles.ResourceProvider; +import io.trino.tests.product.launcher.env.DockerContainer; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.KafkaSsl; +import io.trino.tests.product.launcher.env.common.StandardMultinode; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; + +import javax.inject.Inject; + +import static io.trino.tests.product.launcher.env.EnvironmentContainers.COORDINATOR; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.WORKER; +import static io.trino.tests.product.launcher.env.common.Standard.CONTAINER_PRESTO_ETC; +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public final class MultinodeKafkaSsl + extends EnvironmentProvider +{ + private final ResourceProvider configDir; + + @Inject + public MultinodeKafkaSsl(KafkaSsl kafka, StandardMultinode standardMultinode, DockerFiles dockerFiles) + { + super(standardMultinode, kafka); + requireNonNull(dockerFiles, "dockerFiles is null"); + configDir = dockerFiles.getDockerFilesHostDirectory("conf/environment/multinode-kafka-ssl/"); + } + + @Override + public void extendEnvironment(Environment.Builder builder) + { + builder.configureContainer(COORDINATOR, this::addCatalogs); + builder.configureContainer(WORKER, this::addCatalogs); + + builder.configureContainer(TESTS, container -> container + .withCopyFileToContainer( + forHostPath(configDir.getPath("tempto-configuration.yaml")), + "/docker/presto-product-tests/conf/tempto/tempto-configuration-profile-config-file.yaml") + .withCopyFileToContainer( + forHostPath(configDir.getPath("secrets")), + "/docker/presto-product-tests/conf/tempto/secrets")); + } + + private void addCatalogs(DockerContainer container) + { + container + .withCopyFileToContainer( + forHostPath(configDir.getPath("kafka_schema_registry.properties")), + CONTAINER_PRESTO_ETC + "/catalog/kafka_schema_registry.properties") + .withCopyFileToContainer( + forHostPath(configDir.getPath("kafka.properties")), + CONTAINER_PRESTO_ETC + "/catalog/kafka.properties") + .withCopyFileToContainer( + forHostPath(configDir.getPath("secrets")), + CONTAINER_PRESTO_ETC + "/catalog/secrets"); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite6NonGeneric.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite6NonGeneric.java index 2d0567f36bbd..91a2028b2fd4 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite6NonGeneric.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite6NonGeneric.java @@ -17,6 +17,7 @@ import io.trino.tests.product.launcher.env.EnvironmentConfig; import io.trino.tests.product.launcher.env.EnvironmentDefaults; import io.trino.tests.product.launcher.env.environment.MultinodeKafka; +import io.trino.tests.product.launcher.env.environment.MultinodeKafkaSsl; import io.trino.tests.product.launcher.env.environment.SinglenodeCassandra; import io.trino.tests.product.launcher.env.environment.SinglenodeKerberosKmsHdfsImpersonation; import io.trino.tests.product.launcher.env.environment.SinglenodeKerberosKmsHdfsNoImpersonation; @@ -48,6 +49,7 @@ public List getTestRuns(EnvironmentConfig config) testOnEnvironment(SinglenodeKerberosKmsHdfsNoImpersonation.class).withGroups("storage_formats").build(), testOnEnvironment(SinglenodeKerberosKmsHdfsImpersonation.class).withGroups("storage_formats").build(), testOnEnvironment(SinglenodeCassandra.class).withGroups("cassandra").build(), - testOnEnvironment(MultinodeKafka.class).withGroups("kafka").build()); + testOnEnvironment(MultinodeKafka.class).withGroups("kafka").build(), + testOnEnvironment(MultinodeKafkaSsl.class).withGroups("kafka").build()); } } diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/kafka.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/kafka.properties new file mode 100644 index 000000000000..dd501fe1c792 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/kafka.properties @@ -0,0 +1,28 @@ +connector.name=kafka +kafka.table-names=product_tests.read_simple_key_and_value,\ + product_tests.read_all_datatypes_raw,\ + product_tests.read_all_datatypes_csv,\ + product_tests.read_all_datatypes_json,\ + product_tests.read_all_datatypes_avro,\ + product_tests.read_all_null_avro,\ + product_tests.read_structural_datatype_avro,\ + product_tests.write_simple_key_and_value,\ + product_tests.write_all_datatypes_raw,\ + product_tests.write_all_datatypes_csv,\ + product_tests.write_all_datatypes_json,\ + product_tests.write_all_datatypes_avro,\ + product_tests.write_structural_datatype_avro,\ + product_tests.pushdown_partition,\ + product_tests.pushdown_offset,\ + product_tests.pushdown_create_time +kafka.nodes=kafka:9092 +kafka.table-description-dir=/docker/presto-product-tests/conf/presto/etc/catalog/kafka +kafka.security-protocol=SSL +kafka.ssl.keystore.type=JKS +kafka.ssl.keystore.location=/docker/presto-product-tests/conf/presto/etc/catalog/secrets/kafka.client.keystore +kafka.ssl.keystore.password=confluent +kafka.ssl.truststore.type=JKS +kafka.ssl.truststore.location=/docker/presto-product-tests/conf/presto/etc/catalog/secrets/kafka.client.truststore +kafka.ssl.truststore.password=confluent +kafka.ssl.key.password=confluent +kafka.ssl.endpoint-identification-algorithm=https diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/kafka_schema_registry.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/kafka_schema_registry.properties new file mode 100644 index 000000000000..5fddd5c41db5 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/kafka_schema_registry.properties @@ -0,0 +1,14 @@ +connector.name=kafka +kafka.nodes=kafka:9092 +kafka.table-description-supplier=confluent +kafka.confluent-schema-registry-url=http://schema-registry:8081 +kafka.default-schema=product_tests +kafka.security-protocol=SSL +kafka.ssl.keystore.type=JKS +kafka.ssl.keystore.location=/docker/presto-product-tests/conf/presto/etc/catalog/secrets/kafka.client.keystore +kafka.ssl.keystore.password=confluent +kafka.ssl.truststore.type=JKS +kafka.ssl.truststore.location=/docker/presto-product-tests/conf/presto/etc/catalog/secrets/kafka.client.truststore +kafka.ssl.truststore.password=confluent +kafka.ssl.key.password=confluent +kafka.ssl.endpoint-identification-algorithm=https diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/.gitignore b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/.gitignore new file mode 100644 index 000000000000..cf132b981a59 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/.gitignore @@ -0,0 +1,12 @@ +# automate certs generation in a future PR instead of committing them in git +# docker run --rm -v "$PWD/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets":/secrets -w /secrets confluentinc/cp-kafka ./create-certs.sh +# *_creds +# *.jks +*.crt +*.key +*.pem +*.srl +*.req +*.csr +*.old +*.attr diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/broker1_keystore_creds b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/broker1_keystore_creds new file mode 100644 index 000000000000..2321227364ef --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/broker1_keystore_creds @@ -0,0 +1 @@ +confluent diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/broker1_sslkey_creds b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/broker1_sslkey_creds new file mode 100644 index 000000000000..2321227364ef --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/broker1_sslkey_creds @@ -0,0 +1 @@ +confluent diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/broker1_truststore_creds b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/broker1_truststore_creds new file mode 100644 index 000000000000..2321227364ef --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/broker1_truststore_creds @@ -0,0 +1 @@ +confluent diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/client_keystore_creds b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/client_keystore_creds new file mode 100644 index 000000000000..2321227364ef --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/client_keystore_creds @@ -0,0 +1 @@ +confluent diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/client_sslkey_creds b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/client_sslkey_creds new file mode 100644 index 000000000000..2321227364ef --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/client_sslkey_creds @@ -0,0 +1 @@ +confluent diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/client_truststore_creds b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/client_truststore_creds new file mode 100644 index 000000000000..2321227364ef --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/client_truststore_creds @@ -0,0 +1 @@ +confluent diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/create-certs.sh b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/create-certs.sh new file mode 100755 index 000000000000..2cc25281fec9 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/create-certs.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -xeuo pipefail + +# Generate CA key +openssl req \ + -new \ + -x509 \ + -keyout ca-1.key \ + -out ca-1.crt \ + -days 9999 \ + -subj '/CN=kafka/OU=TEST/O=TRINO/L=Montreal/S=Qc/C=CA' \ + -addext "subjectAltName = DNS:kafka" \ + -passin pass:confluent \ + -passout pass:confluent + +# Verify cert +# openssl x509 -in ca-1.crt -text -noout + +for i in broker1 client; do + echo $i + # Create keystores + keytool -genkey -noprompt \ + -alias $i \ + -dname "CN=kafka, OU=TEST, O=TRINO, L=Montreal, S=Qc, C=CA" \ + -keystore kafka.$i.keystore \ + -keyalg RSA \ + -storepass confluent \ + -keypass confluent \ + -ext SAN=dns:kafka + + # Verify keystore + # keytool -list -v -keystore kafka.broker1.keystore -storepass confluent + + # Create CSR, sign the key and import back into keystore + keytool -keystore kafka.$i.keystore -alias $i -certreq -file $i.csr -storepass confluent -keypass confluent -ext SAN=dns:kafka -noprompt + + openssl x509 -req -CA ca-1.crt -CAkey ca-1.key -in $i.csr -out $i-ca1-signed.crt -days 9999 -CAcreateserial -passin pass:confluent + + keytool -keystore kafka.$i.keystore -alias CARoot -import -file ca-1.crt -storepass confluent -keypass confluent -ext SAN=dns:kafka -noprompt + + keytool -keystore kafka.$i.keystore -alias $i -import -file $i-ca1-signed.crt -storepass confluent -keypass confluent -ext SAN=dns:kafka -noprompt + + # Create truststore and import the CA cert. + keytool -keystore kafka.$i.truststore -alias CARoot -import -file ca-1.crt -storepass confluent -keypass confluent -ext SAN=dns:kafka -noprompt + + echo "confluent" >${i}_sslkey_creds + echo "confluent" >${i}_keystore_creds + echo "confluent" >${i}_truststore_creds +done diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/kafka.broker1.keystore b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/kafka.broker1.keystore new file mode 100644 index 0000000000000000000000000000000000000000..9c687ae4a69c4126be23dee806155be4e42bc784 GIT binary patch literal 4745 zcmY+GWmFW7w#8v!=$sk4yI~0FkS>9tksLrlQjn5VhLWLML~5jl?hue}q&uZcKtjns zT;E&o-Mj9Gv-VnhpL4#Q-#SnPC<1_q1x0}5aqxK~H6pKwF>x{T5FjCJ1c>ji%>_jO zUH&TqT4E!BW`Av?zXih~{J$w80!%<20{ae%z`la=;1K-3{dYMXHY9U(0niul+)7-~ zoo}pVW4qp&pbmhA(F4OqU=J3|l&uHStm`Q%6;*5`R}UxEYjDY#2RF^X|Kr$S<`n0< ztb3l-CJFXVbay6qTuoy*%6r?%4&G4_D~>Bgpfhf;#5Sl&OBx(>vjvh=(WlP`&^P;Ef_S zI#k7If6L*0IShQY4D%M3+{9)skoG$$gX0)YCVRMl!WpvOZv*!};{ZHV)F`8MTsKeh zWM>h{mpfjAzkZ1gUD@_m)7lZdjdI`yk-2e>@X8;@ym}ldk@x27XzP_PJ<_r zi$}sJ6$~H|^E6OdxM+oTcKMFt=a!V3&byWQbS@W9P7uS4Y*MNFeB2im$sQL|@|=#& z7rFGc2LHVGQPim3vr({!bsR#38r1FlB$~N=`bFzJyQ4rLcDg3Z0>2Wuau%cN&c0_p zF#TlrT-U^)1Q+s5$EcI@8IsUf#|~cH>no=DS;hUteCDDps%*>!{K0HbNaWE3EI~u1 zNwe|GP5F%?uoZ49+{EH9+0WStDmVN}$m~OPd{>Tc0%HJmy2UH*dbQ6U@kMxhP=4OH z20y#*Z#Zo)Lzv&kiwd>xMr>U2kfx0u4i)-xOldUpG;t1zf4taz!DS?GNA_6IUjxKe zv*dVc=OV$%tsJmu*lMaD1h;0+9fz!@@!?Z=rLDYZ58h8)KSedmm6bjw(R1F{G2ixX z1f`Tn|2CAp%{tFSdFhmkS{}7nczeCJsnZWk6YyQ_Ki%!=e*wBD_S{(Vn+j;%T-TRo zy;WCuRj%_YCLdRz$lQ5_!v8HTigA~|M&pBQWuX@-_9%-aDpc0c65~*N8Qvb#r<6{* zCCB)g1D3ob-C`^NjvjIN>*3NX~#K3%7!@);4X@7i^0TkiF z;Ox)ryR>$$!Um{Qg!MCQ!1GgOhMz09@k|Zx8cW6XCU?n?-+4PHd=vY1p%F?9rzP0I zBUiILV@E*hy>FlK6WTwRzAG!yrLhNDZWUh~6jWC(&ARuTU@E1_XxbHdhNiw+5R=Mz zekazkA&_u;BQRfrr}`*Ld&-m86_!juD)C#xM{7sb@$>SVq3`9nKiaMBg}0=Hm7om& zMkPK(n2eDG!w%yO#%l~0jF*2^7|QaWm_itW4Ki|ad&w*;E-Wr3DJc$pA|fFLMG)2h zy9pO34?$G;*UATAV*Hg)|4A_ai?KNW!&qbg1*sBpJJ@p$ce6LCmRiZ*Pf!1svF{N? z@F77)d-aoxPI>+q7I=pJb*g9qM`Qtc{;$FjTS4Tt0}&vz-_WE4rbfKlq|IkzV49T zHfyy?!^KL7cl!hyVnO5HLTbwFO?9F-q)fO3aU(ZUXN?2`dmFteaw3 z0zJ(4i? z<16l*q3aygiSNI*j1yDphKO%f6vg%ZXDx2Le8iu%&Mr@+L>uJcxDBhlw;RtHOAK?- zU!si)^ypI;c|)`8ibMGiwJzq!^hc@^v>*y78|-KFtG8qlr_P*GY1 z*ABPadB+VAoLfza(|(aBzp{j6dDG}YO8#MVrkG)WrncTP(@aGYQoYUnO^!!;xhE!< zrW0Ph@3HjA=BE{$^l-7JAjV;6(yUjY?~VVJ4yiZdNxK@lnY~!AeHY$K%`hJ{c`eKL zyFf*UZp9{-ZEIXH6;&YQ*(ZRWF?TK{6vT-ul`p_5=Q_QMo0Mn<)U=uE+hTE?Vm1!@ z+>vL+#)VHCEF#STiAUBZH4@)Y>V9=7J$^OWR`W&9A&C@c%Y~Kxh0KuLJ zi*F6NG~?wFcw=Y0TX}CcmW5| znXYXsnRSsL$BYsIikue$#WlQ>J?KlefX_40*N5b-<{<~N>Ax3F*jVLa%ICOUzTFL) z+0*-W8(dcpbAEN}CO%mMWM^-p&AB#lf5PcL%1s{>hCDCdOxK+>UJFza5u z)4JJ|ysq0rEl_UreC<(ujL6Kn^@93D3AIl4pbI8@(Bf8HbiRnk+e`6)#c2tQx5?D^ zQ!p+yqGZ$D%>Ie#;m(?yLo2pqSEZE7uMiK0I>XK-wFtNceZfbeZyzeqKRg?JPPA11 zXiQ&7e6BD-X7e|9o$FQYM??XLa%dLrD`mn&&sr62*!o8?*=Qp*vp=414!4(-laZ`) z+TDA|0>Q=P8&D?8)7`K@KS@w2E#wIYKAvmxa58M1?MP$KJ&pI^lIhr%0nss_wN?y?V6+W{23b9M zyT#E%KaRbm<$bL>m*~ZaXUdvMOx<~IE=35nOcQ(MC`m=Q&l_E#6*QCEoQ`Y|BimmIJ>Gqb4mcc2h_OYs zRqfUMCc_gj&$|R|T{mpCTvP@xD4?3bMqD-tLi&5gSUgex^iV!T_#XstCY<;7jznTB zt>>o^NVq9|t#fR`VW!-!fcf)OcQO^G{Xs*w=BhzgM(3H?^5rp&PFTNM_)hYK z3|=2%Y>_%pzggw2JWr>8dA-J@eko-UbMLvIlK&d`6w7*>X4}E67t&*nY??SDU9j#6yvB3w1<@Vm#{h@5(C(&+RFddOqpG5-fL-ysiZB_toMP_c$83^_hHT&qMbWzQe+CQ3tQs{bk&8PniY( zNnu!*-(5IZD*($^t8!59d)jWhH3(?a;r2w-)7Oa8 zaJdkSv1!A|4)O8Kq52w@G`tM57K>`Mm3zl9p?QA~n+wE=$-Hy9|B6ptZyZND>S*sI zKkDb0(5LB;9dpL8r!%g3u3A9>FMVX*ue_NoT3K*tsL&KU)D(p!O8c#d$5w>OT4Y}E zO;Js!r@#p#aS>@;f6O?ARs}WN8v5TKCv^s^PY{j`oM000ix&MbvdAkgY=LMCjmZt* zlv_#T_vUx3JZXXniZq2+@rR4HS2;@21mXSia8W{#n<8EZZ=3J)nY*tgCtYC8WPsnC z&Ax~OQE&aJA+eRKx%FaQWLoQdtKclH!cs3H4Y5&wYdgSUkLKoD!WV^m*r0y z-O?R`XXhB>lqGxO&z%6maNWd3T^`G;KX4tk!m<1b`XTy}Lg4C-cwR3+mEaG2*mZ#I zF0f!>+yKN&ly~kKU{zBEY}q z+wEZtLoJ_u;1`#~+I+8Uq1d&-Fe7wGfb}Iw`5H}EB#+~*!(hgkP+TpETeAd=(V~lY zE##O+6^k+&KTw7$_zfK5(BasiRgi6-Id=LzV&O3bar7Wa2r=pXd)x_d`nZ7vV?zQc zDRetMnL&vEyN;m$-cNNwpQ=N_V4Cih5e}LlNV2sLiZkga8r^zZ$5w1OHZiP46U30r zV1$$RefpOD#BY+$n3h!OFicee%XMuH9))lHf?8>eE`1Jncp3e%&RFkI|C*eA4;+F(<@sW=hbS>L{ znm?z)o3xwPn(l!t@8g?L5k7mu)dDtGDLduU;tywuF8qcdo+^IQToBigrJC$)UnGc~ z%bwqAS)&YK1?FeeZIDJyB{T;<1L0~*t|>FwpO{HSG=^&X>4l2ig~tR2aquzTnM#D{ zbj#^YCCG13m#qu6rJ;B}@G+rEV@#aB=2GB1g}~t5S@Aw_g!GMvtALUTz^LD68mjQXBGbQlgq|=pu(A(^f%$s$6uO=|ecTcql_AM1Vopn0vhpF$Pa^UW4ppX2?%fV-tSIxF z1{~lA-52OB<*}|@>go`ifre=zDui>fN}K49Mj0iMG2S++mB{*IIrgR7Zk_X%GD*Yy z9$Oz#%eyYf28+6JTq=!778@hSe6_23T`;Tm<`vbTIO_L_g${6BtWvq&CD-HbgIT!Y z6^p=!*_=4GSzAkDN}7-^uf8n&o?d1X5AFF&0x_3BVO%f02T%tc|46~>U)w}E({?xC zsd+y!AbS>^sb@2C2m2{Yx<;0WZHhpwTy>GYJ*L+%swHld zyRg;Q%WG6x{REDrOmn&<7Q<13B`R?GLjtKPI;LY7<=!rL#EfFMQR~%gM3}juZ?|>P z`wUDX32f(AJQ$(Vw3=-J;AM%w5ylHdCMoQJ?#s2x=!k>|*a!`YQ`pyAle22RK|PB- zXbT@&KN_=E3jzJ{h>cmIMtCod7D5$rD3!Ql(doiK4x=Dq&VG?Fr%I79hv4E-*T14i zVU?oY$t!*v4W|?mkW&;Q_8K{SQdPnVKgfgmodn4u;yKhhE?y4~UEfcq-QZ68vu69) z+1T`m{~swkB=wUsu-YlX{;h26y3vj`dn}A~ZBq5BvQR7^h9SKRO<96MO!{u)AP?XC z^+U88{5gmV|BK)y7y`z{WN=89sp)WvT7w~z?M}~#)^<#qj5sHV zqHKDxWBFnXv_iHj)haC?x=b#%-7KJ0&((E6K=44j-0MAu@5LsR{r;8w-ky4j>ow(- zNow)!=tA|O0d;5Nf}josz1-jZn`=FSEzzlS|E+y*yiPJfB3)DpO*2)ZVLOQgWsi&( z=T9P4x4-MNZh3{kZ0(Q#DjD1I<*=1JVVESdV@l{6Ux$GxKnfB!5r;7bjUP<5r>wqB z&{uYu9;f^KoDUy~arMotHQ`J_Q%DZtBe(N_4 zRr6gT=aswP+IC7}m&8);TB8~R?GTT3BWwy+da$+e>m+KJlSh9yf8j6 zAutIB1uG5%0vZJX1QgOXgkyLhGHRots;3OA9pinYi6#UTETCM6IicgFWLSu@04m@A Ma^US_0s{etpc;ftGXMYp literal 0 HcmV?d00001 diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/kafka.client.keystore b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/secrets/kafka.client.keystore new file mode 100644 index 0000000000000000000000000000000000000000..ded1b8471d7abe15fd5adaaff0730d7e03f5edb1 GIT binary patch literal 4743 zcmY+FWmFW5wuTuP8W|XnfuXxox*Ple2c$zmdMJl3C5DEfy9cF9x|BvrLK;y(X{2+I zJl?zRJ!hRCd+oKJckli4*+NKg7&ZV$2nm+OBjkzHjKn+w-~-S|Fh39p=KhP>g^;-L z|B7(UfJj`UzxeszdWJ{*e_JF(0Bkf8cq4=aVuZNyi2fh{Jx&XxbQKTMSH3&dZ-^wc z*6b?wgqRj>VB=u*>jII$;nC*;ds9v-24xSBHjLrzO$DeNpQIBD2zq#A&m+9F!SEt{ zs#s7qKy8Xjxh$kprcH8~#0-@sJ7k`>#=@CLtRklXBKK9d3YHsOC}2JyS82+K7$nDd zarg#2TF6FWQjQW)QXe<4JL*>#=C5#zGy760kriTFfRoiLgh=gNtL=OOmg+=H(O zeaC;60f0<%JFRpg52c2EQ3|(AMjViYx=A;`TbZ8E`rLpDD7{zrmI#96c{>d z1nEb7UQKCiIa_hKMzXRf>U@7nkrT=luN3Dko4nZdzQ6;xCrNY|9%4w$Zp7K*LKVE! zYob?E0zR3c$=!e4BR@cMyptN{dj9dXO0E_;z|nb~7VGfa6ZgR|O`puGVHe8H?PrV! z3?d^m+-{L-+|6J`oAK|z9ZeQ_Pa<{yIPEqiJbqK1HP(JB^MxG`RoX{;rv;}C(? zGlZ&1`3T1w9lBf&1U2AltF7Al`WpG|}rSs>U z{QQe*h6l0vkC9MEN%5!`9=&TnXqIAXNCM9))2w)Rl}sPc(R=NdVi~a}xJvU6r?O^Y z-C2s_rtj^YA#Ys1QG_h(AIXI9iq^aSqmGEe`iB#3zXjtSkdTB7y>_dLs_;mo+ebc% z`O0L}Sk$6=cYrelMOgjwxTcheK``W8WlBW6^bY%3MdCM!#t;h4I=|v;D{O0{u46md zHZOgj3ST`R;%Hl|9TD6!eo2;n+$@Kp9?DOKR@UBI)Zu(`r6vmG*as1e1nA$4hfQQ# z5nn1gdoAP>7#bfbaO_xK(Vqk{??VKUdXj_=3)M-7Bzf1rGAZtCr&u|}p8DHrH2k13CTF346CRzeVp?V6C6XAU3ldTZYSOkh%$|{PP1As6U`mRYkl0N zU@J^>c%qAvRF$O4gfe=S6B0bX$lcy#qoU@EjVMQU@f2)#^ zQkay11j`1?1q+U4hvkOlCB*!n@Ubu@5NrTF@aDLC^Koa$1SKSmLC!0o98P5<` z6&uCPftIU7IjRPS=Z^4o-{qF%PKP%>%2pD@tUc&3Aqi7E4(@O9UdO7MGDEuRUrX!9 zdu3-ms3!`{2#X8X4aQw-9bM78`FVwE-~-NS#4T=2J$d;x&6GxkjO3}x79Ewp2|w;J zF<{wPzh&6MhuN^C*2}XqLtV9Qck%{xPLdIo>D^Zk+ak8E;Ecb(pCu37NYxPZezJ`x z=$pQvC|fnc-OBb+(8_tBFV5KW%>p%qFbaCh3xCNG2L6Wvu2zkf1cjeXKT>kFi?t=s z)bSP`#)bwXZoU$bX@pbX*Gcu;GZw=XGfu}dlDixS?)bXMz#oJ1A@NN(&^V1xlxJaD z@w4o1Zg*1wUa436eQZobn@fBydW_fh8ltbhQCpO8@*3QHm6G`r$f8dam?#P|BiKPu zkH`Z~_B~T=P#32T%#D5W>S|)};$yXT)DpcrP$8mj1lTG?{4BATCG@lI3Rc-OY^h3f zlm;y!pc3cvm=Vr&@r%}Qb6V}YqPIV&KT!wfiRpM7WnLDl2agoVnqPl(axf4XKYOb= z0-=jmPSYo;Vc}8gFYCMLUY8g6G+aN$kE%9FB-x_2#T(ej(rvoHH%~uqQe7Q>d}Ko4 zazFDuso?SnoYWb=K<-z{27Xs0^Qs9)pi_OQ{EyHRis{f&jEFa;DKnm5VD*PKP0Q0z^LYebtbYFh0v<^m z%J!>l*HrZz<}g!D5o}-~a>t&(g{?imo6WdddLB4NVRtR+_yWId0SNz~ltZN>QMt0|h-rGIyTH=IUAf$)2{3Gr{wc zG?-~untOu9@J!Am9>_|(5q-k6>Fjbj2NT`jMWs<9t#$6f3P(7`_5t~7kii&GYva!K zJo6BaJ@wiu8i1!40tTJBjb7{31a{7)R+I2W&eRD2JJq0yVVU5!Sld2YeU+Wi(O}po5lctM0 zKdaO@Z`{6nr@}XA;2t&iu`rU1yP&1guOq<4?=a{&sYs1CE%})-{o1zfdc0h59MIM2 zy_oFcvV(U|#%!v#ZAABz>zzk3F7@Tc?%IaBb%fHcR*thx6jV;|Z}Z<6n=Ipc8WzPd zGMuLr>(A*aOeez%!N2u(z4M5guSG4U8ND3#?m<_*V(NPyiwszFO#3%M%woKE;w4m} zFQ>Gm%GTO$my{P>h~U14cwE3ao?Mk#TcO(KYgv8v2>}fGS+`cEZWURyD2&x3S)qpD256BT=K zA&9ZDij_Jur@`=aEQR9A8Zok%rEyCDVFOaMtffR7f(!1s6x||VC#zMr z6d2aGoO^uFG<6lcId)y?i*g3p$9zz)(NS)t%c=_wmwl5vc5OqOLdmnnqfd~ZSDQ+C z_p1a!(v#skeW#opz5D~je!j|qG`F?%q+DqShETG@e@tV2gT*pwMn^zLuxyqLI~(mM zIAa2}Y3df{zz0eGZW=a(t-podWX6`V?`lif5~wZNBcN~@@4HEase7PRhEQj*L2c3p z9gTA8?_A`A>wHS@I*lEaH$7QAad+KgA!X;*O{Mt5IVPV+yLg(OOQJdL? zij%_dN*G(iS+M)!>0!|#^*Gz4Gi$3F*7Q!C0a|8ZojyllGV}*!ZXll_6z;};F`X-7G1aY8;@6@mis~wKH_IuJ_;%U zOHX*@Mg5Aez_pp$1I98W&tpG4MQlDg|7@AB*s^{i|J4>`>W6Qa!!qZ}O4UF8i5HUi~ zr`&AF_|2uGR@sR;#o8|uzdV4i z+oJ&|zIQPF!jjFF&WV_cU6U^o#vy&l!2k(nAyxQ+E5nVfooo66KYp5){Cv+Ji4hw*sGEV41PSO{QjVW9U6*Vf$6ybDo5edH zqa*j_olbrH4X`!_2FvEOO)9-`_kUG}HtuLu>VA2u9*}b)DrG#Ho4UA-8oz~14VlN+ znvs4Tksj}Ve?uHQDT7D$irZ1NtNCX{O{T~9w=D0)dTfOCGezjy>2|sC@+UI#LW_0a z-!(tf1*8J}|A3u2U#3<03gS~|?A>}1t_0a}j7%peiM+YVdXD&JxxNS0ArN^Jd=gzv zTp-CxxgHGo^7@ixEs%FS`DM$>~Z6#A{Aq9YfJkSW!E(nG!*A|+Qw=}7Tw|*T+z_BvBX^VhXG6U*U*`5H zv}%pJ{p{$*s|hcU-w#aNWtLIs)Vdr%T!FKp$$zXFw_2SN0t&#xgF0o$rp5kYP8W|h zEwml1G3!<;UwT6WoVmXbEeEu+cwkmdvHNzR*^G8FkT4W_XW`dW;yO=FXh2140_Bbp zK{It6rCk)+?G19&D_W_cv(!hS*<=6je;E*UOYh?5y(WCOr8uUD%i28>9@#OzFZ=%C z$GL0Mavf5$u|@xBr&t5hr*>X1LG^(*D3S4fhXap}>0m(eZ0a2QIrEFLnAIk*TS52Z#tND0c0$qD`cW@!} zBI5^O5vtNGw8Ua}eYrqP?A6(od^uI3Y{r==f&W7 zj!Hv-_~)!XM*VEeAs@Y>)hAz0^CPOLE;3V=RLhiooR7>Lb&-m1IS+*7g_wm1@o;%b zaj>DdSU}3Iqb#-5ZOaxt@Sy<DkHS^Iu(iA;Vv&Zk^!12+`A;@Zr=OzEEchMh2X5*T*kjEu^r&k7uZbDxd{q}z1QUVnhO=z`w__ia8G-MAaxk>H2cpvjaFC<<6##%xkzy2tjZiNV zNbMgS7(ZDcDvP_7It@tx)Y$t!5GVkpd*gOkYDv9piJ?D>*v<*CIK?%0!$puL!$wM{ zn;hSNV;v2Z7f#AcFtF0Vweccku;l>dk$$`Im{w+zkl5@ti7H%>*+Es5XcM2Y7mwv9Er zG&^UmIZzpC#mzNp8YrS(Q*B1*5lBO@P6N0&&VC=lQ%o;&*bAJCaOm>^z*gb5o!DG8 z|6fEvSsa?LFszIu$K$lRkXV@*9@v|l_gXdmk;AMjrAr~-%sCnj$=^mat7XFJm6;!k zIuOOjsNqNdjb7`_9uV!lMqd;Omk0Z;>ILYoksqbM@klg53kgz{If#~aoqcFAxxrLo zaKQ$teN0p3rBx8>;h~_ETbqy=X5>v%f|&82Fbi_Z1i(Q|t2-%fNdW<4;LSGLBqa*G z0DVQiKwTYi&bV|u%(T)$j$vXO&pArw9JEKO`TAao6}Qa+twck3 zKFOG=)FZob8cUdbDAFJ)jqROErrivaD=96TkszvavuyZP7pwEvjY5)l0j_;6-givr?ba?n6a!|UaXno6cAKi;34BEJ z@k5XW{{wnV9StME%6{lt;YZlBT}TB)lL7dp>0W=)yUdew^-*GqW&pnv4D=+op_fTK2de+%jFLgDu?;nwSI>3OO^CpQH3h<%ZCf^aOJL MXC09T0s{etpfC(dFaQ7m literal 0 HcmV?d00001 diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/tempto-configuration.yaml b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/tempto-configuration.yaml new file mode 100644 index 000000000000..a67984bdc989 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-kafka-ssl/tempto-configuration.yaml @@ -0,0 +1,17 @@ +schema-registry: + url: http://schema-registry:8081 + +databases: + kafka: + broker: + host: kafka + port: 9092 + security.protocol: "SSL" + ssl.endpoint.identification.algorithm: "https" + ssl.key.password: "confluent" + ssl.keystore.location: "/docker/presto-product-tests/conf/tempto/secrets/kafka.client.keystore" + ssl.keystore.password: "confluent" + ssl.keystore.type: "JKS" + ssl.truststore.location: "/docker/presto-product-tests/conf/tempto/secrets/kafka.client.truststore" + ssl.truststore.password: "confluent" + ssl.truststore.type: "JKS" From e8d7d17b20dc6ba997de7ebc4404321c210c6afb Mon Sep 17 00:00:00 2001 From: Karol Sobczak Date: Thu, 1 Apr 2021 16:44:51 +0200 Subject: [PATCH 088/146] Remove unused exception --- .../src/test/java/io/trino/operator/TestHashJoinOperator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/trino-main/src/test/java/io/trino/operator/TestHashJoinOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestHashJoinOperator.java index ea05bc47f2df..38ed8a6dfbc9 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestHashJoinOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestHashJoinOperator.java @@ -393,7 +393,6 @@ public void testInnerJoinWithSpill(boolean probeHashEnabled, List whe @Test(dataProvider = "joinWithFailingSpillValues") public void testInnerJoinWithFailingSpill(boolean probeHashEnabled, List whenSpill, WhenSpillFails whenSpillFails, boolean isDictionaryProcessingJoinEnabled) - throws Exception { DummySpillerFactory buildSpillerFactory = new DummySpillerFactory(); DummySpillerFactory joinSpillerFactory = new DummySpillerFactory(); From 27ffdb950bee120d0937442ea72df30929d64d01 Mon Sep 17 00:00:00 2001 From: Karol Sobczak Date: Fri, 9 Apr 2021 12:57:01 +0200 Subject: [PATCH 089/146] Disable join spill for grouped execution --- .../main/java/io/trino/sql/planner/LocalExecutionPlanner.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java index 767b7407c971..ba9990b9c7bf 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java @@ -2110,7 +2110,9 @@ private PhysicalOperation createLookupJoin( PhysicalOperation probeSource = probeNode.accept(this, context); // Plan build - boolean spillEnabled = isSpillEnabled(session) && node.isSpillable().orElseThrow(() -> new IllegalArgumentException("spillable not yet set")); + boolean spillEnabled = isSpillEnabled(session) + && node.isSpillable().orElseThrow(() -> new IllegalArgumentException("spillable not yet set")) + && probeSource.getPipelineExecutionStrategy() == UNGROUPED_EXECUTION; JoinBridgeManager lookupSourceFactory = createLookupSourceFactory(node, buildNode, buildSymbols, buildHashSymbol, probeSource, context, spillEnabled, localDynamicFilters); From 85f4cef3f7791b05998b174592c8e4ab3446689b Mon Sep 17 00:00:00 2001 From: Karol Sobczak Date: Tue, 30 Mar 2021 10:57:03 +0200 Subject: [PATCH 090/146] Fix join spill race condition SpilledLookupSourceHandle needs to be disposed immediately when join operators finished processing probe data. Otherwise SpilledLookupSourceHandle is never disposed as it's not registered in PartitionedConsumption. This prevents HashBuilderOperator from finishing. Fixes: https://github.com/trinodb/trino/issues/7454 --- .../PartitionedLookupSourceFactory.java | 12 +++++-- .../sql/planner/LocalExecutionPlanner.java | 13 ++++--- .../trino/operator/TestHashJoinOperator.java | 35 +++++++++++++++++++ 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/operator/PartitionedLookupSourceFactory.java b/core/trino-main/src/main/java/io/trino/operator/PartitionedLookupSourceFactory.java index 50ca998a707d..19ef6fbe79d8 100644 --- a/core/trino-main/src/main/java/io/trino/operator/PartitionedLookupSourceFactory.java +++ b/core/trino-main/src/main/java/io/trino/operator/PartitionedLookupSourceFactory.java @@ -212,7 +212,7 @@ public void setPartitionSpilledLookupSourceHandle(int partitionIndex, SpilledLoo lock.writeLock().lock(); try { - if (destroyed.isDone()) { + if (partitionsNoLongerNeeded.isDone()) { spilledLookupSourceHandle.dispose(); return; } @@ -302,6 +302,14 @@ public ListenableFuture>> finishPr try { if (!spillingInfo.hasSpilled()) { finishedProbeOperators++; + if (lookupJoinsCount.isPresent()) { + checkState(finishedProbeOperators <= lookupJoinsCount.getAsInt(), "%s probe operators finished out of %s declared", finishedProbeOperators, lookupJoinsCount.getAsInt()); + if (finishedProbeOperators == lookupJoinsCount.getAsInt()) { + // We can dispose partitions now since right outer is not supported with spill and lookupJoinsCount should be absent + freePartitions(); + } + } + return immediateFuture(new PartitionedConsumption<>( 1, emptyList(), @@ -322,7 +330,7 @@ public ListenableFuture>> finishPr finishedProbeOperators++; if (finishedProbeOperators == operatorsCount) { - // We can dispose partitions now since as right outer is not supported with spill + // We can dispose partitions now since right outer is not supported with spill freePartitions(); verify(!partitionedConsumption.isDone()); partitionedConsumption.set(new PartitionedConsumption<>( diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java index ba9990b9c7bf..075ab41c23a0 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java @@ -2110,9 +2110,11 @@ private PhysicalOperation createLookupJoin( PhysicalOperation probeSource = probeNode.accept(this, context); // Plan build + boolean buildOuter = node.getType() == RIGHT || node.getType() == FULL; boolean spillEnabled = isSpillEnabled(session) && node.isSpillable().orElseThrow(() -> new IllegalArgumentException("spillable not yet set")) - && probeSource.getPipelineExecutionStrategy() == UNGROUPED_EXECUTION; + && probeSource.getPipelineExecutionStrategy() == UNGROUPED_EXECUTION + && !buildOuter; JoinBridgeManager lookupSourceFactory = createLookupSourceFactory(node, buildNode, buildSymbols, buildHashSymbol, probeSource, context, spillEnabled, localDynamicFilters); @@ -2221,7 +2223,7 @@ private JoinBridgeManager createLookupSourceFact searchFunctionFactories, 10_000, pagesIndexFactory, - spillEnabled && !buildOuter && partitionCount > 1, + spillEnabled && partitionCount > 1, singleStreamSpillerFactory); context.addDriverFactory( @@ -2325,8 +2327,11 @@ private OperatorFactory createLookupJoin( List probeJoinChannels = ImmutableList.copyOf(getChannelsForSymbols(probeSymbols, probeSource.getLayout())); OptionalInt probeHashChannel = probeHashSymbol.map(channelGetter(probeSource)) .map(OptionalInt::of).orElse(OptionalInt.empty()); - OptionalInt totalOperatorsCount = context.getDriverInstanceCount(); - checkState(!spillEnabled || totalOperatorsCount.isPresent(), "A fixed distribution is required for JOIN when spilling is enabled"); + OptionalInt totalOperatorsCount = OptionalInt.empty(); + if (spillEnabled) { + totalOperatorsCount = context.getDriverInstanceCount(); + checkState(totalOperatorsCount.isPresent(), "A fixed distribution is required for JOIN when spilling is enabled"); + } // Implementation of hash join operator may only take advantage of output duplicates insensitive joins when: // 1. Join is of INNER or LEFT type. For right or full joins all matching build rows must be tagged as visited. diff --git a/core/trino-main/src/test/java/io/trino/operator/TestHashJoinOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestHashJoinOperator.java index 38ed8a6dfbc9..8ec3090b3044 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestHashJoinOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestHashJoinOperator.java @@ -600,6 +600,41 @@ private static MaterializedResult getProperColumns(Operator joinOperator, List lookupSourceFactoryManager = buildSideSetup.getLookupSourceFactoryManager(); + PartitionedLookupSourceFactory lookupSourceFactory = lookupSourceFactoryManager.getJoinBridge(Lifespan.taskWide()); + + // finish probe before any build partition is spilled + lookupSourceFactory.finishProbeOperator(OptionalInt.of(1)); + + // spill build partition after probe is finished + HashBuilderOperator hashBuilderOperator = buildSideSetup.getBuildOperators().get(0); + hashBuilderOperator.startMemoryRevoke().get(); + hashBuilderOperator.finishMemoryRevoke(); + hashBuilderOperator.finish(); + + // hash builder operator should not deadlock waiting for spilled lookup source to be disposed + hashBuilderOperator.isBlocked().get(); + + lookupSourceFactory.destroy(); + assertTrue(hashBuilderOperator.isFinished()); + } + @Test(dataProvider = "hashJoinTestValues") public void testInnerJoinWithNullProbe(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) { From 987b1dcf3f0e10a75bc57a82f4588adb4e9223ec Mon Sep 17 00:00:00 2001 From: Karol Sobczak Date: Fri, 2 Apr 2021 17:35:32 +0200 Subject: [PATCH 091/146] Require that materialized view has owner Materialized views can only be analyzed with definer context. Therefore view owner needs to be present. --- .../io/trino/execution/CreateMaterializedViewTask.java | 2 +- .../java/io/trino/sql/analyzer/StatementAnalyzer.java | 1 + .../test/java/io/trino/sql/analyzer/TestAnalyzer.java | 2 +- .../connector/ConnectorMaterializedViewDefinition.java | 9 +++++++-- .../test/java/io/trino/plugin/hive/AbstractTestHive.java | 2 +- .../java/io/trino/plugin/iceberg/IcebergMetadata.java | 2 +- .../java/io/trino/execution/TestEventListenerBasic.java | 4 ++-- .../src/test/java/io/trino/tests/TestMockConnector.java | 2 +- 8 files changed, 15 insertions(+), 9 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/execution/CreateMaterializedViewTask.java b/core/trino-main/src/main/java/io/trino/execution/CreateMaterializedViewTask.java index fa08dde438a2..a26bb61e4885 100644 --- a/core/trino-main/src/main/java/io/trino/execution/CreateMaterializedViewTask.java +++ b/core/trino-main/src/main/java/io/trino/execution/CreateMaterializedViewTask.java @@ -99,7 +99,7 @@ public ListenableFuture execute( .map(field -> new ConnectorMaterializedViewDefinition.Column(field.getName().get(), field.getType().getTypeId())) .collect(toImmutableList()); - Optional owner = Optional.of(session.getUser()); + String owner = session.getUser(); CatalogName catalogName = metadata.getCatalogHandle(session, name.getCatalogName()) .orElseThrow(() -> new TrinoException(NOT_FOUND, "Catalog does not exist: " + name.getCatalogName())); diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java index 5f34dba6c0c1..a5251e702a74 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java @@ -1447,6 +1447,7 @@ private Scope createScopeForCommonTableExpression(Table table, Optional s private Scope createScopeForMaterializedView(Table table, QualifiedObjectName name, Optional scope, ConnectorMaterializedViewDefinition view) { + checkArgument(view.getOwner().isPresent(), "owner must be present"); return createScopeForView( table, name, diff --git a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java index 789e28d77fb3..a07ff795b53a 100644 --- a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java +++ b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java @@ -3055,7 +3055,7 @@ public void setup() Optional.of("s1"), ImmutableList.of(new ConnectorMaterializedViewDefinition.Column("a", BIGINT.getTypeId())), Optional.of("comment"), - Optional.of("user"), + "user", ImmutableMap.of()); inSetupTransaction(session -> metadata.createMaterializedView(session, new QualifiedObjectName(TPCH_CATALOG, "s1", "mv1"), materializedViewData1, false, true)); diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMaterializedViewDefinition.java b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMaterializedViewDefinition.java index ba6da8447999..c8a2719c03c4 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMaterializedViewDefinition.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMaterializedViewDefinition.java @@ -44,15 +44,16 @@ public ConnectorMaterializedViewDefinition( @JsonProperty("schema") Optional schema, @JsonProperty("columns") List columns, @JsonProperty("comment") Optional comment, - @JsonProperty("owner") Optional owner, + @JsonProperty("owner") String owner, @JsonProperty("properties") Map properties) { - this(originalSql, requireNonNull(storageTable, "storageTable is null").orElse(null), catalog, schema, columns, comment, owner, properties); + this(originalSql, requireNonNull(storageTable, "storageTable is null").orElse(null), catalog, schema, columns, comment, Optional.of(owner), properties); } /* * This constructor is for JSON deserialization only. Do not use. */ + // TODO: Simplify this constructor and getters: https://github.com/trinodb/trino/issues/7537 @Deprecated @JsonCreator public ConnectorMaterializedViewDefinition( @@ -80,6 +81,10 @@ public ConnectorMaterializedViewDefinition( if (columns.isEmpty()) { throw new IllegalArgumentException("columns list is empty"); } + + if (owner.isEmpty()) { + throw new IllegalArgumentException("owner must be present"); + } } @JsonProperty diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java index 6ad6007e2974..a7e0212a00e9 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java @@ -837,7 +837,7 @@ public Optional getMaterializedView(Connect Optional.empty(), ImmutableList.of(new ConnectorMaterializedViewDefinition.Column("abc", TypeId.of("type"))), Optional.empty(), - Optional.empty(), + "alice", ImmutableMap.of())); } }, diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 4f15e263e287..2befb6cec9b7 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -1001,7 +1001,7 @@ public Optional getMaterializedView(Connect Optional.of(viewName.getSchemaName()), definition.getColumns(), definition.getComment(), - Optional.of(materializedView.getOwner()), + materializedView.getOwner(), new HashMap<>(materializedView.getParameters()))); } diff --git a/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java b/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java index 88bbec20e050..fb6dec2509fb 100644 --- a/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java +++ b/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java @@ -135,7 +135,7 @@ public Iterable getConnectorFactories() Optional.empty(), ImmutableList.of(new Column("test_column", BIGINT.getTypeId())), Optional.empty(), - Optional.empty(), + "alice", ImmutableMap.of()); SchemaTableName materializedViewName = new SchemaTableName("default", "test_materialized_view"); return ImmutableMap.of(materializedViewName, definition); @@ -373,7 +373,7 @@ public void testReferencedTablesWithMaterializedViews() assertThat(table.getCatalog()).isEqualTo("tpch"); assertThat(table.getSchema()).isEqualTo("tiny"); assertThat(table.getTable()).isEqualTo("nation"); - assertThat(table.getAuthorization()).isEqualTo("user"); + assertThat(table.getAuthorization()).isEqualTo("alice"); assertThat(table.isDirectlyReferenced()).isFalse(); assertThat(table.getFilters()).isEmpty(); assertThat(table.getColumns()).hasSize(1); diff --git a/testing/trino-tests/src/test/java/io/trino/tests/TestMockConnector.java b/testing/trino-tests/src/test/java/io/trino/tests/TestMockConnector.java index f7c54eac7263..e71c0fecb174 100644 --- a/testing/trino-tests/src/test/java/io/trino/tests/TestMockConnector.java +++ b/testing/trino-tests/src/test/java/io/trino/tests/TestMockConnector.java @@ -59,7 +59,7 @@ protected QueryRunner createQueryRunner() Optional.of("default"), ImmutableList.of(new Column("nationkey", BIGINT.getTypeId())), Optional.empty(), - Optional.empty(), + "alice", ImmutableMap.of()))) .build())); queryRunner.createCatalog("mock", "mock"); From 8bac01572d458e100c98e8b6c432a41215391f52 Mon Sep 17 00:00:00 2001 From: Will Morrison Date: Thu, 18 Mar 2021 23:05:14 -0400 Subject: [PATCH 092/146] Do case insensitive comparison between dereferenced fields and internal ORC field names --- .../plugin/hive/orc/OrcPageSourceFactory.java | 15 +++++--- .../tests/hive/TestHiveStorageFormats.java | 36 +++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSourceFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSourceFactory.java index e69ac8afb82e..97e0fc5ebceb 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSourceFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcPageSourceFactory.java @@ -298,16 +298,14 @@ private static ConnectorPageSource createOrcPageSource( .collect(Collectors.groupingBy( HiveColumnHandle::getBaseColumnName, mapping( - column -> column.getHiveColumnProjectionInfo().map(HiveColumnProjectionInfo::getDereferenceNames).orElse(ImmutableList.of()), - toList()))); + OrcPageSourceFactory::getDereferencesAsList, toList()))); } else { projectionsByColumnIndex = projections.stream() .collect(Collectors.groupingBy( HiveColumnHandle::getBaseHiveColumnIndex, mapping( - column -> column.getHiveColumnProjectionInfo().map(HiveColumnProjectionInfo::getDereferenceNames).orElse(ImmutableList.of()), - toList()))); + OrcPageSourceFactory::getDereferencesAsList, toList()))); } TupleDomainOrcPredicateBuilder predicateBuilder = TupleDomainOrcPredicate.builder() @@ -540,4 +538,13 @@ private static OrcColumn getNestedColumn(OrcColumn baseColumn, Optional getDereferencesAsList(HiveColumnHandle column) + { + return column.getHiveColumnProjectionInfo() + .map(info -> info.getDereferenceNames().stream() + .map(dereference -> dereference.toLowerCase(ENGLISH)) + .collect(toImmutableList())) + .orElse(ImmutableList.of()); + } } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/hive/TestHiveStorageFormats.java b/testing/trino-product-tests/src/main/java/io/trino/tests/hive/TestHiveStorageFormats.java index b8524372fcb1..d6f37b5e98da 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/hive/TestHiveStorageFormats.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/hive/TestHiveStorageFormats.java @@ -480,6 +480,42 @@ public void testSnappyCompressedParquetTableCreatedInHive() onHive().executeQuery("DROP TABLE " + tableName); } + @Test + public void testOrcStructsWithNonLowercaseFields() + throws SQLException + { + String tableName = "orc_structs_with_non_lowercase"; + + ensureDummyExists(); + onHive().executeQuery("DROP TABLE IF EXISTS " + tableName); + + onHive().executeQuery(format( + "CREATE TABLE %s (" + + " c_bigint BIGINT," + + " c_struct struct)" + + "STORED AS ORC ", + tableName)); + + onHive().executeQuery(format( + "INSERT INTO %s" + // insert with SELECT because hive does not support array/map/struct functions in VALUES + + " SELECT" + + " 1," + + " named_struct('testCustId', '1234', 'requestDate', 'some day')" + // some hive versions don't allow INSERT from SELECT without FROM + + " FROM dummy", + tableName)); + + setSessionProperty(onTrino().getConnection(), "hive.projection_pushdown_enabled", "true"); + assertThat(onTrino().executeQuery("SELECT c_struct.testCustId FROM " + tableName)).containsOnly(row("1234")); + assertThat(onTrino().executeQuery("SELECT c_struct.testcustid FROM " + tableName)).containsOnly(row("1234")); + assertThat(onTrino().executeQuery("SELECT c_struct.requestDate FROM " + tableName)).containsOnly(row("some day")); + setSessionProperty(onTrino().getConnection(), "hive.projection_pushdown_enabled", "false"); + assertThat(onTrino().executeQuery("SELECT c_struct.testCustId FROM " + tableName)).containsOnly(row("1234")); + assertThat(onTrino().executeQuery("SELECT c_struct.testcustid FROM " + tableName)).containsOnly(row("1234")); + assertThat(onTrino().executeQuery("SELECT c_struct.requestDate FROM " + tableName)).containsOnly(row("some day")); + } + @Test(dataProvider = "storageFormatsWithNanosecondPrecision") public void testTimestampCreatedFromHive(StorageFormat storageFormat) { From b357388ffeec5392cd3adca8085e4e428ca6b261 Mon Sep 17 00:00:00 2001 From: David Stryker Date: Sat, 6 Mar 2021 14:46:59 -0800 Subject: [PATCH 093/146] Add Trino client protocol documentation --- docs/src/main/sphinx/develop.rst | 1 + .../main/sphinx/develop/client-protocol.rst | 265 ++++++++++++++++++ docs/src/main/sphinx/installation/cli.rst | 4 + docs/src/main/sphinx/installation/jdbc.rst | 4 + 4 files changed, 274 insertions(+) create mode 100644 docs/src/main/sphinx/develop/client-protocol.rst diff --git a/docs/src/main/sphinx/develop.rst b/docs/src/main/sphinx/develop.rst index 73e3dc0de5fe..1a254501fa29 100644 --- a/docs/src/main/sphinx/develop.rst +++ b/docs/src/main/sphinx/develop.rst @@ -18,3 +18,4 @@ This guide is intended for Trino contributors and plugin developers. develop/certificate-authenticator develop/group-provider develop/event-listener + develop/client-protocol diff --git a/docs/src/main/sphinx/develop/client-protocol.rst b/docs/src/main/sphinx/develop/client-protocol.rst new file mode 100644 index 000000000000..7ca48be162c2 --- /dev/null +++ b/docs/src/main/sphinx/develop/client-protocol.rst @@ -0,0 +1,265 @@ +====================== +Trino client REST API +====================== + +The REST API allows clients to submit SQL queries to Trino and receive the +results. Clients include the CLI, the JDBC driver, and others provided by +the community. The preferred method to interact with Trino is using these +existing clients. This document provides details about the API for reference. +It can also be used to implement your own client, if necessary. + +HTTP methods +------------ + +* A ``POST`` to ``/v1/statement`` runs the query string in the ``POST`` body, + and returns a JSON document containing the query results. If there are more + results, the JSON document contains a ``nextUri`` URL attribute. +* A ``GET`` to the ``nextUri`` attribute returns the next batch of query results. +* A ``DELETE`` to ``nextUri`` terminates a running query. + +Overview of query processing +---------------------------- + +A Trino client request is initiated by an HTTP ``POST`` to the endpoint +``/v1/statement``, with a ``POST`` body consisting of the SQL query string. +The caller may set HTTP header field ``X-Trino-User`` to the username for +the session. A number of other optional header fields are documented in +the following sections. + +If the client request returns an HTTP 503, that means the server was busy, +and the client should try again in 50-100 milliseconds. Any HTTP status other +than 503 or 200 means that query processing has failed. + +The ``/v1/statement`` ``POST`` request returns a JSON document of type +``QueryResults``, as well as a collection of response headers. The +``QueryResults`` document contains an ``error`` field of type +``QueryError`` if the query has failed, and if that object is not present, +the query succeeded. Important members of ``QueryResults`` are documented +in the following sections. + +If the ``data`` field of the JSON document is set, it contains a list of the +rows of data. The ``columns`` field is set to a list of the +names and types of the columns returned by the query. Most of the response +headers are treated like browser cookies by the client, and echoed back +as request headers in subsequent client requests, as documented below. + +If the JSON document returned by the ``POST`` to ``/v1/statement`` does not +contain a ``nextUri`` link, the query has completed, either successfully or +unsuccessfully, and no additional requests need to be made. If the +``nextUri`` link is present in the document, there are more query results +to be fetched. The client should loop executing a ``GET`` request +to the ``nextUri`` returned in the ``QueryResults`` response object until +``nextUri`` is absent from the response. + +The ``status`` field of the JSON document is for human consumption only, and +provides a hint about the query state. It can not be used to tell if the +query is finished. + +Important ``QueryResults`` attributes +------------------------------------- + +The most important attributes of the ``QueryResults`` JSON document returned by the REST API +endpoints are listed in this table. Refer to the class ``io.trino.client.QueryResults`` in +module ``trino-client`` for more details. + +.. list-table:: ``QueryResults attributes`` + :widths: 25, 55 + :header-rows: 1 + + * - Attribute + - Description + * - ``id`` + - The ID of the query. + * - ``nextUri`` + - If present, the URL to use for subsequent ``GET`` or + ``DELETE`` requests. If not present, the query is complete or + ended in error. + * - ``columns`` + - A list of the names and types of the columns returned by the query. + * - ``data`` + - The ``data`` attribute contains a list of the rows returned by the + query request. Each row is itself a list that holds values of the + columns in the row, in the order specified by the ``columns`` + attribute. + * - ``updateType`` + - A human-readable string representing the operation. For a + ``CREATE TABLE`` request, the ``updateType`` is + "CREATE TABLE"; for ``SET SESSION`` it is "SET SESSION"; etc. + * - ``error`` + - If query failed, the ``error`` attribute contains a + ``QueryError`` object. That object contains a ``message``, an + ``errorCode`` and other information about the error. See the + ``io.trino.client.QueryError`` class in module ``trino-client`` + for more details. + + +``QueryResults`` diagnostic attributes +-------------------------------------- + +These ``QueryResults`` data members may be useful in tracking down problems: + +.. list-table:: ``QueryResults diagnostic attributes`` + :widths: 20, 20, 40 + :header-rows: 1 + + * - Attribute + - Type + - Description + * - ``queryError`` + - ``QueryError`` + - Non-null only if the query resulted in an error. + * - ``failureInfo`` + - ``FailureInfo`` + - ``failureInfo`` has detail on the reason for the failure, including + a stack trace, and ``FailureInfo.errorLocation``, providing the + query line number and column number where the failure was detected. + * - ``warnings`` + - ``List`` + - A usually-empty list of warnings. + * - ``statementStats`` + - ``StatementStats`` + - A class containing statistics about the query execution. Of + particular interest is ``StatementStats.rootStage``, of type + ``StageStats``, providing statistics on the execution of each of + the stages of query processing. + +Client request headers +---------------------- + +This table lists all supported client request headers. Many of the +headers can be updated in the client as response headers, and supplied +in subsequent requests, just like browser cookies. + +.. list-table:: Client request headers + :widths: 30, 50 + :header-rows: 1 + + * - Header name + - Description + * - ``X-Trino-User`` + - Specifies the session user; must be supplied with every + request to ``/v1/statement``. + * - ``X-Trino-Source`` + - For reporting purposes, this supplies the name of the software + that submitted the query. + * - ``X-Trino-Catalog`` + - The catalog context for query processing. Set by response + header ``X-Trino-Set-Catalog``. + * - ``X-Trino-Schema`` + - The schema context for query processing. Set by response + header ``X-Trino-Set-Schema``. + * - ``X-Trino-Time-Zone`` + - The timezone for query processing. Defaults to the timezone + of the Trino cluster, and not the timezone of the client. + * - ``X-Trino-Language`` + - The language to use when processing the query and formatting + results, formatted as a Java ``Locale`` string, e.g., ``en-US`` + for US English. The language of the + session can be set on a per-query basis using the + ``X-Trino-Language`` HTTP header. + * - ``X-Trino-Trace-Token`` + - Supplies a trace token to the Trino engine to help identify + log lines that originate with this query request. + * - ``X-Trino-Session`` + - Supplies a comma-separated list of name=value pairs as session + properties. When the Trino client run a + ``SET SESSION name=value`` query, the name=value pair + is returned in the ``X-Set-Trino-Session`` response header, + and added to the client's list of session properties. + If the response header ``X-Trino-Clear-Session`` is returned, + its value is the name of a session property that is + removed from the client's accumulated list. + * - ``X-Trino-Role`` + - Sets the "role" for query processing. A "role" is represents + a collection of permissions. Set by response header + ``X-Trino-Set-Role``. See doc:/sql/create-role to + understand roles. + * - ``X-Trino-Prepared-Statement`` + - A comma-separated list of the name=value pairs, where the + names are names of previously prepared SQL statements, and + the values are keys that identify the executable form of the + named prepared statements. + * - ``X-Trino-Transaction-Id`` + - The transaction ID to use for query processing. Set + by response header ``X-Trino-Started-Transaction-Id`` and + cleared by ``X-Trino-Clear-Transaction-Id``. + * - ``X-Trino-Client-Info`` + - Contains arbitrary information about the client program + submitting the query. + * - ``X-Trino-Client-Tags`` + - A comma-separated list of "tag" strings, used to identify + Trino resource groups. + * - ``X-Trino-Resource-Estimate`` + - A comma-separated list of ``resource=value`` type + assigments. The possible choices of ``resource`` are + ``EXECUTION_TIME``, ``CPU_TIME``, ``PEAK_MEMORY`` and + ``PEAK_TASK_MEMORY``. ``EXECUTION_TIME`` and ``CPU_TIME`` + have values specified as airlift ``Duration`` strings + The format is a double precision number followed by + a ``TimeUnit`` string, e.g., of ``s`` for seconds, + ``m`` for minutes, ``h`` for hours, etc. "PEAK_MEMORY" and + "PEAK_TASK_MEMORY" are specified as as airlift ``DataSize`` strings, + whose format is an integer followed by ``B`` for bytes; ``kB`` for + kilobytes; ``mB`` for megabytes, ``gB`` for gigabytes, etc. + * - ``X-Trino-Extra-Credential`` + - Provides extra credentials to the connector. The header is + a name=value string that is saved in the session ``Identity`` + object. The name and value are only meaningful to the connector. + +Client response headers +----------------------- + +This table lists the supported client response headers. After receiving a +response, a client must update the request headers used in +subsequent requests to be consistent with the response headers received. + +.. list-table:: Client response headers + :widths: 30, 50 + :header-rows: 1 + + * - Header name + - Description + * - ``X-Trino-Set-Catalog`` + - Instructs the client to set the catalog in the + ``X-Trino-Catalog`` request header in subsequent client requests. + * - ``X-Trino-Set-Schema`` + - Instructs the client to set the schema in the + ``X-Trino-Schema`` request header in subsequent client requests. + * - ``X-Trino-Set-Session`` + - The value of the ``X-Trino-Set-Session`` response header is a + string of the form *property* = *value*. It + instructs the client include session property *property* with value + *value* in the ``X-Trino-Session`` header of subsequent + client requests. + * - ``X-Trino-Clear-Session`` + - Instructs the client to remove the session property with the + whose name is the value of the ``X-Trino-Clear-Session`` header + from the list of session properties + in the ``X-Trino-Session`` header in subsequent client requests. + * - ``X-Trino-Set-Role`` + - Instructs the client to set ``X-Trino-Role`` request header to the + catalog role supplied by the ``X-Trino-Set-Role`` header + in subsequent client requests. + * - ``X-Trino-Added-Prepare`` + - Instructs the client to add the name=value pair to the set of + prepared statements in the ``X-Trino-Prepared-Statements`` + request header in subsequent client requests. + * - ``X-Trino-Deallocated-Prepare`` + - Instructs the client to remove the prepared statement whose name + is the value of the ``X-Trino-Deallocated-Prepare`` header from + the client's list of prepared statements sent in the + ``X-Trino-Prepared-Statements`` request header in subsequent client + requests. + * - ``X-Trino-Started-Transaction-Id`` + - Provides the transaction ID that the client should pass back in the + ``X-Trino-Transaction-Id`` request header in subsequent requests. + * - ``X-Trino-Clear-Transaction-Id`` + - Instructs the client to clear the ``X-Trino-Transaction-Id`` request + header in subsequent requests. + +``ProtocolHeaders`` +------------------- + +Class ``io.trino.client.ProtocolHeaders``, in module ``trino-client``, +enumerates all the HTTP request and response headers allowed by the +Trino client REST API. diff --git a/docs/src/main/sphinx/installation/cli.rst b/docs/src/main/sphinx/installation/cli.rst index 7a206199823c..32ba242085c9 100644 --- a/docs/src/main/sphinx/installation/cli.rst +++ b/docs/src/main/sphinx/installation/cli.rst @@ -25,6 +25,10 @@ make it executable with ``chmod +x``, then run it: Run the CLI with the ``--help`` option to see the available options. +The CLI uses the HTTP protocol and the +:doc:`Trino client REST API ` to communicate +with Trino. + Authentication -------------- diff --git a/docs/src/main/sphinx/installation/jdbc.rst b/docs/src/main/sphinx/installation/jdbc.rst index c87ed6f57dfa..2b72d865fe2b 100644 --- a/docs/src/main/sphinx/installation/jdbc.rst +++ b/docs/src/main/sphinx/installation/jdbc.rst @@ -42,6 +42,10 @@ classpath, you'll typically need to restart your application in order to recognize the new driver. Then, depending on your application, you may need to manually register and configure the driver. +The CLI uses the HTTP protocol and the +:doc:`Trino client REST API ` to communicate +with Trino. + Registering and configuring the driver -------------------------------------- From 9cf7da2ba4551704103a50737dd692933c54b63f Mon Sep 17 00:00:00 2001 From: Karol Sobczak Date: Fri, 2 Apr 2021 12:34:42 +0200 Subject: [PATCH 094/146] Make materialized view check first --- .../trino/sql/rewrite/ShowQueriesRewrite.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/rewrite/ShowQueriesRewrite.java b/core/trino-main/src/main/java/io/trino/sql/rewrite/ShowQueriesRewrite.java index 720f9db5c6ac..c0c56b653ad8 100644 --- a/core/trino-main/src/main/java/io/trino/sql/rewrite/ShowQueriesRewrite.java +++ b/core/trino-main/src/main/java/io/trino/sql/rewrite/ShowQueriesRewrite.java @@ -474,18 +474,13 @@ private static Expression toExpression(Object value) @Override protected Node visitShowCreate(ShowCreate node, Void context) { - if (node.getType() == VIEW) { + if (node.getType() == MATERIALIZED_VIEW) { QualifiedObjectName objectName = createQualifiedObjectName(session, node, node.getName()); - - if (metadata.getMaterializedView(session, objectName).isPresent()) { - throw semanticException(NOT_SUPPORTED, node, "Relation '%s' is a materialized view, not a view", objectName); - } - - Optional viewDefinition = metadata.getView(session, objectName); + Optional viewDefinition = metadata.getMaterializedView(session, objectName); if (viewDefinition.isEmpty()) { if (metadata.getTableHandle(session, objectName).isPresent()) { - throw semanticException(NOT_SUPPORTED, node, "Relation '%s' is a table, not a view", objectName); + throw semanticException(NOT_SUPPORTED, node, "Relation '%s' is a table, not a materialized view", objectName); } throw semanticException(TABLE_NOT_FOUND, node, "View '%s' does not exist", objectName); } @@ -498,18 +493,23 @@ protected Node visitShowCreate(ShowCreate node, Void context) accessControl.checkCanShowCreateTable(session.toSecurityContext(), new QualifiedObjectName(catalogName.getValue(), schemaName.getValue(), tableName.getValue())); - CreateView.Security security = viewDefinition.get().isRunAsInvoker() ? INVOKER : DEFINER; - String sql = formatSql(new CreateView(QualifiedName.of(ImmutableList.of(catalogName, schemaName, tableName)), query, false, viewDefinition.get().getComment(), Optional.of(security))).trim(); - return singleValueQuery("Create View", sql); + String sql = formatSql(new CreateMaterializedView(Optional.empty(), QualifiedName.of(ImmutableList.of(catalogName, schemaName, tableName)), + query, false, false, new ArrayList<>(), viewDefinition.get().getComment())).trim(); + return singleValueQuery("Create Materialized View", sql); } - if (node.getType() == MATERIALIZED_VIEW) { + if (node.getType() == VIEW) { QualifiedObjectName objectName = createQualifiedObjectName(session, node, node.getName()); - Optional viewDefinition = metadata.getMaterializedView(session, objectName); + + if (metadata.getMaterializedView(session, objectName).isPresent()) { + throw semanticException(NOT_SUPPORTED, node, "Relation '%s' is a materialized view, not a view", objectName); + } + + Optional viewDefinition = metadata.getView(session, objectName); if (viewDefinition.isEmpty()) { if (metadata.getTableHandle(session, objectName).isPresent()) { - throw semanticException(NOT_SUPPORTED, node, "Relation '%s' is a table, not a materialized view", objectName); + throw semanticException(NOT_SUPPORTED, node, "Relation '%s' is a table, not a view", objectName); } throw semanticException(TABLE_NOT_FOUND, node, "View '%s' does not exist", objectName); } @@ -522,9 +522,9 @@ protected Node visitShowCreate(ShowCreate node, Void context) accessControl.checkCanShowCreateTable(session.toSecurityContext(), new QualifiedObjectName(catalogName.getValue(), schemaName.getValue(), tableName.getValue())); - String sql = formatSql(new CreateMaterializedView(Optional.empty(), QualifiedName.of(ImmutableList.of(catalogName, schemaName, tableName)), - query, false, false, new ArrayList<>(), viewDefinition.get().getComment())).trim(); - return singleValueQuery("Create Materialized View", sql); + CreateView.Security security = viewDefinition.get().isRunAsInvoker() ? INVOKER : DEFINER; + String sql = formatSql(new CreateView(QualifiedName.of(ImmutableList.of(catalogName, schemaName, tableName)), query, false, viewDefinition.get().getComment(), Optional.of(security))).trim(); + return singleValueQuery("Create View", sql); } if (node.getType() == TABLE) { From 38960f5e8349870c8c29df83d02608584964e7fb Mon Sep 17 00:00:00 2001 From: Karol Sobczak Date: Fri, 2 Apr 2021 12:43:58 +0200 Subject: [PATCH 095/146] Make SHOW CREATE checks consistent with how user sees objects Table scan existence check order: * Materialized views * Views * Tables --- .../trino/sql/rewrite/ShowQueriesRewrite.java | 14 +++- .../io/trino/sql/analyzer/TestAnalyzer.java | 72 +++++++++++++++++++ 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/rewrite/ShowQueriesRewrite.java b/core/trino-main/src/main/java/io/trino/sql/rewrite/ShowQueriesRewrite.java index c0c56b653ad8..3ca6ef720cf1 100644 --- a/core/trino-main/src/main/java/io/trino/sql/rewrite/ShowQueriesRewrite.java +++ b/core/trino-main/src/main/java/io/trino/sql/rewrite/ShowQueriesRewrite.java @@ -479,10 +479,15 @@ protected Node visitShowCreate(ShowCreate node, Void context) Optional viewDefinition = metadata.getMaterializedView(session, objectName); if (viewDefinition.isEmpty()) { + if (metadata.getView(session, objectName).isPresent()) { + throw semanticException(NOT_SUPPORTED, node, "Relation '%s' is a view, not a materialized view", objectName); + } + if (metadata.getTableHandle(session, objectName).isPresent()) { throw semanticException(NOT_SUPPORTED, node, "Relation '%s' is a table, not a materialized view", objectName); } - throw semanticException(TABLE_NOT_FOUND, node, "View '%s' does not exist", objectName); + + throw semanticException(TABLE_NOT_FOUND, node, "Materialized view '%s' does not exist", objectName); } Query query = parseView(viewDefinition.get().getOriginalSql(), objectName, node); @@ -529,9 +534,12 @@ protected Node visitShowCreate(ShowCreate node, Void context) if (node.getType() == TABLE) { QualifiedObjectName objectName = createQualifiedObjectName(session, node, node.getName()); - Optional viewDefinition = metadata.getView(session, objectName); - if (viewDefinition.isPresent()) { + if (metadata.getMaterializedView(session, objectName).isPresent()) { + throw semanticException(NOT_SUPPORTED, node, "Relation '%s' is a materialized view, not a table", objectName); + } + + if (metadata.getView(session, objectName).isPresent()) { throw semanticException(NOT_SUPPORTED, node, "Relation '%s' is a view, not a table", objectName); } diff --git a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java index a07ff795b53a..835bce5e95a5 100644 --- a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java +++ b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java @@ -44,6 +44,7 @@ import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorMaterializedViewDefinition; +import io.trino.spi.connector.ConnectorMaterializedViewDefinition.Column; import io.trino.spi.connector.ConnectorMetadata; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.ConnectorTransactionHandle; @@ -2501,6 +2502,27 @@ public void testShowCreateView() .hasErrorCode(TABLE_NOT_FOUND); } + // This test validates object resolution order (materialized view, view and table). + // The order is arbitrary (connector should not return different object types with same name). + // However, "SHOW CREATE" command should be consistent with how object resolution is performed + // during table scan. + @Test + public void testShowCreateDuplicateNames() + { + analyze("SHOW CREATE MATERIALIZED VIEW table_view_and_materialized_view"); + assertFails("SHOW CREATE VIEW table_view_and_materialized_view") + .hasErrorCode(NOT_SUPPORTED) + .hasMessageContaining("Relation 'tpch.s1.table_view_and_materialized_view' is a materialized view, not a view"); + assertFails("SHOW CREATE TABLE table_view_and_materialized_view") + .hasErrorCode(NOT_SUPPORTED) + .hasMessageContaining("Relation 'tpch.s1.table_view_and_materialized_view' is a materialized view, not a table"); + + analyze("SHOW CREATE VIEW table_and_view"); + assertFails("SHOW CREATE TABLE table_and_view") + .hasErrorCode(NOT_SUPPORTED) + .hasMessageContaining("Relation 'tpch.s1.table_and_view' is a view, not a table"); + } + @Test public void testStaleView() { @@ -3176,6 +3198,56 @@ public void setup() new ConnectorTableMetadata(t5, ImmutableList.of( new ColumnMetadata("b", singleFieldRowType))), false)); + + QualifiedObjectName tableViewAndMaterializedView = new QualifiedObjectName(TPCH_CATALOG, "s1", "table_view_and_materialized_view"); + inSetupTransaction(session -> metadata.createMaterializedView( + session, + tableViewAndMaterializedView, + new ConnectorMaterializedViewDefinition( + "SELECT a FROM t1", + Optional.of("t1"), + Optional.of(TPCH_CATALOG), + Optional.of("s1"), + ImmutableList.of(new Column("a", BIGINT.getTypeId())), + Optional.empty(), + Optional.empty(), + ImmutableMap.of()), + false, + false)); + ConnectorViewDefinition viewDefinition = new ConnectorViewDefinition( + "SELECT a FROM t2", + Optional.of(TPCH_CATALOG), + Optional.of("s1"), + ImmutableList.of(new ViewColumn("a", BIGINT.getTypeId())), + Optional.empty(), + Optional.empty(), + false); + inSetupTransaction(session -> metadata.createView( + session, + tableViewAndMaterializedView, + viewDefinition, + false)); + inSetupTransaction(session -> metadata.createTable( + session, + CATALOG_FOR_IDENTIFIER_CHAIN_TESTS, + new ConnectorTableMetadata( + tableViewAndMaterializedView.asSchemaTableName(), + ImmutableList.of(new ColumnMetadata("a", BIGINT))), + false)); + + QualifiedObjectName tableAndView = new QualifiedObjectName(TPCH_CATALOG, "s1", "table_and_view"); + inSetupTransaction(session -> metadata.createView( + session, + tableAndView, + viewDefinition, + false)); + inSetupTransaction(session -> metadata.createTable( + session, + CATALOG_FOR_IDENTIFIER_CHAIN_TESTS, + new ConnectorTableMetadata( + tableAndView.asSchemaTableName(), + ImmutableList.of(new ColumnMetadata("a", BIGINT))), + false)); } private void inSetupTransaction(Consumer consumer) From 6f5553f9f1f8638cd6e47278c35844dc0245fd3e Mon Sep 17 00:00:00 2001 From: Karol Sobczak Date: Wed, 7 Apr 2021 14:38:17 +0200 Subject: [PATCH 096/146] Test analyzer name resolution order --- .../io/trino/sql/analyzer/TestAnalyzer.java | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java index 835bce5e95a5..eaab00c021be 100644 --- a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java +++ b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java @@ -36,6 +36,7 @@ import io.trino.metadata.Metadata; import io.trino.metadata.QualifiedObjectName; import io.trino.metadata.SessionPropertyManager; +import io.trino.metadata.TableHandle; import io.trino.plugin.base.security.AllowAllSystemAccessControl; import io.trino.security.AccessControl; import io.trino.security.AccessControlConfig; @@ -59,6 +60,7 @@ import io.trino.sql.planner.TypeAnalyzer; import io.trino.sql.tree.Statement; import io.trino.testing.TestingMetadata; +import io.trino.testing.TestingMetadata.TestingTableHandle; import io.trino.testing.assertions.TrinoExceptionAssert; import io.trino.transaction.TransactionManager; import org.intellij.lang.annotations.Language; @@ -69,6 +71,7 @@ import java.util.Optional; import java.util.function.Consumer; +import static com.google.common.collect.Iterables.getOnlyElement; import static io.trino.connector.CatalogName.createInformationSchemaCatalogName; import static io.trino.connector.CatalogName.createSystemTablesCatalogName; import static io.trino.cost.StatsCalculatorModule.createNewStatsCalculator; @@ -147,6 +150,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.nCopies; +import static org.assertj.core.api.Assertions.assertThat; @Test(singleThreaded = true) public class TestAnalyzer @@ -2523,6 +2527,23 @@ public void testShowCreateDuplicateNames() .hasMessageContaining("Relation 'tpch.s1.table_and_view' is a view, not a table"); } + // This test validates object resolution order (materialized view, view and table). + // The order is arbitrary (connector should not return different object types with same name) + // and can be changed along with test. + @Test + public void testAnalysisDuplicateNames() + { + // Materialized view redirects to "t1" + Analysis analysis = analyze("SELECT * FROM table_view_and_materialized_view"); + TableHandle handle = getOnlyElement(analysis.getTables()); + assertThat(((TestingTableHandle) handle.getConnectorHandle()).getTableName().getTableName()).isEqualTo("t1"); + + // View redirects to "t2" + analysis = analyze("SELECT * FROM table_and_view"); + handle = getOnlyElement(analysis.getTables()); + assertThat(((TestingTableHandle) handle.getConnectorHandle()).getTableName().getTableName()).isEqualTo("t2"); + } + @Test public void testStaleView() { @@ -3273,21 +3294,21 @@ private static Analyzer createAnalyzer(Session session, Metadata metadata) createNewStatsCalculator(metadata, new TypeAnalyzer(SQL_PARSER, metadata))); } - private void analyze(@Language("SQL") String query) + private Analysis analyze(@Language("SQL") String query) { - analyze(CLIENT_SESSION, query); + return analyze(CLIENT_SESSION, query); } - private void analyze(Session clientSession, @Language("SQL") String query) + private Analysis analyze(Session clientSession, @Language("SQL") String query) { - transaction(transactionManager, accessControl) + return transaction(transactionManager, accessControl) .singleStatement() .readUncommitted() .execute(clientSession, session -> { Analyzer analyzer = createAnalyzer(session, metadata); Statement statement = SQL_PARSER.createStatement(query, new ParsingOptions( new FeaturesConfig().isParseDecimalLiteralsAsDouble() ? AS_DOUBLE : AS_DECIMAL)); - analyzer.analyze(statement); + return analyzer.analyze(statement); }); } From a842287f63c32616c0c90231e8005e8e1d9ced44 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Fri, 9 Apr 2021 23:20:14 +0200 Subject: [PATCH 097/146] Fix logical merge conflict --- .../src/test/java/io/trino/sql/analyzer/TestAnalyzer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java index eaab00c021be..ea968796431f 100644 --- a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java +++ b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java @@ -3231,7 +3231,7 @@ public void setup() Optional.of("s1"), ImmutableList.of(new Column("a", BIGINT.getTypeId())), Optional.empty(), - Optional.empty(), + "some user", ImmutableMap.of()), false, false)); From b2888b45240992db6c6081205ccde5d540b83182 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Sat, 10 Apr 2021 15:16:16 +0200 Subject: [PATCH 098/146] Add debug listing to troubleshoot flaky build `./mvnw verify presto-server-rpm` is sometime retried for ever on the CI. Let's see what the on-disk state is between retries. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5aeef143e35..666b2f09bb4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: - name: Test Server RPM run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $RETRY ./mvnw verify -B -P ci -pl :trino-server-rpm + $RETRY bash -c './mvnw verify -B -P ci -pl :trino-server-rpm || find core/trino-server-rpm/ -exec ls -ald {} +' - name: Clean Maven Output run: ./mvnw clean -pl '!:trino-server,!:trino-cli' - name: Test Docker Image From 4f8221e9b0ad9059f733c09ab122418b51b3200a Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Sun, 11 Apr 2021 21:58:10 +0200 Subject: [PATCH 099/146] Fix SNAPSHOT ISOLATION detection in SQL Server Previously we were checking the flag set with `ALTER DATABASE ... SET READ_COMMITTED_SNAPSHOT ...` instead of the one set with `ALTER DATABASE ... SET ALLOW_SNAPSHOT_ISOLATION ...`. The two flags are related, but not the same. More explanation at https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/sql/snapshot-isolation-in-sql-server#snapshot-isolation-level-extensions This also simplifies the `TestingSqlServer` setup and improves coverage of snapshot isolation tests. - `TestingSqlServer` still enables snapshot isolation by default, to avoid regular tests being flaky on CI (as in https://github.com/trinodb/trino/issues/6389). The `READ_COMMITTED_SNAPSHOT` is enabled by default as well. - dedicated tests exercise the three remaining states of the `ALLOW_SNAPSHOT_ISOLATION` and `READ_COMMITTED_SNAPSHOT` options. These tests do not leverage `BaseConnectorSmokeTest` as this could re-introduce flakiness on CI. --- .../plugin/sqlserver/SqlServerClient.java | 2 +- ...BaseSqlServerTransactionIsolationTest.java | 89 +++++++++++++++++++ .../TestSqlServerWithSnapshotIsolation.java | 34 +++++++ ...TestSqlServerWithoutSnapshotIsolation.java | 71 +++------------ ...shotIsolationAndReadCommittedSnapshot.java | 34 +++++++ .../plugin/sqlserver/TestingSqlServer.java | 19 ++-- 6 files changed, 176 insertions(+), 73 deletions(-) create mode 100644 plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerTransactionIsolationTest.java create mode 100644 plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerWithSnapshotIsolation.java create mode 100644 plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerWithoutSnapshotIsolationAndReadCommittedSnapshot.java diff --git a/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClient.java b/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClient.java index 08843bf79fa2..4815b7be4b84 100644 --- a/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClient.java +++ b/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClient.java @@ -558,7 +558,7 @@ private boolean hasSnapshotIsolationEnabled(Connection connection) try { return snapshotIsolationEnabled.get(SnapshotIsolationEnabledCacheKey.INSTANCE, () -> { Handle handle = Jdbi.open(connection); - return handle.createQuery("SELECT is_read_committed_snapshot_on FROM sys.databases WHERE name = :name") + return handle.createQuery("SELECT snapshot_isolation_state FROM sys.databases WHERE name = :name") .bind("name", connection.getCatalog()) .mapTo(Boolean.class) .findOne() diff --git a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerTransactionIsolationTest.java b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerTransactionIsolationTest.java new file mode 100644 index 000000000000..09101c426617 --- /dev/null +++ b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerTransactionIsolationTest.java @@ -0,0 +1,89 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.sqlserver; + +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.MaterializedResult; +import io.trino.testing.QueryRunner; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Map; + +import static io.trino.plugin.sqlserver.SqlServerQueryRunner.createSqlServerQueryRunner; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static io.trino.tpch.TpchTable.NATION; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +public abstract class BaseSqlServerTransactionIsolationTest + extends AbstractTestQueryFramework +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + TestingSqlServer sqlServer = closeAfterClass(new TestingSqlServer()); + sqlServer.start(); + configureDatabase(sqlServer); + return createSqlServerQueryRunner( + sqlServer, + Map.of(), + Map.of(), + List.of(NATION)); + } + + protected abstract void configureDatabase(TestingSqlServer sqlServer); + + @Test + public void testCreateReadTable() + { + assertUpdate("CREATE TABLE ctas_read AS SELECT * FROM tpch.tiny.nation", "SELECT count(*) FROM nation"); + assertQuery("SELECT AVG(LENGTH(name)) FROM ctas_read", "SELECT 7.08"); + assertQuery("SELECT SUM(LENGTH(name)) FROM ctas_read WHERE regionkey = 1", "SELECT 38"); + assertUpdate("DROP TABLE ctas_read"); + } + + @Test + public void testDescribeShowTable() + { + assertUpdate("CREATE TABLE ctas_describe AS SELECT regionkey, nationkey, comment FROM tpch.tiny.nation", "SELECT count(*) FROM nation"); + + MaterializedResult expectedColumns = MaterializedResult.resultBuilder(getQueryRunner().getDefaultSession(), VARCHAR, VARCHAR, VARCHAR, VARCHAR) + .row("regionkey", "bigint", "", "") + .row("nationkey", "bigint", "", "") + .row("comment", "varchar(152)", "", "") + .build(); + + MaterializedResult actualColumns = computeActual("DESCRIBE ctas_describe"); + assertThat(actualColumns).isEqualTo(expectedColumns); + + MaterializedResult expectedTables = MaterializedResult.resultBuilder(getQueryRunner().getDefaultSession(), VARCHAR) + .row("ctas_describe") + .build(); + + MaterializedResult actualTables = computeActual("SHOW TABLES LIKE 'ctas_describe'"); + assertThat(actualTables).isEqualTo(expectedTables); + + assertUpdate("DROP TABLE ctas_describe"); + } + + @Test + public void testCreateInsertReadTable() + { + assertUpdate("CREATE TABLE insert_table (col INTEGER)"); + assertUpdate("INSERT INTO insert_table (col) VALUES (1), (2), (3), (4)", 4); + assertQuery("SELECT AVG(col) FROM insert_table", "SELECT 2.5"); + assertUpdate("DROP TABLE insert_table"); + } +} diff --git a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerWithSnapshotIsolation.java b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerWithSnapshotIsolation.java new file mode 100644 index 000000000000..a69f127ac836 --- /dev/null +++ b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerWithSnapshotIsolation.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.sqlserver; + +import static java.lang.String.format; + +public class TestSqlServerWithSnapshotIsolation + extends BaseSqlServerTransactionIsolationTest +{ + @Override + protected void configureDatabase(TestingSqlServer sqlServer) + { + String databaseName = sqlServer.getDatabaseName(); + + // ALLOW_SNAPSHOT_ISOLATION controls whether SNAPSHOT ISOLATION is actually enabled + sqlServer.execute(format("ALTER DATABASE %s SET ALLOW_SNAPSHOT_ISOLATION ON", databaseName)); + + // READ_COMMITTED_SNAPSHOT that READ COMMITTED transaction isolation uses SNAPSHOT ISOLATION by default + // it has no effect when ALLOW_SNAPSHOT_ISOLATION is disabled + // https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/sql/snapshot-isolation-in-sql-server#snapshot-isolation-level-extensions + sqlServer.execute(format("ALTER DATABASE %s SET READ_COMMITTED_SNAPSHOT OFF", databaseName)); + } +} diff --git a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerWithoutSnapshotIsolation.java b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerWithoutSnapshotIsolation.java index c71961eeada5..8da557614a8f 100644 --- a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerWithoutSnapshotIsolation.java +++ b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerWithoutSnapshotIsolation.java @@ -13,73 +13,22 @@ */ package io.trino.plugin.sqlserver; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.trino.testing.AbstractTestQueryFramework; -import io.trino.testing.MaterializedResult; -import io.trino.testing.QueryRunner; -import io.trino.tpch.TpchTable; -import org.testng.annotations.Test; - -import static io.trino.plugin.sqlserver.SqlServerQueryRunner.createSqlServerQueryRunner; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static org.assertj.core.api.Assertions.assertThat; +import static java.lang.String.format; public class TestSqlServerWithoutSnapshotIsolation - extends AbstractTestQueryFramework + extends BaseSqlServerTransactionIsolationTest { @Override - protected QueryRunner createQueryRunner() - throws Exception - { - TestingSqlServer sqlServer = closeAfterClass(new TestingSqlServer(false)); - sqlServer.start(); - return createSqlServerQueryRunner( - sqlServer, - ImmutableMap.of(), - ImmutableMap.of(), - ImmutableList.of(TpchTable.NATION)); - } - - @Test - public void testCreateReadTable() - { - assertUpdate("CREATE TABLE ctas_read AS SELECT * FROM tpch.tiny.nation", "SELECT count(*) FROM nation"); - assertQuery("SELECT AVG(LENGTH(name)) FROM ctas_read", "SELECT 7.08"); - assertQuery("SELECT SUM(LENGTH(name)) FROM ctas_read WHERE regionkey = 1", "SELECT 38"); - assertUpdate("DROP TABLE ctas_read"); - } - - @Test - public void testDescribeShowTable() + protected void configureDatabase(TestingSqlServer sqlServer) { - assertUpdate("CREATE TABLE ctas_describe AS SELECT regionkey, nationkey, comment FROM tpch.tiny.nation", "SELECT count(*) FROM nation"); + String databaseName = sqlServer.getDatabaseName(); - MaterializedResult expectedColumns = MaterializedResult.resultBuilder(getQueryRunner().getDefaultSession(), VARCHAR, VARCHAR, VARCHAR, VARCHAR) - .row("regionkey", "bigint", "", "") - .row("nationkey", "bigint", "", "") - .row("comment", "varchar(152)", "", "") - .build(); + // ALLOW_SNAPSHOT_ISOLATION controls whether SNAPSHOT ISOLATION is actually enabled + sqlServer.execute(format("ALTER DATABASE %s SET ALLOW_SNAPSHOT_ISOLATION OFF", databaseName)); - MaterializedResult actualColumns = computeActual("DESCRIBE ctas_describe"); - assertThat(actualColumns).isEqualTo(expectedColumns); - - MaterializedResult expectedTables = MaterializedResult.resultBuilder(getQueryRunner().getDefaultSession(), VARCHAR) - .row("ctas_describe") - .build(); - - MaterializedResult actualTables = computeActual("SHOW TABLES LIKE 'ctas_describe'"); - assertThat(actualTables).isEqualTo(expectedTables); - - assertUpdate("DROP TABLE ctas_describe"); - } - - @Test - public void testCreateInsertReadTable() - { - assertUpdate("CREATE TABLE insert_table (col INTEGER)"); - assertUpdate("INSERT INTO insert_table (col) VALUES (1), (2), (3), (4)", 4); - assertQuery("SELECT AVG(col) FROM insert_table", "SELECT 2.5"); - assertUpdate("DROP TABLE insert_table"); + // READ_COMMITTED_SNAPSHOT that READ COMMITTED transaction isolation uses SNAPSHOT ISOLATION by default + // it has no effect when ALLOW_SNAPSHOT_ISOLATION is disabled + // https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/sql/snapshot-isolation-in-sql-server#snapshot-isolation-level-extensions + sqlServer.execute(format("ALTER DATABASE %s SET READ_COMMITTED_SNAPSHOT ON", databaseName)); } } diff --git a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerWithoutSnapshotIsolationAndReadCommittedSnapshot.java b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerWithoutSnapshotIsolationAndReadCommittedSnapshot.java new file mode 100644 index 000000000000..72b96ddd4ecc --- /dev/null +++ b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerWithoutSnapshotIsolationAndReadCommittedSnapshot.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.sqlserver; + +import static java.lang.String.format; + +public class TestSqlServerWithoutSnapshotIsolationAndReadCommittedSnapshot + extends BaseSqlServerTransactionIsolationTest +{ + @Override + protected void configureDatabase(TestingSqlServer sqlServer) + { + String databaseName = sqlServer.getDatabaseName(); + + // ALLOW_SNAPSHOT_ISOLATION controls whether SNAPSHOT ISOLATION is actually enabled + sqlServer.execute(format("ALTER DATABASE %s SET ALLOW_SNAPSHOT_ISOLATION OFF", databaseName)); + + // READ_COMMITTED_SNAPSHOT that READ COMMITTED transaction isolation uses SNAPSHOT ISOLATION by default + // it has no effect when ALLOW_SNAPSHOT_ISOLATION is disabled + // https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/sql/snapshot-isolation-in-sql-server#snapshot-isolation-level-extensions + sqlServer.execute(format("ALTER DATABASE %s SET READ_COMMITTED_SNAPSHOT OFF", databaseName)); + } +} diff --git a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestingSqlServer.java b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestingSqlServer.java index a4b6e2611c48..332835b2fd78 100644 --- a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestingSqlServer.java +++ b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestingSqlServer.java @@ -33,23 +33,21 @@ public final class TestingSqlServer private static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse("microsoft/mssql-server-linux:2017-CU13") .asCompatibleSubstituteFor("mcr.microsoft.com/mssql/server:2017-CU12"); private final MSSQLServerContainer container; - private final boolean snapshotIsolationEnabled; private final String databaseName; private Closeable cleanup = () -> {}; public TestingSqlServer() - { - this(true); - } - - public TestingSqlServer(boolean snapshotIsolationEnabled) { container = new MSSQLServerContainer<>(DOCKER_IMAGE_NAME); container.addEnv("ACCEPT_EULA", "yes"); - this.snapshotIsolationEnabled = snapshotIsolationEnabled; this.databaseName = "database_" + UUID.randomUUID().toString().replace("-", ""); } + public String getDatabaseName() + { + return databaseName; + } + public void execute(String sql) { try (Connection connection = container.createConnection(""); @@ -92,10 +90,9 @@ private void setUpDatabase() { execute("CREATE DATABASE " + databaseName); - if (snapshotIsolationEnabled) { - execute(format("ALTER DATABASE %s SET READ_COMMITTED_SNAPSHOT ON", databaseName)); - execute(format("ALTER DATABASE %s SET ALLOW_SNAPSHOT_ISOLATION ON", databaseName)); - } + // Enable snapshot isolation by default to reduce flakiness on CI + execute(format("ALTER DATABASE %s SET ALLOW_SNAPSHOT_ISOLATION ON", databaseName)); + execute(format("ALTER DATABASE %s SET READ_COMMITTED_SNAPSHOT ON", databaseName)); container.withUrlParam("database", this.databaseName); } From 6e9d28b12cb9e0b30087ed0e0294891eef57aca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Kokosi=C5=84ski?= Date: Sat, 10 Apr 2021 20:52:19 +0200 Subject: [PATCH 100/146] Allow to configure eventListener in DistributedQueryRunner --- .../server/testing/TestingTrinoServer.java | 17 ++++++++-- .../trino/testing/DistributedQueryRunner.java | 33 ++++++++++++++++--- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/server/testing/TestingTrinoServer.java b/core/trino-main/src/main/java/io/trino/server/testing/TestingTrinoServer.java index 41bf994aea11..42bb7fb6cd6b 100644 --- a/core/trino-main/src/main/java/io/trino/server/testing/TestingTrinoServer.java +++ b/core/trino-main/src/main/java/io/trino/server/testing/TestingTrinoServer.java @@ -69,6 +69,7 @@ import io.trino.server.security.ServerSecurityModule; import io.trino.spi.Plugin; import io.trino.spi.QueryId; +import io.trino.spi.eventlistener.EventListener; import io.trino.spi.security.GroupProvider; import io.trino.spi.security.SystemAccessControl; import io.trino.split.PageSourceManager; @@ -194,7 +195,8 @@ private TestingTrinoServer( Optional discoveryUri, Module additionalModule, Optional baseDataDir, - List systemAccessControls) + List systemAccessControls, + List eventListeners) { this.coordinator = coordinator; @@ -317,6 +319,9 @@ private TestingTrinoServer( accessControl.setSystemAccessControls(systemAccessControls); + EventListenerManager eventListenerManager = injector.getInstance(EventListenerManager.class); + eventListeners.forEach(eventListenerManager::addEventListener); + announcer.forceAnnounce(); refreshNodes(); @@ -593,6 +598,7 @@ public static class Builder private Module additionalModule = EMPTY_MODULE; private Optional baseDataDir = Optional.empty(); private List systemAccessControls = ImmutableList.of(); + private List eventListeners = ImmutableList.of(); public Builder setCoordinator(boolean coordinator) { @@ -636,6 +642,12 @@ public Builder setSystemAccessControls(List systemAccessCon return this; } + public Builder setEventListeners(List eventListeners) + { + this.eventListeners = ImmutableList.copyOf(requireNonNull(eventListeners, "eventListeners is null")); + return this; + } + public TestingTrinoServer build() { return new TestingTrinoServer( @@ -645,7 +657,8 @@ public TestingTrinoServer build() discoveryUri, additionalModule, baseDataDir, - systemAccessControls); + systemAccessControls, + eventListeners); } } } diff --git a/testing/trino-testing/src/main/java/io/trino/testing/DistributedQueryRunner.java b/testing/trino-testing/src/main/java/io/trino/testing/DistributedQueryRunner.java index b3e3b09c21ba..26a30dccc856 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/DistributedQueryRunner.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/DistributedQueryRunner.java @@ -36,6 +36,7 @@ import io.trino.server.testing.TestingTrinoServer; import io.trino.spi.Plugin; import io.trino.spi.QueryId; +import io.trino.spi.eventlistener.EventListener; import io.trino.spi.security.SystemAccessControl; import io.trino.split.PageSourceManager; import io.trino.split.SplitManager; @@ -99,7 +100,8 @@ private DistributedQueryRunner( String environment, Module additionalModule, Optional baseDataDir, - List systemAccessControls) + List systemAccessControls, + List eventListeners) throws Exception { requireNonNull(defaultSession, "defaultSession is null"); @@ -124,6 +126,7 @@ private DistributedQueryRunner( environment, additionalModule, baseDataDir, + ImmutableList.of(), ImmutableList.of())); servers.add(worker); } @@ -146,7 +149,8 @@ private DistributedQueryRunner( environment, additionalModule, baseDataDir, - systemAccessControls)); + systemAccessControls, + eventListeners)); servers.add(coordinator); if (backupCoordinatorProperties.isPresent()) { Map extraBackupCoordinatorProperties = new HashMap<>(); @@ -159,7 +163,8 @@ private DistributedQueryRunner( environment, additionalModule, baseDataDir, - systemAccessControls))); + systemAccessControls, + eventListeners))); servers.add(backupCoordinator.get()); } else { @@ -197,7 +202,8 @@ private static TestingTrinoServer createTestingTrinoServer( String environment, Module additionalModule, Optional baseDataDir, - List systemAccessControls) + List systemAccessControls, + List eventListeners) { long start = System.nanoTime(); ImmutableMap.Builder propertiesBuilder = ImmutableMap.builder() @@ -232,6 +238,7 @@ private static TestingTrinoServer createTestingTrinoServer( .setAdditionalModule(additionalModule) .setBaseDataDir(baseDataDir) .setSystemAccessControls(systemAccessControls) + .setEventListeners(eventListeners) .build(); String nodeRole = coordinator ? "coordinator" : "worker"; @@ -253,6 +260,7 @@ public void addServers(int nodeCount) ENVIRONMENT, EMPTY_MODULE, Optional.empty(), + ImmutableList.of(), ImmutableList.of())); serverBuilder.add(server); // add functions @@ -564,6 +572,7 @@ public static class Builder private Module additionalModule = EMPTY_MODULE; private Optional baseDataDir = Optional.empty(); private List systemAccessControls = ImmutableList.of(); + private List eventListeners = ImmutableList.of(); protected Builder(Session defaultSession) { @@ -648,6 +657,19 @@ public Builder setSystemAccessControls(List systemAccessCon return this; } + @SuppressWarnings("unused") + public Builder setEventListener(EventListener eventListener) + { + return setEventListeners(ImmutableList.of(requireNonNull(eventListener, "eventListener is null"))); + } + + @SuppressWarnings("unused") + public Builder setEventListeners(List eventListeners) + { + this.eventListeners = ImmutableList.copyOf(requireNonNull(eventListeners, "eventListeners is null")); + return this; + } + public Builder enableBackupCoordinator() { if (backupCoordinatorProperties.isEmpty()) { @@ -668,7 +690,8 @@ public DistributedQueryRunner build() environment, additionalModule, baseDataDir, - systemAccessControls); + systemAccessControls, + eventListeners); } } } From 5619084291da0c37dbf285d03ad53e506c532017 Mon Sep 17 00:00:00 2001 From: "Jacob I. Komissar" Date: Fri, 2 Apr 2021 15:36:29 -0400 Subject: [PATCH 101/146] Merge loops in ParquetPageSourceFactory --- .../parquet/ParquetPageSourceFactory.java | 35 ++++++------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java index b6c08412cbda..cc9d6aa0b000 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java @@ -68,7 +68,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.nullToEmpty; -import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static io.trino.parquet.ParquetTypeUtils.getColumnIO; import static io.trino.parquet.ParquetTypeUtils.getDescriptors; @@ -205,23 +204,18 @@ public static ReaderPageSource createPageSource( requestedSchema = message.orElse(new MessageType(fileSchema.getName(), ImmutableList.of())); messageColumn = getColumnIO(fileSchema, requestedSchema); - ImmutableList.Builder footerBlocks = ImmutableList.builder(); - for (BlockMetaData block : parquetMetadata.getBlocks()) { - long firstDataPage = block.getColumns().get(0).getFirstDataPageOffset(); - if (firstDataPage >= start && firstDataPage < start + length) { - footerBlocks.add(block); - } - } - Map, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, requestedSchema); TupleDomain parquetTupleDomain = options.isIgnoreStatistics() ? TupleDomain.all() : getParquetTupleDomain(descriptorsByPath, effectivePredicate, fileSchema, useColumnNames); Predicate parquetPredicate = buildPredicate(requestedSchema, parquetTupleDomain, descriptorsByPath, timeZone); + ImmutableList.Builder blocks = ImmutableList.builder(); - for (BlockMetaData block : footerBlocks.build()) { - if (predicateMatches(parquetPredicate, block, dataSource, descriptorsByPath, parquetTupleDomain)) { + for (BlockMetaData block : parquetMetadata.getBlocks()) { + long firstDataPage = block.getColumns().get(0).getFirstDataPageOffset(); + if (start <= firstDataPage && firstDataPage < start + length + && predicateMatches(parquetPredicate, block, dataSource, descriptorsByPath, parquetTupleDomain)) { blocks.add(block); } } @@ -270,22 +264,15 @@ public static ReaderPageSource createPageSource( checkArgument(column.getColumnType() == REGULAR, "column type must be REGULAR: %s", column); } - List> parquetFields = baseColumns.stream() - .map(column -> getParquetType(column, fileSchema, useColumnNames)) - .map(Optional::ofNullable) - .collect(toImmutableList()); ImmutableList.Builder trinoTypes = ImmutableList.builder(); ImmutableList.Builder> internalFields = ImmutableList.builder(); - for (int columnIndex = 0; columnIndex < baseColumns.size(); columnIndex++) { - HiveColumnHandle column = baseColumns.get(columnIndex); - Optional parquetField = parquetFields.get(columnIndex); - + for (HiveColumnHandle column : baseColumns) { trinoTypes.add(column.getBaseType()); - - internalFields.add(parquetField.flatMap(field -> { - String columnName = useColumnNames ? column.getBaseColumnName() : fileSchema.getFields().get(column.getBaseHiveColumnIndex()).getName(); - return constructField(column.getBaseType(), lookupColumnByName(messageColumn, columnName)); - })); + internalFields.add(Optional.ofNullable(getParquetType(column, fileSchema, useColumnNames)) + .flatMap(field -> { + String columnName = useColumnNames ? column.getBaseColumnName() : fileSchema.getFields().get(column.getBaseHiveColumnIndex()).getName(); + return constructField(column.getBaseType(), lookupColumnByName(messageColumn, columnName)); + })); } ConnectorPageSource parquetPageSource = new ParquetPageSource(parquetReader, trinoTypes.build(), internalFields.build()); From bd66e4a1968a341766c86457fd16fbd972029d0f Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Mon, 12 Apr 2021 11:40:29 +0200 Subject: [PATCH 102/146] Remove unnecessary parenthesis This is meant to improve readability. --- .../java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java index 320d97bd260e..735be9b3c827 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java @@ -471,7 +471,7 @@ private static ConnectorPageSource createParquetPageSource( List blocks = new ArrayList<>(); for (BlockMetaData block : parquetMetadata.getBlocks()) { long firstDataPage = block.getColumns().get(0).getFirstDataPageOffset(); - if ((firstDataPage >= start) && (firstDataPage < (start + length)) && + if (start <= firstDataPage && firstDataPage < start + length && predicateMatches(parquetPredicate, block, dataSource, descriptorsByPath, parquetTupleDomain)) { blocks.add(block); } From 0dbb2b523a7b2e4c00d395db22bd90b1867d1c55 Mon Sep 17 00:00:00 2001 From: "Jacob I. Komissar" Date: Tue, 6 Apr 2021 13:26:15 -0400 Subject: [PATCH 103/146] Extend Parquet reader system to add an optional row-index column This is neccessary to allow predicate pushdown alongside row-level deletes on Parquet data (such as allowed by the Iceberg v2 draft). Previously, the row-group filter made it impossible for a downstream connector to correctly identify rows after passing the Parquet reader a predicate. --- .../trino/parquet/reader/ParquetReader.java | 25 ++++++- .../hive/parquet/ParquetPageSource.java | 66 +++++++++++++++++-- .../parquet/ParquetPageSourceFactory.java | 46 +++++++++++-- .../iceberg/IcebergPageSourceProvider.java | 1 + 4 files changed, 125 insertions(+), 13 deletions(-) diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java index 5e7b4fe5ad7b..51f403df0277 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java @@ -71,6 +71,7 @@ public class ParquetReader private final Optional fileCreatedBy; private final List blocks; + private final Optional> firstRowsOfBlocks; private final List columns; private final ParquetDataSource dataSource; private final DateTimeZone timeZone; @@ -79,6 +80,13 @@ public class ParquetReader private int currentRowGroup = -1; private BlockMetaData currentBlockMetadata; private long currentGroupRowCount; + /** + * Index in the Parquet file of the first row of the current group + */ + private Optional firstRowIndexInGroup = Optional.empty(); + /** + * Index in the current group of the next row + */ private long nextRowInGroup; private int batchSize; private int nextBatchSize = INITIAL_BATCH_SIZE; @@ -95,6 +103,7 @@ public ParquetReader( Optional fileCreatedBy, MessageColumnIO messageColumnIO, List blocks, + Optional> firstRowsOfBlocks, ParquetDataSource dataSource, DateTimeZone timeZone, AggregatedMemoryContext systemMemoryContext, @@ -104,6 +113,7 @@ public ParquetReader( this.fileCreatedBy = requireNonNull(fileCreatedBy, "fileCreatedBy is null"); this.columns = requireNonNull(messageColumnIO, "messageColumnIO is null").getLeaves(); this.blocks = requireNonNull(blocks, "blocks is null"); + this.firstRowsOfBlocks = requireNonNull(firstRowsOfBlocks, "firstRowsOfBlocks is null"); this.dataSource = requireNonNull(dataSource, "dataSource is null"); this.timeZone = requireNonNull(timeZone, "timeZone is null"); this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); @@ -112,6 +122,10 @@ public ParquetReader( this.columnReaders = new PrimitiveColumnReader[columns.size()]; this.maxBytesPerCell = new long[columns.size()]; + firstRowsOfBlocks.ifPresent(firstRows -> { + checkArgument(blocks.size() == firstRows.size(), "elements of firstRowsOfBlocks must correspond to blocks"); + }); + Map ranges = new HashMap<>(); for (int rowGroup = 0; rowGroup < blocks.size(); rowGroup++) { BlockMetaData metadata = blocks.get(rowGroup); @@ -135,6 +149,15 @@ public void close() dataSource.close(); } + /** + * Get the global row index of the first row in the last batch. + */ + public long lastBatchStartRow() + { + long baseIndex = firstRowIndexInGroup.orElseThrow(() -> new IllegalStateException("row index unavailable")); + return baseIndex + nextRowInGroup - batchSize; + } + public int nextBatch() { if (nextRowInGroup >= currentGroupRowCount && !advanceToNextRowGroup()) { @@ -162,7 +185,7 @@ private boolean advanceToNextRowGroup() return false; } currentBlockMetadata = blocks.get(currentRowGroup); - + firstRowIndexInGroup = firstRowsOfBlocks.map(firstRows -> firstRows.get(currentRowGroup)); nextRowInGroup = 0L; currentGroupRowCount = currentBlockMetadata.getRowCount(); initializeColumnReaders(); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java index c1dddc0571cb..3f21be4c86c5 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSource.java @@ -14,6 +14,7 @@ package io.trino.plugin.hive.parquet; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Streams; import io.trino.parquet.Field; import io.trino.parquet.ParquetCorruptionException; import io.trino.parquet.reader.ParquetReader; @@ -22,6 +23,7 @@ import io.trino.spi.block.Block; import io.trino.spi.block.LazyBlock; import io.trino.spi.block.LazyBlockLoader; +import io.trino.spi.block.LongArrayBlock; import io.trino.spi.block.RunLengthEncodedBlock; import io.trino.spi.connector.ConnectorPageSource; import io.trino.spi.type.Type; @@ -31,10 +33,12 @@ import java.util.List; import java.util.Optional; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static io.trino.plugin.hive.HiveErrorCode.HIVE_BAD_DATA; import static io.trino.plugin.hive.HiveErrorCode.HIVE_CURSOR_ERROR; import static java.lang.String.format; +import static java.util.Collections.nCopies; import static java.util.Objects.requireNonNull; public class ParquetPageSource @@ -43,15 +47,53 @@ public class ParquetPageSource private final ParquetReader parquetReader; private final List types; private final List> fields; + /** + * Indicates whether the column at each index should be populated with the + * indices of its rows + */ + private final List rowIndexLocations; private int batchId; private boolean closed; public ParquetPageSource(ParquetReader parquetReader, List types, List> fields) + { + this(parquetReader, types, nCopies(types.size(), false), fields); + } + + /** + * @param types Column types + * @param rowIndexLocations Whether each column should be populated with the indices of its rows + * @param fields List of field descriptions. Empty optionals will result in columns populated with {@code NULL} + */ + public ParquetPageSource( + ParquetReader parquetReader, + List types, + List rowIndexLocations, + List> fields) { this.parquetReader = requireNonNull(parquetReader, "parquetReader is null"); this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); + this.rowIndexLocations = requireNonNull(rowIndexLocations, "rowIndexLocations is null"); this.fields = ImmutableList.copyOf(requireNonNull(fields, "fields is null")); + + // TODO: Instead of checking that the three list arguments go together correctly, + // we should do something like the ORC reader's ColumnAdatpation, using + // subclasses that contain only the necessary information for each column. + checkArgument( + types.size() == rowIndexLocations.size() && types.size() == fields.size(), + "types, rowIndexLocations, and fields must correspond one-to-one-to-one"); + Streams.forEachPair( + rowIndexLocations.stream(), + fields.stream(), + (isIndexColumn, field) -> checkArgument( + !(isIndexColumn && field.isPresent()), + "Field info for row index column must be empty Optional")); + } + + private boolean isIndexColumn(int column) + { + return rowIndexLocations.get(column); } @Override @@ -91,11 +133,16 @@ public Page getNextPage() } Block[] blocks = new Block[fields.size()]; - for (int fieldId = 0; fieldId < blocks.length; fieldId++) { - Type type = types.get(fieldId); - blocks[fieldId] = fields.get(fieldId) - .map(field -> new LazyBlock(batchSize, new ParquetBlockLoader(field))) - .orElseGet(() -> RunLengthEncodedBlock.create(type, null, batchSize)); + for (int column = 0; column < blocks.length; column++) { + if (isIndexColumn(column)) { + blocks[column] = getRowIndexColumn(parquetReader.lastBatchStartRow(), batchSize); + } + else { + Type type = types.get(column); + blocks[column] = fields.get(column) + .map(field -> new LazyBlock(batchSize, new ParquetBlockLoader(field))) + .orElseGet(() -> RunLengthEncodedBlock.create(type, null, batchSize)); + } } return new Page(batchSize, blocks); } @@ -177,4 +224,13 @@ public final Block load() return block; } } + + private static Block getRowIndexColumn(long baseIndex, int size) + { + long[] rowIndices = new long[size]; + for (int position = 0; position < size; position++) { + rowIndices[position] = baseIndex + position; + } + return new LongArrayBlock(size, Optional.empty(), rowIndices); + } } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java index cc9d6aa0b000..cf312dd95cc9 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java @@ -31,6 +31,7 @@ import io.trino.plugin.hive.HiveColumnHandle; import io.trino.plugin.hive.HiveConfig; import io.trino.plugin.hive.HivePageSourceFactory; +import io.trino.plugin.hive.HiveType; import io.trino.plugin.hive.ReaderColumns; import io.trino.plugin.hive.ReaderPageSource; import io.trino.plugin.hive.acid.AcidTransaction; @@ -86,6 +87,7 @@ import static io.trino.plugin.hive.HiveSessionProperties.isUseParquetColumnNames; import static io.trino.plugin.hive.parquet.ParquetColumnIOConverter.constructField; import static io.trino.plugin.hive.util.HiveUtil.getDeserializerClassName; +import static io.trino.spi.type.BigintType.BIGINT; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toUnmodifiableList; @@ -94,6 +96,20 @@ public class ParquetPageSourceFactory implements HivePageSourceFactory { + /** + * If this object is passed as one of the columns for {@code createPageSource}, + * it will be populated as an additional column containing the index of each + * row read. + */ + public static final HiveColumnHandle PARQUET_ROW_INDEX_COLUMN = new HiveColumnHandle( + "$parquet$row_index", + -1, // no real column index + HiveType.HIVE_LONG, + BIGINT, + Optional.empty(), + HiveColumnHandle.ColumnType.SYNTHESIZED, + Optional.empty()); + private static final Set PARQUET_SERDE_CLASS_NAMES = ImmutableSet.builder() .add("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe") .add("parquet.hive.serde.ParquetHiveSerDe") @@ -211,18 +227,23 @@ public static ReaderPageSource createPageSource( Predicate parquetPredicate = buildPredicate(requestedSchema, parquetTupleDomain, descriptorsByPath, timeZone); + long nextStart = 0; ImmutableList.Builder blocks = ImmutableList.builder(); + ImmutableList.Builder blockStarts = ImmutableList.builder(); for (BlockMetaData block : parquetMetadata.getBlocks()) { long firstDataPage = block.getColumns().get(0).getFirstDataPageOffset(); if (start <= firstDataPage && firstDataPage < start + length && predicateMatches(parquetPredicate, block, dataSource, descriptorsByPath, parquetTupleDomain)) { blocks.add(block); + blockStarts.add(nextStart); } + nextStart += block.getRowCount(); } parquetReader = new ParquetReader( Optional.ofNullable(fileMetaData.getCreatedBy()), messageColumn, blocks.build(), + Optional.of(blockStarts.build()), dataSource, timeZone, newSimpleAggregatedMemoryContext(), @@ -261,21 +282,32 @@ && predicateMatches(parquetPredicate, block, dataSource, descriptorsByPath, parq .orElse(columns); for (HiveColumnHandle column : baseColumns) { - checkArgument(column.getColumnType() == REGULAR, "column type must be REGULAR: %s", column); + checkArgument(column == PARQUET_ROW_INDEX_COLUMN || column.getColumnType() == REGULAR, "column type must be REGULAR: %s", column); } ImmutableList.Builder trinoTypes = ImmutableList.builder(); ImmutableList.Builder> internalFields = ImmutableList.builder(); + ImmutableList.Builder rowIndexColumns = ImmutableList.builder(); for (HiveColumnHandle column : baseColumns) { trinoTypes.add(column.getBaseType()); - internalFields.add(Optional.ofNullable(getParquetType(column, fileSchema, useColumnNames)) - .flatMap(field -> { - String columnName = useColumnNames ? column.getBaseColumnName() : fileSchema.getFields().get(column.getBaseHiveColumnIndex()).getName(); - return constructField(column.getBaseType(), lookupColumnByName(messageColumn, columnName)); - })); + rowIndexColumns.add(column == PARQUET_ROW_INDEX_COLUMN); + if (column == PARQUET_ROW_INDEX_COLUMN) { + internalFields.add(Optional.empty()); + } + else { + internalFields.add(Optional.ofNullable(getParquetType(column, fileSchema, useColumnNames)) + .flatMap(field -> { + String columnName = useColumnNames ? column.getBaseColumnName() : fileSchema.getFields().get(column.getBaseHiveColumnIndex()).getName(); + return constructField(column.getBaseType(), lookupColumnByName(messageColumn, columnName)); + })); + } } - ConnectorPageSource parquetPageSource = new ParquetPageSource(parquetReader, trinoTypes.build(), internalFields.build()); + ConnectorPageSource parquetPageSource = new ParquetPageSource( + parquetReader, + trinoTypes.build(), + rowIndexColumns.build(), + internalFields.build()); return new ReaderPageSource(parquetPageSource, readerProjections); } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java index 735be9b3c827..992761d2a847 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java @@ -482,6 +482,7 @@ private static ConnectorPageSource createParquetPageSource( Optional.ofNullable(fileMetaData.getCreatedBy()), messageColumnIO, blocks, + Optional.empty(), dataSource, UTC, systemMemoryContext, From 692d0fd532ddca73632ed5860db5aae43861433d Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Mon, 12 Apr 2021 23:10:34 +0200 Subject: [PATCH 104/146] Add kill switch for SQL Server transaction isolation --- plugin/trino-sqlserver/pom.xml | 5 ++ .../plugin/sqlserver/SqlServerClient.java | 10 +++- .../sqlserver/SqlServerClientModule.java | 2 + .../plugin/sqlserver/SqlServerConfig.java | 35 ++++++++++++++ .../plugin/sqlserver/TestSqlServerClient.java | 1 + .../plugin/sqlserver/TestSqlServerConfig.java | 46 +++++++++++++++++++ 6 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerConfig.java create mode 100644 plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerConfig.java diff --git a/plugin/trino-sqlserver/pom.xml b/plugin/trino-sqlserver/pom.xml index 1cc12b60908c..11aafa8d30cc 100644 --- a/plugin/trino-sqlserver/pom.xml +++ b/plugin/trino-sqlserver/pom.xml @@ -28,6 +28,11 @@ trino-matching + + io.airlift + configuration + + io.airlift log diff --git a/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClient.java b/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClient.java index 4815b7be4b84..32e1744d1549 100644 --- a/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClient.java +++ b/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClient.java @@ -130,6 +130,7 @@ import static java.lang.String.join; import static java.math.RoundingMode.UNNECESSARY; import static java.time.Duration.ofMinutes; +import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; public class SqlServerClient @@ -140,6 +141,7 @@ public class SqlServerClient private static final Joiner DOT_JOINER = Joiner.on("."); + private final boolean snapshotIsolationDisabled; private final Cache snapshotIsolationEnabled = CacheBuilder.newBuilder() .maximumSize(1) .expireAfterWrite(ofMinutes(5)) @@ -150,10 +152,13 @@ public class SqlServerClient private static final int MAX_SUPPORTED_TEMPORAL_PRECISION = 7; @Inject - public SqlServerClient(BaseJdbcConfig config, ConnectionFactory connectionFactory) + public SqlServerClient(BaseJdbcConfig config, SqlServerConfig sqlServerConfig, ConnectionFactory connectionFactory) { super(config, "\"", connectionFactory); + requireNonNull(sqlServerConfig, "sqlServerConfig is null"); + snapshotIsolationDisabled = sqlServerConfig.isSnapshotIsolationDisabled(); + JdbcTypeHandle bigintTypeHandle = new JdbcTypeHandle(Types.BIGINT, Optional.of("bigint"), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); this.aggregateFunctionRewriter = new AggregateFunctionRewriter( this::quoted, @@ -538,6 +543,9 @@ public Connection getConnection(ConnectorSession session, JdbcSplit split) private Connection configureConnectionTransactionIsolation(Connection connection) throws SQLException { + if (snapshotIsolationDisabled) { + return connection; + } try { if (hasSnapshotIsolationEnabled(connection)) { // SQL Server's READ COMMITTED + SNAPSHOT ISOLATION is equivalent to ordinary READ COMMITTED in e.g. Oracle, PostgreSQL. diff --git a/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClientModule.java b/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClientModule.java index 6e6e39231fe0..7e2791ff77cd 100644 --- a/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClientModule.java +++ b/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClientModule.java @@ -29,6 +29,7 @@ import io.trino.plugin.jdbc.credential.CredentialProvider; import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; +import static io.airlift.configuration.ConfigBinder.configBinder; import static io.trino.plugin.jdbc.JdbcModule.bindTablePropertiesProvider; import static io.trino.plugin.sqlserver.SqlServerClient.SQL_SERVER_MAX_LIST_EXPRESSIONS; @@ -38,6 +39,7 @@ public class SqlServerClientModule @Override public void configure(Binder binder) { + configBinder(binder).bindConfig(SqlServerConfig.class); binder.bind(JdbcClient.class).annotatedWith(ForBaseJdbc.class).to(SqlServerClient.class).in(Scopes.SINGLETON); bindTablePropertiesProvider(binder, SqlServerTableProperties.class); newOptionalBinder(binder, Key.get(int.class, MaxDomainCompactionThreshold.class)).setBinding().toInstance(SQL_SERVER_MAX_LIST_EXPRESSIONS); diff --git a/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerConfig.java b/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerConfig.java new file mode 100644 index 000000000000..340de7e01b78 --- /dev/null +++ b/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerConfig.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.sqlserver; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; + +public class SqlServerConfig +{ + private boolean snapshotIsolationDisabled; + + public boolean isSnapshotIsolationDisabled() + { + return snapshotIsolationDisabled; + } + + @Config("sqlserver.snapshot-isolation.disabled") + @ConfigDescription("Disables automatic use of snapshot isolation for transactions issued by Trino in SQL Server") + public SqlServerConfig setSnapshotIsolationDisabled(boolean snapshotIsolationDisabled) + { + this.snapshotIsolationDisabled = snapshotIsolationDisabled; + return this; + } +} diff --git a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerClient.java b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerClient.java index c7f0bb951ebf..c5519986fc9e 100644 --- a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerClient.java +++ b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerClient.java @@ -56,6 +56,7 @@ public class TestSqlServerClient private static final JdbcClient JDBC_CLIENT = new SqlServerClient( new BaseJdbcConfig(), + new SqlServerConfig(), session -> { throw new UnsupportedOperationException(); }); diff --git a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerConfig.java b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerConfig.java new file mode 100644 index 000000000000..c2e5b1750613 --- /dev/null +++ b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestSqlServerConfig.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.sqlserver; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestSqlServerConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(SqlServerConfig.class) + .setSnapshotIsolationDisabled(false)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("sqlserver.snapshot-isolation.disabled", "true") + .build(); + + SqlServerConfig expected = new SqlServerConfig() + .setSnapshotIsolationDisabled(true); + + assertFullMapping(properties, expected); + } +} From 41a36d43195a9d15c2d2eec26fa391718f3b5087 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Mon, 12 Apr 2021 23:43:47 +0200 Subject: [PATCH 105/146] Remove bogus option for unsupported-type-handling from docs --- docs/src/main/sphinx/connector/oracle.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/src/main/sphinx/connector/oracle.rst b/docs/src/main/sphinx/connector/oracle.rst index 5481eb9fa1a3..c20c1c59685e 100644 --- a/docs/src/main/sphinx/connector/oracle.rst +++ b/docs/src/main/sphinx/connector/oracle.rst @@ -163,8 +163,6 @@ If an Oracle table uses a type not listed in the above table, then you can use t ``unsupported-type-handling`` configuration property to specify Trino behavior. For example: -- If ``unsupported-type-handling`` is set to ``FAIL``, then the - querying of an unsupported table fails. - If ``unsupported-type-handling`` is set to ``IGNORE``, then you can't see the unsupported types in Trino. - If ``unsupported-type-handling`` is set to ``CONVERT_TO_VARCHAR``, From cc4db4164d2cb3304f5896182094360b43f6d4eb Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Fri, 9 Apr 2021 15:31:12 +0200 Subject: [PATCH 106/146] Add OrcWriterOptions builder --- .../java/io/trino/orc/OrcWriterOptions.java | 145 ++++++++++++++++-- 1 file changed, 135 insertions(+), 10 deletions(-) diff --git a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java index b1325936a483..3b2cc8b1a3d1 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java @@ -31,12 +31,13 @@ public class OrcWriterOptions static final DataSize DEFAULT_MAX_STRING_STATISTICS_LIMIT = DataSize.ofBytes(64); @VisibleForTesting static final DataSize DEFAULT_MAX_COMPRESSION_BUFFER_SIZE = DataSize.of(256, KILOBYTE); - static final double DEFAULT_BLOOM_FILTER_FPP = 0.05; + private static final double DEFAULT_BLOOM_FILTER_FPP = 0.05; private static final DataSize DEFAULT_STRIPE_MIN_SIZE = DataSize.of(32, MEGABYTE); private static final DataSize DEFAULT_STRIPE_MAX_SIZE = DataSize.of(64, MEGABYTE); private static final int DEFAULT_STRIPE_MAX_ROW_COUNT = 10_000_000; private static final int DEFAULT_ROW_GROUP_MAX_ROW_COUNT = 10_000; private static final DataSize DEFAULT_DICTIONARY_MAX_MEMORY = DataSize.of(16, MEGABYTE); + private final DataSize stripeMinSize; private final DataSize stripeMaxSize; private final int stripeMaxRowCount; @@ -140,47 +141,65 @@ public boolean isBloomFilterColumn(String columnName) public OrcWriterOptions withStripeMinSize(DataSize stripeMinSize) { - return new OrcWriterOptions(stripeMinSize, stripeMaxSize, stripeMaxRowCount, rowGroupMaxRowCount, dictionaryMaxMemory, maxStringStatisticsLimit, maxCompressionBufferSize, bloomFilterColumns, bloomFilterFpp); + return builderFrom(this) + .setStripeMinSize(stripeMinSize) + .build(); } public OrcWriterOptions withStripeMaxSize(DataSize stripeMaxSize) { - return new OrcWriterOptions(stripeMinSize, stripeMaxSize, stripeMaxRowCount, rowGroupMaxRowCount, dictionaryMaxMemory, maxStringStatisticsLimit, maxCompressionBufferSize, bloomFilterColumns, bloomFilterFpp); + return builderFrom(this) + .setStripeMaxSize(stripeMaxSize) + .build(); } public OrcWriterOptions withStripeMaxRowCount(int stripeMaxRowCount) { - return new OrcWriterOptions(stripeMinSize, stripeMaxSize, stripeMaxRowCount, rowGroupMaxRowCount, dictionaryMaxMemory, maxStringStatisticsLimit, maxCompressionBufferSize, bloomFilterColumns, bloomFilterFpp); + return builderFrom(this) + .setStripeMaxRowCount(stripeMaxRowCount) + .build(); } public OrcWriterOptions withRowGroupMaxRowCount(int rowGroupMaxRowCount) { - return new OrcWriterOptions(stripeMinSize, stripeMaxSize, stripeMaxRowCount, rowGroupMaxRowCount, dictionaryMaxMemory, maxStringStatisticsLimit, maxCompressionBufferSize, bloomFilterColumns, bloomFilterFpp); + return builderFrom(this) + .setRowGroupMaxRowCount(rowGroupMaxRowCount) + .build(); } public OrcWriterOptions withDictionaryMaxMemory(DataSize dictionaryMaxMemory) { - return new OrcWriterOptions(stripeMinSize, stripeMaxSize, stripeMaxRowCount, rowGroupMaxRowCount, dictionaryMaxMemory, maxStringStatisticsLimit, maxCompressionBufferSize, bloomFilterColumns, bloomFilterFpp); + return builderFrom(this) + .setDictionaryMaxMemory(dictionaryMaxMemory) + .build(); } public OrcWriterOptions withMaxStringStatisticsLimit(DataSize maxStringStatisticsLimit) { - return new OrcWriterOptions(stripeMinSize, stripeMaxSize, stripeMaxRowCount, rowGroupMaxRowCount, dictionaryMaxMemory, maxStringStatisticsLimit, maxCompressionBufferSize, bloomFilterColumns, bloomFilterFpp); + return builderFrom(this) + .setMaxStringStatisticsLimit(maxStringStatisticsLimit) + .build(); } public OrcWriterOptions withMaxCompressionBufferSize(DataSize maxCompressionBufferSize) { - return new OrcWriterOptions(stripeMinSize, stripeMaxSize, stripeMaxRowCount, rowGroupMaxRowCount, dictionaryMaxMemory, maxStringStatisticsLimit, maxCompressionBufferSize, bloomFilterColumns, bloomFilterFpp); + return builderFrom(this) + .setMaxCompressionBufferSize(maxCompressionBufferSize) + .build(); } public OrcWriterOptions withBloomFilterColumns(Set bloomFilterColumns) { - return new OrcWriterOptions(stripeMinSize, stripeMaxSize, stripeMaxRowCount, rowGroupMaxRowCount, dictionaryMaxMemory, maxStringStatisticsLimit, maxCompressionBufferSize, bloomFilterColumns, bloomFilterFpp); + return builderFrom(this) + .setBloomFilterColumns(bloomFilterColumns) + .build(); } public OrcWriterOptions withBloomFilterFpp(double bloomFilterFpp) { - return new OrcWriterOptions(stripeMinSize, stripeMaxSize, stripeMaxRowCount, rowGroupMaxRowCount, dictionaryMaxMemory, maxStringStatisticsLimit, maxCompressionBufferSize, bloomFilterColumns, bloomFilterFpp); + return builderFrom(this) + .setBloomFilterFpp(bloomFilterFpp) + .build(); } @Override @@ -198,4 +217,110 @@ public String toString() .add("bloomFilterFpp", bloomFilterFpp) .toString(); } + + public static Builder builder() + { + return builderFrom(new OrcWriterOptions()); + } + + public static Builder builderFrom(OrcWriterOptions options) + { + return new Builder(options); + } + + public static final class Builder + { + private DataSize stripeMinSize; + private DataSize stripeMaxSize; + private int stripeMaxRowCount; + private int rowGroupMaxRowCount; + private DataSize dictionaryMaxMemory; + private DataSize maxStringStatisticsLimit; + private DataSize maxCompressionBufferSize; + private Set bloomFilterColumns; + private double bloomFilterFpp; + + private Builder(OrcWriterOptions options) + { + requireNonNull(options, "options is null"); + + this.stripeMinSize = options.stripeMinSize; + this.stripeMaxSize = options.stripeMaxSize; + this.stripeMaxRowCount = options.stripeMaxRowCount; + this.rowGroupMaxRowCount = options.rowGroupMaxRowCount; + this.dictionaryMaxMemory = options.dictionaryMaxMemory; + this.maxStringStatisticsLimit = options.maxStringStatisticsLimit; + this.maxCompressionBufferSize = options.maxCompressionBufferSize; + this.bloomFilterColumns = ImmutableSet.copyOf(options.bloomFilterColumns); + this.bloomFilterFpp = options.bloomFilterFpp; + } + + public Builder setStripeMinSize(DataSize stripeMinSize) + { + this.stripeMinSize = stripeMinSize; + return this; + } + + public Builder setStripeMaxSize(DataSize stripeMaxSize) + { + this.stripeMaxSize = stripeMaxSize; + return this; + } + + public Builder setStripeMaxRowCount(int stripeMaxRowCount) + { + this.stripeMaxRowCount = stripeMaxRowCount; + return this; + } + + public Builder setRowGroupMaxRowCount(int rowGroupMaxRowCount) + { + this.rowGroupMaxRowCount = rowGroupMaxRowCount; + return this; + } + + public Builder setDictionaryMaxMemory(DataSize dictionaryMaxMemory) + { + this.dictionaryMaxMemory = dictionaryMaxMemory; + return this; + } + + public Builder setMaxStringStatisticsLimit(DataSize maxStringStatisticsLimit) + { + this.maxStringStatisticsLimit = maxStringStatisticsLimit; + return this; + } + + public Builder setMaxCompressionBufferSize(DataSize maxCompressionBufferSize) + { + this.maxCompressionBufferSize = maxCompressionBufferSize; + return this; + } + + public Builder setBloomFilterColumns(Set bloomFilterColumns) + { + this.bloomFilterColumns = bloomFilterColumns; + return this; + } + + public Builder setBloomFilterFpp(double bloomFilterFpp) + { + this.bloomFilterFpp = bloomFilterFpp; + return this; + } + + public OrcWriterOptions build() + { + return new OrcWriterOptions( + stripeMinSize, + stripeMaxSize, + stripeMaxRowCount, + rowGroupMaxRowCount, + dictionaryMaxMemory, + maxStringStatisticsLimit, + maxCompressionBufferSize, + bloomFilterColumns, + bloomFilterFpp); + } + } } From 4407c995ec57bf8babdf94006b8e45b2adf7f8ff Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Fri, 9 Apr 2021 15:35:20 +0200 Subject: [PATCH 107/146] Reorder OrcWriterOptions with* methods Place each one next to corresponding getter. --- .../java/io/trino/orc/OrcWriterOptions.java | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java index 3b2cc8b1a3d1..97a7b5658929 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java @@ -99,46 +99,6 @@ public DataSize getStripeMinSize() return stripeMinSize; } - public DataSize getStripeMaxSize() - { - return stripeMaxSize; - } - - public int getStripeMaxRowCount() - { - return stripeMaxRowCount; - } - - public int getRowGroupMaxRowCount() - { - return rowGroupMaxRowCount; - } - - public DataSize getDictionaryMaxMemory() - { - return dictionaryMaxMemory; - } - - public DataSize getMaxStringStatisticsLimit() - { - return maxStringStatisticsLimit; - } - - public DataSize getMaxCompressionBufferSize() - { - return maxCompressionBufferSize; - } - - public double getBloomFilterFpp() - { - return bloomFilterFpp; - } - - public boolean isBloomFilterColumn(String columnName) - { - return bloomFilterColumns.contains(columnName); - } - public OrcWriterOptions withStripeMinSize(DataSize stripeMinSize) { return builderFrom(this) @@ -146,6 +106,11 @@ public OrcWriterOptions withStripeMinSize(DataSize stripeMinSize) .build(); } + public DataSize getStripeMaxSize() + { + return stripeMaxSize; + } + public OrcWriterOptions withStripeMaxSize(DataSize stripeMaxSize) { return builderFrom(this) @@ -153,6 +118,11 @@ public OrcWriterOptions withStripeMaxSize(DataSize stripeMaxSize) .build(); } + public int getStripeMaxRowCount() + { + return stripeMaxRowCount; + } + public OrcWriterOptions withStripeMaxRowCount(int stripeMaxRowCount) { return builderFrom(this) @@ -160,6 +130,11 @@ public OrcWriterOptions withStripeMaxRowCount(int stripeMaxRowCount) .build(); } + public int getRowGroupMaxRowCount() + { + return rowGroupMaxRowCount; + } + public OrcWriterOptions withRowGroupMaxRowCount(int rowGroupMaxRowCount) { return builderFrom(this) @@ -167,6 +142,11 @@ public OrcWriterOptions withRowGroupMaxRowCount(int rowGroupMaxRowCount) .build(); } + public DataSize getDictionaryMaxMemory() + { + return dictionaryMaxMemory; + } + public OrcWriterOptions withDictionaryMaxMemory(DataSize dictionaryMaxMemory) { return builderFrom(this) @@ -174,6 +154,11 @@ public OrcWriterOptions withDictionaryMaxMemory(DataSize dictionaryMaxMemory) .build(); } + public DataSize getMaxStringStatisticsLimit() + { + return maxStringStatisticsLimit; + } + public OrcWriterOptions withMaxStringStatisticsLimit(DataSize maxStringStatisticsLimit) { return builderFrom(this) @@ -181,6 +166,11 @@ public OrcWriterOptions withMaxStringStatisticsLimit(DataSize maxStringStatistic .build(); } + public DataSize getMaxCompressionBufferSize() + { + return maxCompressionBufferSize; + } + public OrcWriterOptions withMaxCompressionBufferSize(DataSize maxCompressionBufferSize) { return builderFrom(this) @@ -188,6 +178,11 @@ public OrcWriterOptions withMaxCompressionBufferSize(DataSize maxCompressionBuff .build(); } + public boolean isBloomFilterColumn(String columnName) + { + return bloomFilterColumns.contains(columnName); + } + public OrcWriterOptions withBloomFilterColumns(Set bloomFilterColumns) { return builderFrom(this) @@ -195,6 +190,11 @@ public OrcWriterOptions withBloomFilterColumns(Set bloomFilterColumns) .build(); } + public double getBloomFilterFpp() + { + return bloomFilterFpp; + } + public OrcWriterOptions withBloomFilterFpp(double bloomFilterFpp) { return builderFrom(this) From 70982336bd5fcf0a5ba64aaa3c5dd6e34d8e6881 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Fri, 9 Apr 2021 15:41:46 +0200 Subject: [PATCH 108/146] Move useLegacyVersion to OrcWriterOptions --- .../src/main/java/io/trino/orc/OrcWriter.java | 3 +-- .../java/io/trino/orc/OrcWriterOptions.java | 25 +++++++++++++++++++ .../src/test/java/io/trino/orc/OrcTester.java | 2 -- .../test/java/io/trino/orc/TestOrcWriter.java | 1 - .../io/trino/orc/TestStructColumnReader.java | 1 - .../trino/plugin/hive/orc/OrcFileWriter.java | 2 -- .../plugin/hive/orc/OrcFileWriterFactory.java | 6 ----- .../plugin/hive/orc/OrcWriterConfig.java | 5 ++-- .../plugin/hive/util/TempFileWriter.java | 1 - .../io/trino/plugin/hive/HiveTestUtils.java | 1 - .../plugin/hive/TestHiveFileFormats.java | 2 +- .../plugin/hive/benchmark/FileFormat.java | 1 - .../plugin/hive/orc/TestOrcPredicates.java | 2 +- .../iceberg/IcebergFileWriterFactory.java | 1 - .../plugin/iceberg/IcebergOrcFileWriter.java | 3 +-- 15 files changed, 31 insertions(+), 25 deletions(-) diff --git a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java index 92cc86611172..f539fbd33784 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java @@ -135,7 +135,6 @@ public OrcWriter( ColumnMetadata orcTypes, CompressionKind compression, OrcWriterOptions options, - boolean writeLegacyVersion, Map userMetadata, boolean validate, OrcWriteValidationMode validationMode, @@ -162,7 +161,7 @@ public OrcWriter( this.userMetadata.putAll(requireNonNull(userMetadata, "userMetadata is null")); this.userMetadata.put(PRESTO_ORC_WRITER_VERSION_METADATA_KEY, PRESTO_ORC_WRITER_VERSION); - this.metadataWriter = new CompressedMetadataWriter(new OrcMetadataWriter(writeLegacyVersion), compression, maxCompressionBufferSize); + this.metadataWriter = new CompressedMetadataWriter(new OrcMetadataWriter(options.isUseLegacyVersion()), compression, maxCompressionBufferSize); this.stats = requireNonNull(stats, "stats is null"); requireNonNull(columnNames, "columnNames is null"); diff --git a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java index 97a7b5658929..4abb7726b6f4 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java @@ -38,6 +38,7 @@ public class OrcWriterOptions private static final int DEFAULT_ROW_GROUP_MAX_ROW_COUNT = 10_000; private static final DataSize DEFAULT_DICTIONARY_MAX_MEMORY = DataSize.of(16, MEGABYTE); + private final boolean useLegacyVersion; private final DataSize stripeMinSize; private final DataSize stripeMaxSize; private final int stripeMaxRowCount; @@ -51,6 +52,7 @@ public class OrcWriterOptions public OrcWriterOptions() { this( + false, DEFAULT_STRIPE_MIN_SIZE, DEFAULT_STRIPE_MAX_SIZE, DEFAULT_STRIPE_MAX_ROW_COUNT, @@ -63,6 +65,7 @@ public OrcWriterOptions() } private OrcWriterOptions( + boolean useLegacyVersion, DataSize stripeMinSize, DataSize stripeMaxSize, int stripeMaxRowCount, @@ -83,6 +86,7 @@ private OrcWriterOptions( requireNonNull(bloomFilterColumns, "bloomFilterColumns is null"); checkArgument(bloomFilterFpp > 0.0 && bloomFilterFpp < 1.0, "bloomFilterFpp should be > 0.0 & < 1.0"); + this.useLegacyVersion = useLegacyVersion; this.stripeMinSize = stripeMinSize; this.stripeMaxSize = stripeMaxSize; this.stripeMaxRowCount = stripeMaxRowCount; @@ -94,6 +98,18 @@ private OrcWriterOptions( this.bloomFilterFpp = bloomFilterFpp; } + public boolean isUseLegacyVersion() + { + return useLegacyVersion; + } + + public OrcWriterOptions withUseLegacyVersion(boolean useLegacyVersion) + { + return builderFrom(this) + .setUseLegacyVersion(useLegacyVersion) + .build(); + } + public DataSize getStripeMinSize() { return stripeMinSize; @@ -230,6 +246,7 @@ public static Builder builderFrom(OrcWriterOptions options) public static final class Builder { + private boolean useLegacyVersion; private DataSize stripeMinSize; private DataSize stripeMaxSize; private int stripeMaxRowCount; @@ -244,6 +261,7 @@ private Builder(OrcWriterOptions options) { requireNonNull(options, "options is null"); + this.useLegacyVersion = options.useLegacyVersion; this.stripeMinSize = options.stripeMinSize; this.stripeMaxSize = options.stripeMaxSize; this.stripeMaxRowCount = options.stripeMaxRowCount; @@ -255,6 +273,12 @@ private Builder(OrcWriterOptions options) this.bloomFilterFpp = options.bloomFilterFpp; } + public Builder setUseLegacyVersion(boolean useLegacyVersion) + { + this.useLegacyVersion = useLegacyVersion; + return this; + } + public Builder setStripeMinSize(DataSize stripeMinSize) { this.stripeMinSize = stripeMinSize; @@ -312,6 +336,7 @@ public Builder setBloomFilterFpp(double bloomFilterFpp) public OrcWriterOptions build() { return new OrcWriterOptions( + useLegacyVersion, stripeMinSize, stripeMaxSize, stripeMaxRowCount, diff --git a/lib/trino-orc/src/test/java/io/trino/orc/OrcTester.java b/lib/trino-orc/src/test/java/io/trino/orc/OrcTester.java index d581ee503afb..7d9c65d246ee 100644 --- a/lib/trino-orc/src/test/java/io/trino/orc/OrcTester.java +++ b/lib/trino-orc/src/test/java/io/trino/orc/OrcTester.java @@ -618,7 +618,6 @@ public static void writeOrcPages(File outputFile, CompressionKind compression, L OrcType.createRootOrcType(columnNames, types), compression, new OrcWriterOptions(), - false, ImmutableMap.of(), true, BOTH, @@ -645,7 +644,6 @@ public static void writeOrcColumnTrino(File outputFile, CompressionKind compress OrcType.createRootOrcType(columnNames, types), compression, new OrcWriterOptions(), - false, ImmutableMap.of(), true, BOTH, diff --git a/lib/trino-orc/src/test/java/io/trino/orc/TestOrcWriter.java b/lib/trino-orc/src/test/java/io/trino/orc/TestOrcWriter.java index 0c3c0c16cf39..044b167e9faa 100644 --- a/lib/trino-orc/src/test/java/io/trino/orc/TestOrcWriter.java +++ b/lib/trino-orc/src/test/java/io/trino/orc/TestOrcWriter.java @@ -79,7 +79,6 @@ public void testWriteOutputStreamsInOrder() .withRowGroupMaxRowCount(ORC_ROW_GROUP_SIZE) .withDictionaryMaxMemory(DataSize.of(32, MEGABYTE)) .withBloomFilterColumns(ImmutableSet.copyOf(columnNames)), - false, ImmutableMap.of(), true, validationMode, diff --git a/lib/trino-orc/src/test/java/io/trino/orc/TestStructColumnReader.java b/lib/trino-orc/src/test/java/io/trino/orc/TestStructColumnReader.java index bb5f135c1416..fa7552c65e0b 100644 --- a/lib/trino-orc/src/test/java/io/trino/orc/TestStructColumnReader.java +++ b/lib/trino-orc/src/test/java/io/trino/orc/TestStructColumnReader.java @@ -228,7 +228,6 @@ private void write(TempFile tempFile, Type writerType, List data) .withStripeMaxRowCount(ORC_STRIPE_SIZE) .withRowGroupMaxRowCount(ORC_ROW_GROUP_SIZE) .withDictionaryMaxMemory(DataSize.of(32, MEGABYTE)), - false, ImmutableMap.of(), true, BOTH, diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriter.java index ef95c83f7310..53afa9d1eb10 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriter.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriter.java @@ -91,7 +91,6 @@ public OrcFileWriter( ColumnMetadata fileColumnOrcTypes, CompressionKind compression, OrcWriterOptions options, - boolean writeLegacyVersion, int[] fileInputColumnIndexes, Map metadata, Optional> validationInputFactory, @@ -123,7 +122,6 @@ public OrcFileWriter( fileColumnOrcTypes, compression, options, - writeLegacyVersion, metadata, validationInputFactory.isPresent(), validationMode, diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriterFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriterFactory.java index 79ec9163ed8f..43c6d3eb4461 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriterFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriterFactory.java @@ -85,14 +85,12 @@ public class OrcFileWriterFactory private final FileFormatDataSourceStats readStats; private final OrcWriterStats stats = new OrcWriterStats(); private final OrcWriterOptions orcWriterOptions; - private final boolean writeLegacyVersion; @Inject public OrcFileWriterFactory( HdfsEnvironment hdfsEnvironment, TypeManager typeManager, NodeVersion nodeVersion, - OrcWriterConfig orcWriterConfig, FileFormatDataSourceStats readStats, OrcWriterConfig config) { @@ -100,7 +98,6 @@ public OrcFileWriterFactory( hdfsEnvironment, typeManager, nodeVersion, - requireNonNull(orcWriterConfig, "orcWriterConfig is null").isUseLegacyVersion(), readStats, requireNonNull(config, "config is null").toOrcWriterOptions()); } @@ -109,14 +106,12 @@ public OrcFileWriterFactory( HdfsEnvironment hdfsEnvironment, TypeManager typeManager, NodeVersion nodeVersion, - boolean writeLegacyVersion, FileFormatDataSourceStats readStats, OrcWriterOptions orcWriterOptions) { this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.nodeVersion = requireNonNull(nodeVersion, "nodeVersion is null"); - this.writeLegacyVersion = writeLegacyVersion; this.readStats = requireNonNull(readStats, "readStats is null"); this.orcWriterOptions = requireNonNull(orcWriterOptions, "orcWriterOptions is null"); } @@ -215,7 +210,6 @@ public Optional createFileWriter( .withStripeMaxRowCount(getOrcOptimizedWriterMaxStripeRows(session)) .withDictionaryMaxMemory(getOrcOptimizedWriterMaxDictionaryMemory(session)) .withMaxStringStatisticsLimit(getOrcStringStatisticsLimit(session)), - writeLegacyVersion, fileInputColumnIndexes, ImmutableMap.builder() .put(PRESTO_VERSION_NAME, nodeVersion.toString()) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcWriterConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcWriterConfig.java index 4c3a5d2525e6..57d8e82bc4b6 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcWriterConfig.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcWriterConfig.java @@ -31,7 +31,6 @@ public class OrcWriterConfig private OrcWriterOptions options = new OrcWriterOptions(); private double defaultBloomFilterFpp = 0.05; - private boolean useLegacyVersion; private double validationPercentage; private OrcWriteValidationMode validationMode = OrcWriteValidationMode.BOTH; @@ -139,14 +138,14 @@ public OrcWriterConfig setDefaultBloomFilterFpp(double defaultBloomFilterFpp) public boolean isUseLegacyVersion() { - return useLegacyVersion; + return options.isUseLegacyVersion(); } @Config("hive.orc.writer.use-legacy-version-number") @ConfigDescription("Write ORC files with a version number that is readable by Hive 2.0.0 to 2.2.0") public OrcWriterConfig setUseLegacyVersion(boolean useLegacyVersion) { - this.useLegacyVersion = useLegacyVersion; + this.options = options.withUseLegacyVersion(useLegacyVersion); return this; } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/TempFileWriter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/TempFileWriter.java index 6c4cde4b80a6..7c188dfb6480 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/TempFileWriter.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/TempFileWriter.java @@ -82,7 +82,6 @@ private static OrcWriter createOrcFileWriter(OrcDataSink sink, List types) .withMaxStringStatisticsLimit(DataSize.ofBytes(0)) .withStripeMinSize(DataSize.of(64, MEGABYTE)) .withDictionaryMaxMemory(DataSize.of(1, MEGABYTE)), - false, ImmutableMap.of(), false, OrcWriteValidationMode.BOTH, diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveTestUtils.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveTestUtils.java index 89d5bd548568..599e87c57fd4 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveTestUtils.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveTestUtils.java @@ -163,7 +163,6 @@ private static OrcFileWriterFactory getDefaultOrcFileWriterFactory(HdfsEnvironme hdfsEnvironment, TYPE_MANAGER, new NodeVersion("test_version"), - new OrcWriterConfig(), new FileFormatDataSourceStats(), new OrcWriterConfig()); } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java index abd03ef5d4ad..e134c0e10948 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java @@ -323,7 +323,7 @@ public void testOrcOptimizedWriter(int rowCount, long fileSizePadding) .withRowsCount(rowCount) .withSession(session) .withFileSizePadding(fileSizePadding) - .withFileWriterFactory(new OrcFileWriterFactory(HDFS_ENVIRONMENT, TYPE_MANAGER, new NodeVersion("test"), false, STATS, new OrcWriterOptions())) + .withFileWriterFactory(new OrcFileWriterFactory(HDFS_ENVIRONMENT, TYPE_MANAGER, new NodeVersion("test"), STATS, new OrcWriterOptions())) .isReadableByRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT)) .isReadableByPageSource(new OrcPageSourceFactory(new OrcReaderOptions(), HDFS_ENVIRONMENT, STATS, UTC)); } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/benchmark/FileFormat.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/benchmark/FileFormat.java index 02623edc5ad7..de353b8f993a 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/benchmark/FileFormat.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/benchmark/FileFormat.java @@ -603,7 +603,6 @@ public PrestoOrcFormatWriter(File targetFile, List columnNames, List testingPredicate; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java index 1a5aecf77e50..d3c07dcda263 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java @@ -220,7 +220,6 @@ private IcebergFileWriter createOrcWriter( .withStripeMaxRowCount(getOrcWriterMaxStripeRows(session)) .withDictionaryMaxMemory(getOrcWriterMaxDictionaryMemory(session)) .withMaxStringStatisticsLimit(getOrcStringStatisticsLimit(session)), - false, IntStream.range(0, fileColumnNames.size()).toArray(), ImmutableMap.builder() .put(PRESTO_VERSION_NAME, nodeVersion.toString()) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergOrcFileWriter.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergOrcFileWriter.java index cbe3aa1b0a2d..d0d2c60eea2b 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergOrcFileWriter.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergOrcFileWriter.java @@ -72,14 +72,13 @@ public IcebergOrcFileWriter( ColumnMetadata fileColumnOrcTypes, CompressionKind compression, OrcWriterOptions options, - boolean writeLegacyVersion, int[] fileInputColumnIndexes, Map metadata, Optional> validationInputFactory, OrcWriteValidation.OrcWriteValidationMode validationMode, OrcWriterStats stats) { - super(orcDataSink, WriterKind.INSERT, NO_ACID_TRANSACTION, false, OptionalInt.empty(), rollbackAction, columnNames, fileColumnTypes, fileColumnOrcTypes, compression, options, writeLegacyVersion, fileInputColumnIndexes, metadata, validationInputFactory, validationMode, stats); + super(orcDataSink, WriterKind.INSERT, NO_ACID_TRANSACTION, false, OptionalInt.empty(), rollbackAction, columnNames, fileColumnTypes, fileColumnOrcTypes, compression, options, fileInputColumnIndexes, metadata, validationInputFactory, validationMode, stats); this.icebergSchema = requireNonNull(icebergSchema, "icebergSchema is null"); orcColumns = fileColumnOrcTypes; } From 05a21c59d9de862aa79f1295f56d09ad726b9721 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Fri, 9 Apr 2021 16:58:35 +0200 Subject: [PATCH 109/146] Use default writer version in a test The test does not test writer version, so should use the default. --- .../src/test/java/io/trino/orc/TestOrcBloomFilters.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/trino-orc/src/test/java/io/trino/orc/TestOrcBloomFilters.java b/lib/trino-orc/src/test/java/io/trino/orc/TestOrcBloomFilters.java index 222c38acc801..85563b894b53 100644 --- a/lib/trino-orc/src/test/java/io/trino/orc/TestOrcBloomFilters.java +++ b/lib/trino-orc/src/test/java/io/trino/orc/TestOrcBloomFilters.java @@ -123,7 +123,7 @@ public void testOrcHiveBloomFilterSerde() assertTrue(bloomFilterWrite.test(TEST_STRING)); assertTrue(bloomFilterWrite.testSlice(wrappedBuffer(TEST_STRING))); - Slice bloomFilterBytes = new CompressedMetadataWriter(new OrcMetadataWriter(true), CompressionKind.NONE, 1024) + Slice bloomFilterBytes = new CompressedMetadataWriter(new OrcMetadataWriter(false), CompressionKind.NONE, 1024) .writeBloomFilters(ImmutableList.of(bloomFilterWrite)); // Read through method From e033a95551c73f837ce0e772116dd5a136266c8e Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Fri, 9 Apr 2021 15:59:56 +0200 Subject: [PATCH 110/146] Use Trino writer identification when writing ORC --- .../src/main/java/io/trino/orc/OrcWriter.java | 10 ++-- .../java/io/trino/orc/OrcWriterOptions.java | 44 +++++++++++----- .../trino/orc/metadata/OrcMetadataWriter.java | 51 ++++++++++++++++--- .../io/trino/orc/TestOrcBloomFilters.java | 3 +- .../plugin/hive/orc/OrcWriterConfig.java | 23 +++++++-- .../plugin/hive/TestOrcWriterConfig.java | 7 +-- 6 files changed, 106 insertions(+), 32 deletions(-) diff --git a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java index f539fbd33784..b688afc754c7 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java @@ -88,13 +88,13 @@ public final class OrcWriter { private static final int INSTANCE_SIZE = ClassLayout.parseClass(OrcWriter.class).instanceSize(); - private static final String PRESTO_ORC_WRITER_VERSION_METADATA_KEY = "presto.writer.version"; - private static final String PRESTO_ORC_WRITER_VERSION; + private static final String TRINO_ORC_WRITER_VERSION_METADATA_KEY = "trino.writer.version"; + private static final String TRINO_ORC_WRITER_VERSION; private final OrcWriterStats stats; static { String version = OrcWriter.class.getPackage().getImplementationVersion(); - PRESTO_ORC_WRITER_VERSION = version == null ? "UNKNOWN" : version; + TRINO_ORC_WRITER_VERSION = version == null ? "UNKNOWN" : version; } private final OrcDataSink orcDataSink; @@ -160,8 +160,8 @@ public OrcWriter( this.maxCompressionBufferSize = toIntExact(options.getMaxCompressionBufferSize().toBytes()); this.userMetadata.putAll(requireNonNull(userMetadata, "userMetadata is null")); - this.userMetadata.put(PRESTO_ORC_WRITER_VERSION_METADATA_KEY, PRESTO_ORC_WRITER_VERSION); - this.metadataWriter = new CompressedMetadataWriter(new OrcMetadataWriter(options.isUseLegacyVersion()), compression, maxCompressionBufferSize); + this.userMetadata.put(TRINO_ORC_WRITER_VERSION_METADATA_KEY, TRINO_ORC_WRITER_VERSION); + this.metadataWriter = new CompressedMetadataWriter(new OrcMetadataWriter(options.getWriterIdentification()), compression, maxCompressionBufferSize); this.stats = requireNonNull(stats, "stats is null"); requireNonNull(columnNames, "columnNames is null"); diff --git a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java index 4abb7726b6f4..8edbe42c03c3 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriterOptions.java @@ -27,6 +27,24 @@ public class OrcWriterOptions { + public enum WriterIdentification + { + /** + * Write ORC files with a writer identification and version number that is readable by Hive 2.0.0 to 2.2.0 + */ + LEGACY_HIVE_COMPATIBLE, + + /** + * Write ORC files with the legacy writer identification of PrestoSQL + */ + PRESTO, + + /** + * Write ORC files with Trino writer identification. + */ + TRINO, + } + @VisibleForTesting static final DataSize DEFAULT_MAX_STRING_STATISTICS_LIMIT = DataSize.ofBytes(64); @VisibleForTesting @@ -38,7 +56,7 @@ public class OrcWriterOptions private static final int DEFAULT_ROW_GROUP_MAX_ROW_COUNT = 10_000; private static final DataSize DEFAULT_DICTIONARY_MAX_MEMORY = DataSize.of(16, MEGABYTE); - private final boolean useLegacyVersion; + private final WriterIdentification writerIdentification; private final DataSize stripeMinSize; private final DataSize stripeMaxSize; private final int stripeMaxRowCount; @@ -52,7 +70,7 @@ public class OrcWriterOptions public OrcWriterOptions() { this( - false, + WriterIdentification.TRINO, DEFAULT_STRIPE_MIN_SIZE, DEFAULT_STRIPE_MAX_SIZE, DEFAULT_STRIPE_MAX_ROW_COUNT, @@ -65,7 +83,7 @@ public OrcWriterOptions() } private OrcWriterOptions( - boolean useLegacyVersion, + WriterIdentification writerIdentification, DataSize stripeMinSize, DataSize stripeMaxSize, int stripeMaxRowCount, @@ -86,7 +104,7 @@ private OrcWriterOptions( requireNonNull(bloomFilterColumns, "bloomFilterColumns is null"); checkArgument(bloomFilterFpp > 0.0 && bloomFilterFpp < 1.0, "bloomFilterFpp should be > 0.0 & < 1.0"); - this.useLegacyVersion = useLegacyVersion; + this.writerIdentification = requireNonNull(writerIdentification, "writerIdentification is null"); this.stripeMinSize = stripeMinSize; this.stripeMaxSize = stripeMaxSize; this.stripeMaxRowCount = stripeMaxRowCount; @@ -98,15 +116,15 @@ private OrcWriterOptions( this.bloomFilterFpp = bloomFilterFpp; } - public boolean isUseLegacyVersion() + public WriterIdentification getWriterIdentification() { - return useLegacyVersion; + return writerIdentification; } - public OrcWriterOptions withUseLegacyVersion(boolean useLegacyVersion) + public OrcWriterOptions withWriterIdentification(WriterIdentification writerIdentification) { return builderFrom(this) - .setUseLegacyVersion(useLegacyVersion) + .setWriterIdentification(writerIdentification) .build(); } @@ -246,7 +264,7 @@ public static Builder builderFrom(OrcWriterOptions options) public static final class Builder { - private boolean useLegacyVersion; + private WriterIdentification writerIdentification; private DataSize stripeMinSize; private DataSize stripeMaxSize; private int stripeMaxRowCount; @@ -261,7 +279,7 @@ private Builder(OrcWriterOptions options) { requireNonNull(options, "options is null"); - this.useLegacyVersion = options.useLegacyVersion; + this.writerIdentification = options.writerIdentification; this.stripeMinSize = options.stripeMinSize; this.stripeMaxSize = options.stripeMaxSize; this.stripeMaxRowCount = options.stripeMaxRowCount; @@ -273,9 +291,9 @@ private Builder(OrcWriterOptions options) this.bloomFilterFpp = options.bloomFilterFpp; } - public Builder setUseLegacyVersion(boolean useLegacyVersion) + public Builder setWriterIdentification(WriterIdentification writerIdentification) { - this.useLegacyVersion = useLegacyVersion; + this.writerIdentification = writerIdentification; return this; } @@ -336,7 +354,7 @@ public Builder setBloomFilterFpp(double bloomFilterFpp) public OrcWriterOptions build() { return new OrcWriterOptions( - useLegacyVersion, + writerIdentification, stripeMinSize, stripeMaxSize, stripeMaxRowCount, diff --git a/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataWriter.java b/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataWriter.java index 1d77a83eab87..aff31d05c455 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataWriter.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataWriter.java @@ -18,6 +18,7 @@ import com.google.common.primitives.Longs; import io.airlift.slice.Slice; import io.airlift.slice.SliceOutput; +import io.trino.orc.OrcWriterOptions.WriterIdentification; import io.trino.orc.metadata.ColumnEncoding.ColumnEncodingKind; import io.trino.orc.metadata.OrcType.OrcTypeKind; import io.trino.orc.metadata.Stream.StreamKind; @@ -42,24 +43,32 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.orc.metadata.PostScript.MAGIC; import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; public class OrcMetadataWriter implements MetadataWriter { + // see https://github.com/trinodb/orc-protobuf/blob/master/src/main/protobuf/orc_proto.proto + private static final int TRINO_WRITER_ID = 4; + // in order to change this value, the master Apache ORC proto file must be updated + private static final int TRINO_WRITER_VERSION = 6; + // see https://github.com/trinodb/orc-protobuf/blob/master/src/main/protobuf/orc_proto.proto private static final int PRESTO_WRITER_ID = 2; // in order to change this value, the master Apache ORC proto file must be updated private static final int PRESTO_WRITER_VERSION = 6; + // maximum version readable by Hive 2.x before the ORC-125 fix private static final int HIVE_LEGACY_WRITER_VERSION = 4; + private static final List ORC_METADATA_VERSION = ImmutableList.of(0, 12); - private final boolean useLegacyVersion; + private final WriterIdentification writerIdentification; - public OrcMetadataWriter(boolean useLegacyVersion) + public OrcMetadataWriter(WriterIdentification writerIdentification) { - this.useLegacyVersion = useLegacyVersion; + this.writerIdentification = requireNonNull(writerIdentification, "writerIdentification is null"); } @Override @@ -78,13 +87,26 @@ public int writePostscript(SliceOutput output, int footerLength, int metadataLen .setMetadataLength(metadataLength) .setCompression(toCompression(compression)) .setCompressionBlockSize(compressionBlockSize) - .setWriterVersion(useLegacyVersion ? HIVE_LEGACY_WRITER_VERSION : PRESTO_WRITER_VERSION) + .setWriterVersion(getOrcWriterVersion()) .setMagic(MAGIC.toStringUtf8()) .build(); return writeProtobufObject(output, postScriptProtobuf); } + private int getOrcWriterVersion() + { + switch (writerIdentification) { + case LEGACY_HIVE_COMPATIBLE: + return HIVE_LEGACY_WRITER_VERSION; + case PRESTO: + return PRESTO_WRITER_VERSION; + case TRINO: + return TRINO_WRITER_VERSION; + } + throw new IllegalStateException("Unexpected value: " + writerIdentification); + } + @Override public int writeMetadata(SliceOutput output, Metadata metadata) throws IOException @@ -128,13 +150,28 @@ public int writeFooter(SliceOutput output, Footer footer) .map(OrcMetadataWriter::toUserMetadata) .collect(toList())); - if (!useLegacyVersion) { - builder.setWriter(PRESTO_WRITER_ID); - } + setWriter(builder); return writeProtobufObject(output, builder.build()); } + private void setWriter(OrcProto.Footer.Builder builder) + { + switch (writerIdentification) { + case LEGACY_HIVE_COMPATIBLE: + return; + + case PRESTO: + builder.setWriter(PRESTO_WRITER_ID); + return; + + case TRINO: + builder.setWriter(TRINO_WRITER_ID); + return; + } + throw new IllegalStateException("Unexpected value: " + writerIdentification); + } + private static OrcProto.StripeInformation toStripeInformation(StripeInformation stripe) { return OrcProto.StripeInformation.newBuilder() diff --git a/lib/trino-orc/src/test/java/io/trino/orc/TestOrcBloomFilters.java b/lib/trino-orc/src/test/java/io/trino/orc/TestOrcBloomFilters.java index 85563b894b53..b6dc072e4b88 100644 --- a/lib/trino-orc/src/test/java/io/trino/orc/TestOrcBloomFilters.java +++ b/lib/trino-orc/src/test/java/io/trino/orc/TestOrcBloomFilters.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Longs; import io.airlift.slice.Slice; +import io.trino.orc.OrcWriterOptions.WriterIdentification; import io.trino.orc.metadata.ColumnMetadata; import io.trino.orc.metadata.CompressedMetadataWriter; import io.trino.orc.metadata.CompressionKind; @@ -123,7 +124,7 @@ public void testOrcHiveBloomFilterSerde() assertTrue(bloomFilterWrite.test(TEST_STRING)); assertTrue(bloomFilterWrite.testSlice(wrappedBuffer(TEST_STRING))); - Slice bloomFilterBytes = new CompressedMetadataWriter(new OrcMetadataWriter(false), CompressionKind.NONE, 1024) + Slice bloomFilterBytes = new CompressedMetadataWriter(new OrcMetadataWriter(WriterIdentification.TRINO), CompressionKind.NONE, 1024) .writeBloomFilters(ImmutableList.of(bloomFilterWrite)); // Read through method diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcWriterConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcWriterConfig.java index 57d8e82bc4b6..46c2ebc41dfc 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcWriterConfig.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcWriterConfig.java @@ -16,9 +16,11 @@ import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; import io.airlift.configuration.DefunctConfig; +import io.airlift.configuration.LegacyConfig; import io.airlift.units.DataSize; import io.trino.orc.OrcWriteValidation.OrcWriteValidationMode; import io.trino.orc.OrcWriterOptions; +import io.trino.orc.OrcWriterOptions.WriterIdentification; import javax.validation.constraints.DecimalMax; import javax.validation.constraints.DecimalMin; @@ -136,16 +138,31 @@ public OrcWriterConfig setDefaultBloomFilterFpp(double defaultBloomFilterFpp) return this; } + @Deprecated public boolean isUseLegacyVersion() { - return options.isUseLegacyVersion(); + return options.getWriterIdentification() == WriterIdentification.LEGACY_HIVE_COMPATIBLE; } - @Config("hive.orc.writer.use-legacy-version-number") + @Deprecated + @LegacyConfig(value = "hive.orc.writer.use-legacy-version-number", replacedBy = "hive.orc.writer.writer-identification") @ConfigDescription("Write ORC files with a version number that is readable by Hive 2.0.0 to 2.2.0") public OrcWriterConfig setUseLegacyVersion(boolean useLegacyVersion) { - this.options = options.withUseLegacyVersion(useLegacyVersion); + this.options = options.withWriterIdentification(useLegacyVersion ? WriterIdentification.LEGACY_HIVE_COMPATIBLE : WriterIdentification.TRINO); + return this; + } + + @NotNull + public WriterIdentification getWriterIdentification() + { + return options.getWriterIdentification(); + } + + @Config("hive.orc.writer.writer-identification") + public OrcWriterConfig setWriterIdentification(WriterIdentification writerIdentification) + { + options = options.withWriterIdentification(writerIdentification); return this; } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestOrcWriterConfig.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestOrcWriterConfig.java index b3e3f3c82769..c9c533c5350c 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestOrcWriterConfig.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestOrcWriterConfig.java @@ -16,6 +16,7 @@ import com.google.common.collect.ImmutableMap; import io.airlift.units.DataSize; import io.trino.orc.OrcWriteValidation.OrcWriteValidationMode; +import io.trino.orc.OrcWriterOptions.WriterIdentification; import io.trino.plugin.hive.orc.OrcWriterConfig; import org.testng.annotations.Test; @@ -41,7 +42,7 @@ public void testDefaults() .setStringStatisticsLimit(DataSize.ofBytes(64)) .setMaxCompressionBufferSize(DataSize.of(256, KILOBYTE)) .setDefaultBloomFilterFpp(0.05) - .setUseLegacyVersion(false) + .setWriterIdentification(WriterIdentification.TRINO) .setValidationPercentage(0.0) .setValidationMode(OrcWriteValidationMode.BOTH)); } @@ -58,7 +59,7 @@ public void testExplicitPropertyMappings() .put("hive.orc.writer.string-statistics-limit", "17MB") .put("hive.orc.writer.max-compression-buffer-size", "19MB") .put("hive.orc.default-bloom-filter-fpp", "0.96") - .put("hive.orc.writer.use-legacy-version-number", "true") + .put("hive.orc.writer.writer-identification", "LEGACY_HIVE_COMPATIBLE") .put("hive.orc.writer.validation-percentage", "0.16") .put("hive.orc.writer.validation-mode", "DETAILED") .build(); @@ -72,7 +73,7 @@ public void testExplicitPropertyMappings() .setStringStatisticsLimit(DataSize.of(17, MEGABYTE)) .setMaxCompressionBufferSize(DataSize.of(19, MEGABYTE)) .setDefaultBloomFilterFpp(0.96) - .setUseLegacyVersion(true) + .setWriterIdentification(WriterIdentification.LEGACY_HIVE_COMPATIBLE) .setValidationPercentage(0.16) .setValidationMode(OrcWriteValidationMode.DETAILED); From 7cea7289e615e6b7399504fc1fe0e3521fb4a3b4 Mon Sep 17 00:00:00 2001 From: Karol Sobczak Date: Mon, 12 Apr 2021 12:09:25 +0200 Subject: [PATCH 111/146] Refer to table-scan-node-partitioning-min-bucket-to-task-ratio in doc --- docs/src/main/sphinx/admin/properties-optimizer.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/src/main/sphinx/admin/properties-optimizer.rst b/docs/src/main/sphinx/admin/properties-optimizer.rst index 96cd1facec68..62be3c1c5ea3 100644 --- a/docs/src/main/sphinx/admin/properties-optimizer.rst +++ b/docs/src/main/sphinx/admin/properties-optimizer.rst @@ -125,7 +125,10 @@ join output rows can be skipped. Use connector provided table node partitioning when reading tables. For example, table node partitioning corresponds to Hive table buckets. -When set to ``true`` each table partition is read by a separate worker. +When set to ``true`` and minimal partition to task ratio is matched or exceeded, +each table partition is read by a separate worker. The minimal ratio is defined in +``optimizer.table-scan-node-partitioning-min-bucket-to-task-ratio``. + Partition reader assignments are distributed across workers for parallel processing. Use of table scan node partitioning can improve query performance by reducing query complexity. For example, From 3ab3ca918cc7f8e74440c6ef9acc1e0ffa6beb9c Mon Sep 17 00:00:00 2001 From: Ashhar Hasan Date: Sun, 11 Apr 2021 20:36:46 +0530 Subject: [PATCH 112/146] Format JdbcSortItem in EXPLAIN output --- .../java/io/trino/spi/connector/SortOrder.java | 10 ++++++++++ .../java/io/trino/plugin/jdbc/JdbcSortItem.java | 10 ++++++++++ .../trino/plugin/druid/BaseDruidConnectorTest.java | 1 + .../trino/plugin/redis/TestRedisConnectorTest.java | 3 +++ .../java/io/trino/testing/BaseConnectorTest.java | 14 ++++++++++++++ 5 files changed, 38 insertions(+) diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/SortOrder.java b/core/trino-spi/src/main/java/io/trino/spi/connector/SortOrder.java index 90d34053c330..ddfbf08a8e03 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/SortOrder.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/SortOrder.java @@ -13,6 +13,8 @@ */ package io.trino.spi.connector; +import static java.lang.String.format; + public enum SortOrder { ASC_NULLS_FIRST(true, true), @@ -38,4 +40,12 @@ public boolean isNullsFirst() { return nullsFirst; } + + @Override + public String toString() + { + return format("%s %s", + ascending ? "ASC" : "DESC", + nullsFirst ? "NULLS FIRST" : "NULLS LAST"); + } } diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcSortItem.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcSortItem.java index f33e45f85c80..6ce11f1b0c4e 100644 --- a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcSortItem.java +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcSortItem.java @@ -67,4 +67,14 @@ public int hashCode() { return Objects.hash(column, sortOrder); } + + @Override + public String toString() + { + return new StringBuilder() + .append(column) + .append(" ") + .append(sortOrder) + .toString(); + } } diff --git a/plugin/trino-druid/src/test/java/io/trino/plugin/druid/BaseDruidConnectorTest.java b/plugin/trino-druid/src/test/java/io/trino/plugin/druid/BaseDruidConnectorTest.java index 4792fba22544..7b4e4adf09d6 100644 --- a/plugin/trino-druid/src/test/java/io/trino/plugin/druid/BaseDruidConnectorTest.java +++ b/plugin/trino-druid/src/test/java/io/trino/plugin/druid/BaseDruidConnectorTest.java @@ -128,6 +128,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) case SUPPORTS_RENAME_TABLE: case SUPPORTS_COMMENT_ON_COLUMN: case SUPPORTS_COMMENT_ON_TABLE: + case SUPPORTS_TOPN_PUSHDOWN: return false; default: return super.hasBehavior(connectorBehavior); diff --git a/plugin/trino-redis/src/test/java/io/trino/plugin/redis/TestRedisConnectorTest.java b/plugin/trino-redis/src/test/java/io/trino/plugin/redis/TestRedisConnectorTest.java index 31c2de7315fe..8fd8cdde4b18 100644 --- a/plugin/trino-redis/src/test/java/io/trino/plugin/redis/TestRedisConnectorTest.java +++ b/plugin/trino-redis/src/test/java/io/trino/plugin/redis/TestRedisConnectorTest.java @@ -45,6 +45,9 @@ public void destroy() protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) { switch (connectorBehavior) { + case SUPPORTS_TOPN_PUSHDOWN: + return false; + case SUPPORTS_CREATE_SCHEMA: return false; diff --git a/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java b/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java index 5c9cd8e903e6..d398b19c8536 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java @@ -36,6 +36,7 @@ import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_INSERT; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_RENAME_TABLE; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_RENAME_TABLE_ACROSS_SCHEMAS; +import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_TOPN_PUSHDOWN; import static io.trino.testing.assertions.Assert.assertEquals; import static io.trino.testing.sql.TestTable.randomTableSuffix; import static java.lang.String.join; @@ -284,6 +285,19 @@ public void testPredicateReflectedInExplain() "(predicate|filterPredicate|constraint).{0,10}(nationkey|NATIONKEY)"); } + @Test + public void testSortItemsReflectedInExplain() + { + // Even if the sort items are pushed down into the table scan, it should still be reflected in EXPLAIN (via ConnectorTableHandle.toString) + @Language("RegExp") String expectedPattern = hasBehavior(SUPPORTS_TOPN_PUSHDOWN) + ? "sortOrder=\\[nationkey:\\w+:\\w+ DESC NULLS LAST] limit=5" + : "\\[5 by \\(nationkey DESC NULLS LAST\\)]"; + + assertExplain( + "EXPLAIN SELECT name FROM nation ORDER BY nationkey DESC NULLS LAST LIMIT 5", + expectedPattern); + } + @Test public void testConcurrentScans() { From ee001f8d281c0573e1ba47b1a54b0adbb269e4e2 Mon Sep 17 00:00:00 2001 From: Karol Sobczak Date: Wed, 27 Jan 2021 18:12:37 +0100 Subject: [PATCH 113/146] Collect build side before local dynamic filters are consumed In practice, join currently doesn't support probe short circuit. Therefore, this commit makes it explicit that when local dynamic filters are consumed by table scan then join should wait for build side before requesting any probe rows. This way table scan has a chance to consume local dynamic filters before producing any data. In the future, join shirt-circuit should be fixed by fetching probe rows first. This should be preceeded by adding node local waiting for DFs at probe table scan level (this is what current join semantics gives). --- .../io/trino/operator/LookupJoinOperator.java | 19 +++++- .../operator/LookupJoinOperatorFactory.java | 6 ++ .../trino/operator/LookupJoinOperators.java | 8 +++ .../sql/planner/LocalExecutionPlanner.java | 31 +++++++++- .../BenchmarkHashBuildAndJoinOperators.java | 1 + .../trino/operator/TestHashJoinOperator.java | 59 +++++++++++++++++-- .../benchmark/HashBuildAndJoinBenchmark.java | 1 + .../trino/benchmark/HashBuildBenchmark.java | 1 + .../io/trino/benchmark/HashJoinBenchmark.java | 1 + 9 files changed, 116 insertions(+), 11 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/operator/LookupJoinOperator.java b/core/trino-main/src/main/java/io/trino/operator/LookupJoinOperator.java index 4f22d5493845..a6839b0a8881 100644 --- a/core/trino-main/src/main/java/io/trino/operator/LookupJoinOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/LookupJoinOperator.java @@ -72,6 +72,7 @@ public class LookupJoinOperator implements AdapterWorkProcessorOperator { private final ListenableFuture lookupSourceProviderFuture; + private final boolean waitForBuild; private final PageBuffer pageBuffer; private final WorkProcessor pages; private final SpillingJoinProcessor joinProcessor; @@ -82,6 +83,7 @@ public class LookupJoinOperator List buildOutputTypes, JoinType joinType, boolean outputSingleMatch, + boolean waitForBuild, LookupSourceFactory lookupSourceFactory, JoinProbeFactory joinProbeFactory, Runnable afterClose, @@ -92,8 +94,9 @@ public class LookupJoinOperator Optional> sourcePages) { this.statisticsCounter = new JoinStatisticsCounter(joinType); + this.waitForBuild = waitForBuild; lookupSourceProviderFuture = lookupSourceFactory.createLookupSourceProvider(); - pageBuffer = new PageBuffer(lookupSourceProviderFuture); + pageBuffer = new PageBuffer(); joinProcessor = new SpillingJoinProcessor( processorContext, afterClose, @@ -102,6 +105,7 @@ public class LookupJoinOperator buildOutputTypes, joinType, outputSingleMatch, + waitForBuild, hashGenerator, joinProbeFactory, lookupSourceFactory, @@ -121,7 +125,7 @@ public Optional getOperatorInfo() @Override public boolean needsInput() { - return lookupSourceProviderFuture.isDone() && pageBuffer.isEmpty() && !pageBuffer.isFinished(); + return (!waitForBuild || lookupSourceProviderFuture.isDone()) && pageBuffer.isEmpty() && !pageBuffer.isFinished(); } @Override @@ -518,9 +522,11 @@ private static class SpillingJoinProcessor private final List buildOutputTypes; private final JoinType joinType; private final boolean outputSingleMatch; + private final boolean waitForBuild; private final HashGenerator hashGenerator; private final JoinProbeFactory joinProbeFactory; private final LookupSourceFactory lookupSourceFactory; + private final ListenableFuture lookupSourceProvider; private final JoinStatisticsCounter statisticsCounter; private final PageJoiner sourcePagesJoiner; private final WorkProcessor joinedSourcePages; @@ -544,6 +550,7 @@ private SpillingJoinProcessor( List buildOutputTypes, JoinType joinType, boolean outputSingleMatch, + boolean waitForBuild, HashGenerator hashGenerator, JoinProbeFactory joinProbeFactory, LookupSourceFactory lookupSourceFactory, @@ -559,9 +566,11 @@ private SpillingJoinProcessor( this.buildOutputTypes = requireNonNull(buildOutputTypes, "buildOutputTypes is null"); this.joinType = requireNonNull(joinType, "joinType is null"); this.outputSingleMatch = outputSingleMatch; + this.waitForBuild = waitForBuild; this.hashGenerator = requireNonNull(hashGenerator, "hashGenerator is null"); this.joinProbeFactory = requireNonNull(joinProbeFactory, "joinProbeFactory is null"); this.lookupSourceFactory = requireNonNull(lookupSourceFactory, "lookupSourceFactory is null"); + this.lookupSourceProvider = requireNonNull(lookupSourceProvider, "lookupSourceProvider is null"); this.statisticsCounter = requireNonNull(statisticsCounter, "statisticsCounter is null"); sourcePagesJoiner = new PageJoiner( processorContext, @@ -582,6 +591,12 @@ private SpillingJoinProcessor( @Override public ProcessState> process() { + // wait for build side to be completed before fetching any probe data + // TODO: fix support for probe short-circuit: https://github.com/trinodb/trino/issues/3957 + if (waitForBuild && !lookupSourceProvider.isDone()) { + return ProcessState.blocked(lookupSourceProvider); + } + if (!joinedSourcePages.isFinished()) { return ProcessState.ofResult(joinedSourcePages); } diff --git a/core/trino-main/src/main/java/io/trino/operator/LookupJoinOperatorFactory.java b/core/trino-main/src/main/java/io/trino/operator/LookupJoinOperatorFactory.java index 196a48e50a8d..d8e5d828e4b8 100644 --- a/core/trino-main/src/main/java/io/trino/operator/LookupJoinOperatorFactory.java +++ b/core/trino-main/src/main/java/io/trino/operator/LookupJoinOperatorFactory.java @@ -46,6 +46,7 @@ public class LookupJoinOperatorFactory private final List buildOutputTypes; private final JoinType joinType; private final boolean outputSingleMatch; + private final boolean waitForBuild; private final JoinProbeFactory joinProbeFactory; private final Optional outerOperatorFactoryResult; private final JoinBridgeManager joinBridgeManager; @@ -64,6 +65,7 @@ public LookupJoinOperatorFactory( List buildOutputTypes, JoinType joinType, boolean outputSingleMatch, + boolean waitForBuild, JoinProbeFactory joinProbeFactory, BlockTypeOperators blockTypeOperators, OptionalInt totalOperatorsCount, @@ -77,6 +79,7 @@ public LookupJoinOperatorFactory( this.buildOutputTypes = ImmutableList.copyOf(requireNonNull(buildOutputTypes, "buildOutputTypes is null")); this.joinType = requireNonNull(joinType, "joinType is null"); this.outputSingleMatch = outputSingleMatch; + this.waitForBuild = waitForBuild; this.joinProbeFactory = requireNonNull(joinProbeFactory, "joinProbeFactory is null"); this.joinBridgeManager = lookupSourceFactoryManager; @@ -123,6 +126,7 @@ private LookupJoinOperatorFactory(LookupJoinOperatorFactory other) buildOutputTypes = other.buildOutputTypes; joinType = other.joinType; outputSingleMatch = other.outputSingleMatch; + waitForBuild = other.waitForBuild; joinProbeFactory = other.joinProbeFactory; joinBridgeManager = other.joinBridgeManager; outerOperatorFactoryResult = other.outerOperatorFactoryResult; @@ -193,6 +197,7 @@ public WorkProcessorOperator create(ProcessorContext processorContext, WorkProce buildOutputTypes, joinType, outputSingleMatch, + waitForBuild, lookupSourceFactory, joinProbeFactory, () -> joinBridgeManager.probeOperatorClosed(processorContext.getLifespan()), @@ -215,6 +220,7 @@ public AdapterWorkProcessorOperator createAdapterOperator(ProcessorContext proce buildOutputTypes, joinType, outputSingleMatch, + waitForBuild, lookupSourceFactory, joinProbeFactory, () -> joinBridgeManager.probeOperatorClosed(processorContext.getLifespan()), diff --git a/core/trino-main/src/main/java/io/trino/operator/LookupJoinOperators.java b/core/trino-main/src/main/java/io/trino/operator/LookupJoinOperators.java index 857009866945..ebce8869f22b 100644 --- a/core/trino-main/src/main/java/io/trino/operator/LookupJoinOperators.java +++ b/core/trino-main/src/main/java/io/trino/operator/LookupJoinOperators.java @@ -49,6 +49,7 @@ public OperatorFactory innerJoin( JoinBridgeManager lookupSourceFactory, List probeTypes, boolean outputSingleMatch, + boolean waitForBuild, List probeJoinChannel, OptionalInt probeHashChannel, Optional> probeOutputChannels, @@ -66,6 +67,7 @@ public OperatorFactory innerJoin( probeOutputChannels.orElse(rangeList(probeTypes.size())), JoinType.INNER, outputSingleMatch, + waitForBuild, totalOperatorsCount, partitioningSpillerFactory, blockTypeOperators); @@ -94,6 +96,7 @@ public OperatorFactory probeOuterJoin( probeOutputChannels.orElse(rangeList(probeTypes.size())), JoinType.PROBE_OUTER, outputSingleMatch, + false, totalOperatorsCount, partitioningSpillerFactory, blockTypeOperators); @@ -104,6 +107,7 @@ public OperatorFactory lookupOuterJoin( PlanNodeId planNodeId, JoinBridgeManager lookupSourceFactory, List probeTypes, + boolean waitForBuild, List probeJoinChannel, OptionalInt probeHashChannel, Optional> probeOutputChannels, @@ -121,6 +125,7 @@ public OperatorFactory lookupOuterJoin( probeOutputChannels.orElse(rangeList(probeTypes.size())), JoinType.LOOKUP_OUTER, false, + waitForBuild, totalOperatorsCount, partitioningSpillerFactory, blockTypeOperators); @@ -148,6 +153,7 @@ public OperatorFactory fullOuterJoin( probeOutputChannels.orElse(rangeList(probeTypes.size())), JoinType.FULL_OUTER, false, + false, totalOperatorsCount, partitioningSpillerFactory, blockTypeOperators); @@ -170,6 +176,7 @@ private OperatorFactory createJoinOperatorFactory( List probeOutputChannels, JoinType joinType, boolean outputSingleMatch, + boolean waitForBuild, OptionalInt totalOperatorsCount, PartitioningSpillerFactory partitioningSpillerFactory, BlockTypeOperators blockTypeOperators) @@ -187,6 +194,7 @@ private OperatorFactory createJoinOperatorFactory( lookupSourceFactoryManager.getBuildOutputTypes(), joinType, outputSingleMatch, + waitForBuild, new JoinProbeFactory(probeOutputChannels.stream().mapToInt(i -> i).toArray(), probeJoinChannel, probeHashChannel), blockTypeOperators, totalOperatorsCount, diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java index 075ab41c23a0..49f700050df1 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java @@ -1743,6 +1743,7 @@ public PhysicalOperation visitIndexJoin(IndexJoinNode node, LocalExecutionPlanCo lookupSourceFactoryManager, probeSource.getTypes(), false, + false, probeChannels, probeHashChannel, Optional.empty(), @@ -2118,7 +2119,15 @@ private PhysicalOperation createLookupJoin( JoinBridgeManager lookupSourceFactory = createLookupSourceFactory(node, buildNode, buildSymbols, buildHashSymbol, probeSource, context, spillEnabled, localDynamicFilters); - OperatorFactory operator = createLookupJoin(node, probeSource, probeSymbols, probeHashSymbol, lookupSourceFactory, context, spillEnabled); + OperatorFactory operator = createLookupJoin( + node, + probeSource, + probeSymbols, + probeHashSymbol, + lookupSourceFactory, + context, + spillEnabled, + !localDynamicFilters.isEmpty()); ImmutableMap.Builder outputMappings = ImmutableMap.builder(); List outputSymbols = node.getOutputSymbols(); @@ -2320,7 +2329,8 @@ private OperatorFactory createLookupJoin( Optional probeHashSymbol, JoinBridgeManager lookupSourceFactoryManager, LocalExecutionPlanContext context, - boolean spillEnabled) + boolean spillEnabled, + boolean consumedLocalDynamicFilters) { List probeTypes = probeSource.getTypes(); List probeOutputChannels = ImmutableList.copyOf(getChannelsForSymbols(node.getLeftOutputSymbols(), probeSource.getLayout())); @@ -2342,6 +2352,9 @@ private OperatorFactory createLookupJoin( .map(JoinNode.EquiJoinClause::getRight) .collect(toImmutableSet()) .containsAll(node.getRightOutputSymbols()); + // Wait for build side to be collected before local dynamic filters are + // consumed by table scan. This way table scan can filter data more efficiently. + boolean waitForBuild = consumedLocalDynamicFilters; switch (node.getType()) { case INNER: return lookupJoinOperators.innerJoin( @@ -2350,6 +2363,7 @@ private OperatorFactory createLookupJoin( lookupSourceFactoryManager, probeTypes, outputSingleMatch, + waitForBuild, probeJoinChannels, probeHashChannel, Optional.of(probeOutputChannels), @@ -2370,7 +2384,18 @@ private OperatorFactory createLookupJoin( partitioningSpillerFactory, blockTypeOperators); case RIGHT: - return lookupJoinOperators.lookupOuterJoin(context.getNextOperatorId(), node.getId(), lookupSourceFactoryManager, probeTypes, probeJoinChannels, probeHashChannel, Optional.of(probeOutputChannels), totalOperatorsCount, partitioningSpillerFactory, blockTypeOperators); + return lookupJoinOperators.lookupOuterJoin( + context.getNextOperatorId(), + node.getId(), + lookupSourceFactoryManager, + probeTypes, + waitForBuild, + probeJoinChannels, + probeHashChannel, + Optional.of(probeOutputChannels), + totalOperatorsCount, + partitioningSpillerFactory, + blockTypeOperators); case FULL: return lookupJoinOperators.fullOuterJoin(context.getNextOperatorId(), node.getId(), lookupSourceFactoryManager, probeTypes, probeJoinChannels, probeHashChannel, Optional.of(probeOutputChannels), totalOperatorsCount, partitioningSpillerFactory, blockTypeOperators); } diff --git a/core/trino-main/src/test/java/io/trino/operator/BenchmarkHashBuildAndJoinOperators.java b/core/trino-main/src/test/java/io/trino/operator/BenchmarkHashBuildAndJoinOperators.java index dc46034b6ae1..ac0fa350bddd 100644 --- a/core/trino-main/src/test/java/io/trino/operator/BenchmarkHashBuildAndJoinOperators.java +++ b/core/trino-main/src/test/java/io/trino/operator/BenchmarkHashBuildAndJoinOperators.java @@ -225,6 +225,7 @@ public void setup() lookupSourceFactory, types, false, + false, hashChannels, hashChannel, Optional.of(outputChannels), diff --git a/core/trino-main/src/test/java/io/trino/operator/TestHashJoinOperator.java b/core/trino-main/src/test/java/io/trino/operator/TestHashJoinOperator.java index 8ec3090b3044..9ec7c5bf8b60 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestHashJoinOperator.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestHashJoinOperator.java @@ -243,6 +243,7 @@ public void testUnwrapsLazyBlocks() lookupSourceFactory, probePages.getTypes(), false, + false, Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), @@ -295,6 +296,7 @@ public void testYield() lookupSourceFactory, probePages.getTypes(), false, + false, Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), @@ -1180,6 +1182,7 @@ public void testInnerJoinWithEmptyLookupSource(boolean parallelBuild, boolean pr lookupSourceFactoryManager, probePages.getTypes(), false, + false, Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), @@ -1217,6 +1220,7 @@ public void testLookupOuterJoinWithEmptyLookupSource(boolean parallelBuild, bool new PlanNodeId("test"), lookupSourceFactoryManager, probePages.getTypes(), + false, Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), @@ -1353,6 +1357,7 @@ public void testInnerJoinWithNonEmptyLookupSourceAndEmptyProbe(boolean parallelB lookupSourceFactoryManager, probePages.getTypes(), false, + false, Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), @@ -1373,16 +1378,32 @@ public void testInnerJoinWithNonEmptyLookupSourceAndEmptyProbe(boolean parallelB public void testInnerJoinWithBlockingLookupSourceAndEmptyProbe(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception { + // join that waits for build side to be collected TaskContext taskContext = createTaskContext(); - OperatorFactory joinOperatorFactory = createJoinOperatorFactoryWithBlockingLookupSource(taskContext, parallelBuild, probeHashEnabled, buildHashEnabled); - + OperatorFactory joinOperatorFactory = createJoinOperatorFactoryWithBlockingLookupSource(taskContext, parallelBuild, probeHashEnabled, buildHashEnabled, true); DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext(); try (Operator joinOperator = joinOperatorFactory.createOperator(driverContext)) { joinOperatorFactory.noMoreOperators(); assertFalse(joinOperator.needsInput()); joinOperator.finish(); - // lookup join operator will yield once before finishing assertNull(joinOperator.getOutput()); + + // lookup join operator got blocked waiting for build side + assertFalse(joinOperator.isBlocked().isDone()); + assertFalse(joinOperator.isFinished()); + } + + // join that doesn't wait for build side to be collected + taskContext = createTaskContext(); + joinOperatorFactory = createJoinOperatorFactoryWithBlockingLookupSource(taskContext, parallelBuild, probeHashEnabled, buildHashEnabled, false); + driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext(); + try (Operator joinOperator = joinOperatorFactory.createOperator(driverContext)) { + joinOperatorFactory.noMoreOperators(); + assertTrue(joinOperator.needsInput()); + joinOperator.finish(); + assertNull(joinOperator.getOutput()); + + // lookup join operator will yield once before finishing assertNull(joinOperator.getOutput()); assertTrue(joinOperator.isBlocked().isDone()); assertTrue(joinOperator.isFinished()); @@ -1393,14 +1414,38 @@ public void testInnerJoinWithBlockingLookupSourceAndEmptyProbe(boolean parallelB public void testInnerJoinWithBlockingLookupSource(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception { - TaskContext taskContext = createTaskContext(); - OperatorFactory joinOperatorFactory = createJoinOperatorFactoryWithBlockingLookupSource(taskContext, parallelBuild, probeHashEnabled, buildHashEnabled); + RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR)); + Page probePage = getOnlyElement(probePages.addSequencePage(1, 0).build()); + // join that waits for build side to be collected + TaskContext taskContext = createTaskContext(); + OperatorFactory joinOperatorFactory = createJoinOperatorFactoryWithBlockingLookupSource(taskContext, parallelBuild, probeHashEnabled, buildHashEnabled, true); DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext(); try (Operator joinOperator = joinOperatorFactory.createOperator(driverContext)) { joinOperatorFactory.noMoreOperators(); assertFalse(joinOperator.needsInput()); assertNull(joinOperator.getOutput()); + + // lookup join operator got blocked waiting for build side + assertFalse(joinOperator.isBlocked().isDone()); + assertFalse(joinOperator.isFinished()); + } + + // join that doesn't wait for build side to be collected + taskContext = createTaskContext(); + joinOperatorFactory = createJoinOperatorFactoryWithBlockingLookupSource(taskContext, parallelBuild, probeHashEnabled, buildHashEnabled, false); + driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext(); + try (Operator joinOperator = joinOperatorFactory.createOperator(driverContext)) { + joinOperatorFactory.noMoreOperators(); + assertTrue(joinOperator.needsInput()); + assertNull(joinOperator.getOutput()); + + // join needs input page + assertTrue(joinOperator.isBlocked().isDone()); + assertFalse(joinOperator.isFinished()); + joinOperator.addInput(probePage); + assertNull(joinOperator.getOutput()); + // lookup join operator got blocked waiting for build side assertFalse(joinOperator.isBlocked().isDone()); assertFalse(joinOperator.isFinished()); @@ -1487,7 +1532,7 @@ public void testInnerJoinLoadsPagesInOrder() assertTrue(outputPages.isFinished()); } - private OperatorFactory createJoinOperatorFactoryWithBlockingLookupSource(TaskContext taskContext, boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) + private OperatorFactory createJoinOperatorFactoryWithBlockingLookupSource(TaskContext taskContext, boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled, boolean waitForBuild) { // build factory List buildTypes = ImmutableList.of(VARCHAR); @@ -1504,6 +1549,7 @@ private OperatorFactory createJoinOperatorFactoryWithBlockingLookupSource(TaskCo lookupSourceFactoryManager, probePages.getTypes(), false, + waitForBuild, Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), @@ -1577,6 +1623,7 @@ private OperatorFactory innerJoinOperatorFactory( lookupSourceFactoryManager, probePages.getTypes(), outputSingleMatch, + false, Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), diff --git a/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashBuildAndJoinBenchmark.java b/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashBuildAndJoinBenchmark.java index ea5d79420932..0eecf87c5989 100644 --- a/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashBuildAndJoinBenchmark.java +++ b/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashBuildAndJoinBenchmark.java @@ -137,6 +137,7 @@ protected List createDrivers(TaskContext taskContext) lookupSourceFactoryManager, sourceTypes, false, + false, Ints.asList(0), hashChannel, Optional.empty(), diff --git a/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashBuildBenchmark.java b/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashBuildBenchmark.java index b9786ab8be4a..bb2eb8d59a86 100644 --- a/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashBuildBenchmark.java +++ b/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashBuildBenchmark.java @@ -96,6 +96,7 @@ protected List createDrivers(TaskContext taskContext) lookupSourceFactoryManager, ImmutableList.of(BIGINT), false, + false, Ints.asList(0), OptionalInt.empty(), Optional.empty(), diff --git a/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashJoinBenchmark.java b/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashJoinBenchmark.java index 6b4211aa80fa..6de480bf58b3 100644 --- a/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashJoinBenchmark.java +++ b/testing/trino-benchmark/src/main/java/io/trino/benchmark/HashJoinBenchmark.java @@ -106,6 +106,7 @@ protected List createDrivers(TaskContext taskContext) lookupSourceFactoryManager, lineItemTypes, false, + false, Ints.asList(0), OptionalInt.empty(), Optional.empty(), From fe608f2723842037ff620d612a706900e79c52c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Kokosi=C5=84ski?= Date: Tue, 13 Apr 2021 09:24:21 +0200 Subject: [PATCH 114/146] Make JdbcMetadata::getSchemaTableName public --- .../src/main/java/io/trino/plugin/jdbc/JdbcMetadata.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadata.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadata.java index 9c5d2e6fef35..0dfafec7e428 100644 --- a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadata.java +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadata.java @@ -542,7 +542,7 @@ public ConnectorTableMetadata getTableMetadata(ConnectorSession session, Connect jdbcClient.getTableProperties(session, handle)); } - protected static SchemaTableName getSchemaTableName(JdbcTableHandle handle) + public static SchemaTableName getSchemaTableName(JdbcTableHandle handle) { return handle.isNamedRelation() ? handle.getRequiredNamedRelation().getSchemaTableName() From b94026852e7aa253a34a354b42cf958d4bd0794c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Kokosi=C5=84ski?= Date: Tue, 13 Apr 2021 12:56:57 +0200 Subject: [PATCH 115/146] Ignore not existing logs when collecting log files --- .../product/launcher/env/DockerContainer.java | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/DockerContainer.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/DockerContainer.java index 4c1007b5b0ab..25bd0d226203 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/DockerContainer.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/DockerContainer.java @@ -238,6 +238,16 @@ private void copyFileToContainer(String containerPath, CheckedRunnable copy) } public String execCommand(String... command) + { + ExecResult result = execCommandForResult(command); + if (result.getExitCode() == 0) { + return result.getStdout(); + } + String fullCommand = Joiner.on(" ").join(command); + throw new RuntimeException(format("Could not execute command '%s' in container %s: %s", fullCommand, logicalName, result.getStderr())); + } + + public ExecResult execCommandForResult(String... command) { String fullCommand = Joiner.on(" ").join(command); if (!isRunning()) { @@ -247,12 +257,7 @@ public String execCommand(String... command) log.info("Executing command '%s' in container %s", fullCommand, logicalName); try { - ExecResult result = (ExecResult) executor.getAsync(() -> execInContainer(command)).get(); - if (result.getExitCode() == 0) { - return result.getStdout(); - } - - throw new RuntimeException(format("Could not execute command '%s' in container %s: %s", fullCommand, logicalName, result.getStderr())); + return (ExecResult) executor.getAsync(() -> execInContainer(command)).get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -348,9 +353,14 @@ private void copyFileFromContainer(String filename, Path targetPath) private List listFilesInContainer(String path) { try { + ExecResult execResult = execCommandForResult("/usr/bin/find", path, "-type", "f", "-print"); + if (execResult.getExitCode() != 0) { + log.warn("Could not list files in container '%s' path %s: %s", logicalName, path, execResult.getStderr()); + return ImmutableList.of(); + } return Splitter.on("\n") .omitEmptyStrings() - .splitToList(execCommand("/usr/bin/find", path, "-type", "f", "-print")); + .splitToList(execResult.getStdout()); } catch (RuntimeException e) { log.warn(e, "Could not list files in container '%s' path %s", logicalName, path); From 01cf2e82981cc4035a1eeeb4b2fc3bd387b8915e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Osipiuk?= Date: Mon, 12 Apr 2021 12:41:49 +0100 Subject: [PATCH 116/146] Rename test methods --- .../sql/planner/iterative/rule/TestPushTopNIntoTableScan.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/iterative/rule/TestPushTopNIntoTableScan.java b/core/trino-main/src/test/java/io/trino/sql/planner/iterative/rule/TestPushTopNIntoTableScan.java index 9ca4fb06acd7..1ed8981d5bfb 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/iterative/rule/TestPushTopNIntoTableScan.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/iterative/rule/TestPushTopNIntoTableScan.java @@ -103,7 +103,7 @@ public void testDoesNotFire() } @Test - public void testPushTopNIntoTableScan() + public void testPushSingleTopNIntoTableScan() { try (RuleTester ruleTester = defaultRuleTester()) { MockConnectorTableHandle connectorHandle = new MockConnectorTableHandle(TEST_SCHEMA_TABLE); @@ -135,7 +135,7 @@ public void testPushTopNIntoTableScan() } @Test - public void testPushTopNIntoTableScanPartial() + public void testPushSingleTopNIntoTableScanNotGuaranteed() { try (RuleTester ruleTester = defaultRuleTester()) { MockConnectorTableHandle connectorHandle = new MockConnectorTableHandle(TEST_SCHEMA_TABLE); From 98fbb58451b01fa3b47b451b5c05602d41250de2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Osipiuk?= Date: Mon, 12 Apr 2021 12:40:51 +0100 Subject: [PATCH 117/146] Push PARTIAL TopN into table scan --- .../io/trino/sql/planner/PlanOptimizers.java | 3 +- .../iterative/rule/PushTopNIntoTableScan.java | 21 ++++-- .../rule/TestPushTopNIntoTableScan.java | 68 +++++++++++++++++++ .../plugin/jdbc/BaseJdbcConnectorTest.java | 13 ++++ 4 files changed, 100 insertions(+), 5 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/PlanOptimizers.java b/core/trino-main/src/main/java/io/trino/sql/planner/PlanOptimizers.java index 88b6c93a88a3..d09c21b4e767 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/PlanOptimizers.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/PlanOptimizers.java @@ -734,7 +734,8 @@ public PlanOptimizers( new CreatePartialTopN(), new PushTopNThroughProject(), new PushTopNThroughOuterJoin(), - new PushTopNThroughUnion()))); + new PushTopNThroughUnion(), + new PushTopNIntoTableScan(metadata)))); builder.add(new IterativeOptimizer( ruleStats, statsCalculator, diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushTopNIntoTableScan.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushTopNIntoTableScan.java index 96fe58c40231..fd11af8e371c 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushTopNIntoTableScan.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushTopNIntoTableScan.java @@ -25,7 +25,6 @@ import io.trino.sql.planner.plan.PlanNode; import io.trino.sql.planner.plan.TableScanNode; import io.trino.sql.planner.plan.TopNNode; -import io.trino.sql.planner.plan.TopNNode.Step; import java.util.List; import java.util.Map; @@ -37,16 +36,18 @@ import static io.trino.sql.planner.plan.Patterns.source; import static io.trino.sql.planner.plan.Patterns.tableScan; import static io.trino.sql.planner.plan.Patterns.topN; +import static io.trino.sql.planner.plan.TopNNode.Step.PARTIAL; +import static io.trino.sql.planner.plan.TopNNode.Step.SINGLE; public class PushTopNIntoTableScan implements Rule { private static final Capture TABLE_SCAN = newCapture(); - // Currently the rule is applied at the optimization phase where PARTIAL and FINAL TopNNode do not exist. - // The rule can be further made to work with PARTIAL and FINAL if needed. + // Rule is executed in two planning phases. Initially we try to pushdown SINGLE TopN into + // table scan. If that fails, we repeat the exercise for PARTIAL TopN nodes after SINGLE -> PARTIAL/FINAL split. private static final Pattern PATTERN = topN() - .matching(node -> node.getStep().equals(Step.SINGLE)) + .matching(node -> node.getStep() == SINGLE || node.getStep() == PARTIAL) .with(source().matching(tableScan().capturedAs(TABLE_SCAN))); private final Metadata metadata; @@ -91,6 +92,18 @@ public Result apply(TopNNode topNNode, Captures captures, Context context) // table scan partitioning might have changed with new table handle Optional.empty()); + // If possible we are getting rid of TopN node. + // + // If we are operating in `SINGLE` step and connector + // TopN pushdown is guaranteed we are removing TopN node from plan altogether. + + // For PARTIAL step it would be semantically correct to always drop TopN node from the plan, no matter if connector + // declares pushdown as guaranteed or not. But we decided to leave it in the plan for non-guaranteed pushdown, as there is no way + // to determine the size of output returned by connector. If connector pushdown support is very limited, and still a lot of data is returned + // after pushdown, removing PARTIAL TopN node would make query execution significantly more expensive. + // + // FINAL step of TopN node is never removed as it is needed to perform final filter higher in the query execution. + if (!result.isTopNGuaranteed()) { node = topNNode.replaceChildren(ImmutableList.of(node)); } diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/iterative/rule/TestPushTopNIntoTableScan.java b/core/trino-main/src/test/java/io/trino/sql/planner/iterative/rule/TestPushTopNIntoTableScan.java index 1ed8981d5bfb..0e45ff33e0a5 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/iterative/rule/TestPushTopNIntoTableScan.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/iterative/rule/TestPushTopNIntoTableScan.java @@ -170,6 +170,74 @@ dimensionName, equalTo(dimensionColumn), } } + @Test + public void testPushPartialTopNIntoTableScan() + { + try (RuleTester ruleTester = defaultRuleTester()) { + MockConnectorTableHandle connectorHandle = new MockConnectorTableHandle(TEST_SCHEMA_TABLE); + // make the mock connector return a new connectorHandle + MockConnectorFactory.ApplyTopN applyTopN = + (session, handle, topNCount, sortItems, tableAssignments) -> Optional.of(new TopNApplicationResult<>(connectorHandle, true)); + MockConnectorFactory mockFactory = createMockFactory(assignments, Optional.of(applyTopN)); + + ruleTester.getQueryRunner().createCatalog(MOCK_CATALOG, mockFactory, ImmutableMap.of()); + + ruleTester.assertThat(new PushTopNIntoTableScan(ruleTester.getMetadata())) + .on(p -> { + Symbol dimension = p.symbol(dimensionName, VARCHAR); + Symbol metric = p.symbol(metricName, BIGINT); + return p.topN(1, ImmutableList.of(dimension), TopNNode.Step.PARTIAL, + p.tableScan(TEST_TABLE_HANDLE, + ImmutableList.of(dimension, metric), + ImmutableMap.of( + dimension, dimensionColumn, + metric, metricColumn))); + }) + .withSession(MOCK_SESSION) + .matches( + tableScan( + equalTo(connectorHandle), + TupleDomain.all(), + new HashMap<>())); + } + } + + @Test + public void testPushPartialTopNIntoTableScanNotGuaranteed() + { + try (RuleTester ruleTester = defaultRuleTester()) { + MockConnectorTableHandle connectorHandle = new MockConnectorTableHandle(TEST_SCHEMA_TABLE); + // make the mock connector return a new connectorHandle + MockConnectorFactory.ApplyTopN applyTopN = + (session, handle, topNCount, sortItems, tableAssignments) -> Optional.of(new TopNApplicationResult<>(connectorHandle, false)); + MockConnectorFactory mockFactory = createMockFactory(assignments, Optional.of(applyTopN)); + + ruleTester.getQueryRunner().createCatalog(MOCK_CATALOG, mockFactory, ImmutableMap.of()); + + ruleTester.assertThat(new PushTopNIntoTableScan(ruleTester.getMetadata())) + .on(p -> { + Symbol dimension = p.symbol(dimensionName, VARCHAR); + Symbol metric = p.symbol(metricName, BIGINT); + return p.topN(1, ImmutableList.of(dimension), TopNNode.Step.PARTIAL, + p.tableScan(TEST_TABLE_HANDLE, + ImmutableList.of(dimension, metric), + ImmutableMap.of( + dimension, dimensionColumn, + metric, metricColumn))); + }) + .withSession(MOCK_SESSION) + .matches( + topN(1, ImmutableList.of(sort(dimensionName, ASCENDING, FIRST)), + TopNNode.Step.PARTIAL, + tableScan( + equalTo(connectorHandle), + TupleDomain.all(), + ImmutableMap.of( + dimensionName, equalTo(dimensionColumn), + metricName, equalTo(metricColumn))))); + } + } + private MockConnectorFactory createMockFactory(Map assignments, Optional applyTopN) { List metadata = assignments.entrySet().stream() diff --git a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java index 50db9cada8ff..d5af8ad47e97 100644 --- a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java +++ b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java @@ -19,6 +19,7 @@ import io.trino.sql.planner.assertions.PlanMatchPattern; import io.trino.sql.planner.plan.ExchangeNode; import io.trino.sql.planner.plan.JoinNode; +import io.trino.sql.planner.plan.ProjectNode; import io.trino.sql.planner.plan.TableScanNode; import io.trino.sql.planner.plan.TopNNode; import io.trino.sql.query.QueryAssertions.QueryAssert; @@ -171,6 +172,18 @@ public void testTopNPushdown() .ordered() .isFullyPushedDown(); } + + // TopN over LEFT join (enforces SINGLE TopN cannot be pushed below OUTER side of join) + // We expect PARTIAL TopN on the LEFT side of join to be pushed down. + assertThat(query("SELECT * " + + "FROM nation n LEFT JOIN region r ON n.regionkey = r.regionkey " + + "ORDER BY n.nationkey LIMIT 3")) + .ordered() + .isNotFullyPushedDown( + node(TopNNode.class, // FINAL TopN + anyTree(node(JoinNode.class, + node(ExchangeNode.class, node(ProjectNode.class, node(TableScanNode.class))), // no PARTIAL TopN + anyTree(node(TableScanNode.class)))))); } @Test From 5c8ae0a61721d3ebaa5a8afc5bd5822b6ec77973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Osipiuk?= Date: Mon, 12 Apr 2021 13:46:16 +0100 Subject: [PATCH 118/146] Use .trino as hidden files prefix in file hive metastore --- .../SemiTransactionalHiveMetastore.java | 8 ++++---- .../metastore/file/FileHiveMetastore.java | 20 +++++++++---------- .../trino/plugin/hive/AbstractTestHive.java | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java index e8fe14dba0f7..67d6dceb0fef 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java @@ -2417,8 +2417,8 @@ private static RecursiveDeleteResult recursiveDeleteFiles(HdfsEnvironment hdfsEn private static RecursiveDeleteResult doRecursiveDeleteFiles(FileSystem fileSystem, Path directory, Set queryIds, boolean deleteEmptyDirectories) { - // don't delete hidden presto directories - if (directory.getName().startsWith(".presto")) { + // don't delete hidden Trino directories use by FileHiveMetastore + if (directory.getName().startsWith(".trino")) { return new RecursiveDeleteResult(false, ImmutableList.of()); } @@ -2439,8 +2439,8 @@ private static RecursiveDeleteResult doRecursiveDeleteFiles(FileSystem fileSyste Path filePath = fileStatus.getPath(); String fileName = filePath.getName(); boolean eligible = false; - // never delete presto dot files - if (!fileName.startsWith(".presto")) { + // don't delete hidden Trino directories use by FileHiveMetastore + if (!fileName.startsWith(".trino")) { eligible = queryIds.stream().anyMatch(id -> fileName.startsWith(id) || fileName.endsWith(id)); } if (eligible) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java index 21db80f79be2..563bd35aff55 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java @@ -130,8 +130,8 @@ public class FileHiveMetastore { private static final String PUBLIC_ROLE_NAME = "public"; private static final String ADMIN_ROLE_NAME = "admin"; - private static final String PRESTO_SCHEMA_FILE_NAME = ".prestoSchema"; - private static final String PRESTO_PERMISSIONS_DIRECTORY_NAME = ".prestoPermissions"; + private static final String TRINO_SCHEMA_FILE_NAME = ".trinoSchema"; + private static final String TRINO_PERMISSIONS_DIRECTORY_NAME = ".trinoPermissions"; // todo there should be a way to manage the admins list private static final Set ADMIN_USERS = ImmutableSet.of("admin", "hive", "hdfs"); private static final String ICEBERG_TABLE_TYPE_NAME = "table_type"; @@ -721,7 +721,7 @@ public synchronized void addPartitions(HiveIdentity identity, String databaseNam Partition partition = partitionWithStatistics.getPartition(); verifiedPartition(table, partition); Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues()); - Path schemaPath = new Path(partitionMetadataDirectory, PRESTO_SCHEMA_FILE_NAME); + Path schemaPath = new Path(partitionMetadataDirectory, TRINO_SCHEMA_FILE_NAME); if (metadataFileSystem.exists(schemaPath)) { throw new TrinoException(HIVE_METASTORE_ERROR, "Partition already exists"); } @@ -998,7 +998,7 @@ private synchronized Optional> getAllPartitionNames(HiveIdentity id private boolean isValidPartition(Table table, String partitionName) { try { - return metadataFileSystem.exists(new Path(getPartitionMetadataDirectory(table, partitionName), PRESTO_SCHEMA_FILE_NAME)); + return metadataFileSystem.exists(new Path(getPartitionMetadataDirectory(table, partitionName), TRINO_SCHEMA_FILE_NAME)); } catch (IOException e) { return false; @@ -1218,7 +1218,7 @@ private Path getPartitionMetadataDirectory(Table table, String partitionName) private Path getPermissionsDirectory(Table table) { - return new Path(getTableMetadataDirectory(table), PRESTO_PERMISSIONS_DIRECTORY_NAME); + return new Path(getTableMetadataDirectory(table), TRINO_PERMISSIONS_DIRECTORY_NAME); } private static Path getPermissionsPath(Path permissionsDirectory, HivePrincipal grantee) @@ -1242,7 +1242,7 @@ private List getChildSchemaDirectories(Path metadataDirectory) if (childPath.getName().startsWith(".")) { continue; } - if (metadataFileSystem.isFile(new Path(childPath, PRESTO_SCHEMA_FILE_NAME))) { + if (metadataFileSystem.isFile(new Path(childPath, TRINO_SCHEMA_FILE_NAME))) { childSchemaDirectories.add(childPath); } } @@ -1287,7 +1287,7 @@ private Set readAllPermissions(Path permissionsDirectory) private void deleteMetadataDirectory(Path metadataDirectory) { try { - Path schemaPath = new Path(metadataDirectory, PRESTO_SCHEMA_FILE_NAME); + Path schemaPath = new Path(metadataDirectory, TRINO_SCHEMA_FILE_NAME); if (!metadataFileSystem.isFile(schemaPath)) { // if there is no schema file, assume this is not a database, partition or table return; @@ -1324,7 +1324,7 @@ private void checkVersion(Optional writerVersion) private Optional readSchemaFile(String type, Path metadataDirectory, JsonCodec codec) { - Path schemaPath = new Path(metadataDirectory, PRESTO_SCHEMA_FILE_NAME); + Path schemaPath = new Path(metadataDirectory, TRINO_SCHEMA_FILE_NAME); return readFile(type + " schema", schemaPath, codec); } @@ -1347,7 +1347,7 @@ private Optional readFile(String type, Path path, JsonCodec codec) private void writeSchemaFile(String type, Path directory, JsonCodec codec, T value, boolean overwrite) { - Path schemaPath = new Path(directory, PRESTO_SCHEMA_FILE_NAME); + Path schemaPath = new Path(directory, TRINO_SCHEMA_FILE_NAME); writeFile(type + " schema", schemaPath, codec, value, overwrite); } @@ -1377,7 +1377,7 @@ private void writeFile(String type, Path path, JsonCodec codec, T value, private void deleteSchemaFile(String type, Path metadataDirectory) { try { - if (!metadataFileSystem.delete(new Path(metadataDirectory, PRESTO_SCHEMA_FILE_NAME), false)) { + if (!metadataFileSystem.delete(new Path(metadataDirectory, TRINO_SCHEMA_FILE_NAME), false)) { throw new TrinoException(HIVE_METASTORE_ERROR, "Could not delete " + type + " schema"); } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java index a7e0212a00e9..9a5e23b9f83b 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java @@ -3926,7 +3926,7 @@ protected Set listAllDataFiles(HdfsContext context, Path path) FileSystem fileSystem = hdfsEnvironment.getFileSystem(context, path); if (fileSystem.exists(path)) { for (FileStatus fileStatus : fileSystem.listStatus(path)) { - if (fileStatus.getPath().getName().startsWith(".presto")) { + if (fileStatus.getPath().getName().startsWith(".trino")) { // skip hidden files } else if (fileStatus.isFile()) { @@ -5021,7 +5021,7 @@ private List listDirectory(HdfsContext context, Path path) return Arrays.stream(fileSystem.listStatus(path)) .map(FileStatus::getPath) .map(Path::getName) - .filter(name -> !name.startsWith(".presto")) + .filter(name -> !name.startsWith(".trino")) .collect(toList()); } From 5435fc90c34e4cc936aefacd5d32f3db7521192e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Walkiewicz?= Date: Tue, 13 Apr 2021 17:20:54 +0200 Subject: [PATCH 119/146] Extract security config from Kafka config --- .../plugin/kafka/KafkaClientsModule.java | 4 +- .../io/trino/plugin/kafka/KafkaConfig.java | 16 ------- .../plugin/kafka/KafkaSecurityConfig.java | 39 +++++++++++++++ .../kafka/PlainTextKafkaAdminFactory.java | 7 ++- .../kafka/PlainTextKafkaConsumerFactory.java | 7 ++- .../kafka/PlainTextKafkaProducerFactory.java | 5 +- .../kafka/security/KafkaSecurityModule.java | 4 +- .../trino/plugin/kafka/TestKafkaConfig.java | 8 +--- .../plugin/kafka/TestKafkaSecurityConfig.java | 47 +++++++++++++++++++ 9 files changed, 106 insertions(+), 31 deletions(-) create mode 100644 plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaSecurityConfig.java create mode 100644 plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaSecurityConfig.java diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaClientsModule.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaClientsModule.java index 9b8c0c8411ef..34bfec9ef339 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaClientsModule.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaClientsModule.java @@ -21,6 +21,7 @@ import io.trino.plugin.kafka.security.SecurityProtocol; import static io.airlift.configuration.ConditionalModule.installModuleIf; +import static io.airlift.configuration.ConfigBinder.configBinder; public class KafkaClientsModule extends AbstractConfigurationAwareModule @@ -28,6 +29,7 @@ public class KafkaClientsModule @Override protected void setup(Binder binder) { + configBinder(binder).bindConfig(KafkaSecurityConfig.class); installClientModule(SecurityProtocol.PLAINTEXT, KafkaClientsModule::configurePlainText); installClientModule(SecurityProtocol.SSL, KafkaClientsModule::configureSsl); } @@ -35,7 +37,7 @@ protected void setup(Binder binder) private void installClientModule(SecurityProtocol securityProtocol, Module module) { install(installModuleIf( - KafkaConfig.class, + KafkaSecurityConfig.class, config -> config.getSecurityProtocol().equals(securityProtocol), module)); } diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConfig.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConfig.java index b348403e5d76..78e24db15428 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConfig.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConfig.java @@ -21,7 +21,6 @@ import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; import io.trino.plugin.kafka.schema.file.FileTableDescriptionSupplier; -import io.trino.plugin.kafka.security.SecurityProtocol; import io.trino.spi.HostAddress; import javax.validation.constraints.Min; @@ -32,7 +31,6 @@ import java.util.stream.StreamSupport; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static io.trino.plugin.kafka.security.SecurityProtocol.PLAINTEXT; @DefunctConfig("kafka.connect-timeout") public class KafkaConfig @@ -46,7 +44,6 @@ public class KafkaConfig private int messagesPerSplit = 100_000; private boolean timestampUpperBoundPushDownEnabled; private String tableDescriptionSupplier = FileTableDescriptionSupplier.NAME; - private SecurityProtocol securityProtocol = PLAINTEXT; @Size(min = 1) public Set getNodes() @@ -155,17 +152,4 @@ public KafkaConfig setTimestampUpperBoundPushDownEnabled(boolean timestampUpperB this.timestampUpperBoundPushDownEnabled = timestampUpperBoundPushDownEnabled; return this; } - - @Config("kafka.security-protocol") - @ConfigDescription("Security protocol used for Kafka connection") - public KafkaConfig setSecurityProtocol(SecurityProtocol securityProtocol) - { - this.securityProtocol = securityProtocol; - return this; - } - - public SecurityProtocol getSecurityProtocol() - { - return securityProtocol; - } } diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaSecurityConfig.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaSecurityConfig.java new file mode 100644 index 000000000000..0a6c11dcc49e --- /dev/null +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaSecurityConfig.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kafka; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import io.trino.plugin.kafka.security.SecurityProtocol; + +import javax.validation.constraints.NotNull; + +public class KafkaSecurityConfig +{ + private SecurityProtocol securityProtocol = SecurityProtocol.PLAINTEXT; + + @NotNull + public SecurityProtocol getSecurityProtocol() + { + return securityProtocol; + } + + @Config("kafka.security-protocol") + @ConfigDescription("Kafka communication security protocol") + public KafkaSecurityConfig setSecurityProtocol(SecurityProtocol securityProtocol) + { + this.securityProtocol = securityProtocol; + return this; + } +} diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaAdminFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaAdminFactory.java index b34f911d129b..b53b8dbb7308 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaAdminFactory.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaAdminFactory.java @@ -35,12 +35,15 @@ public class PlainTextKafkaAdminFactory private final SecurityProtocol securityProtocol; @Inject - public PlainTextKafkaAdminFactory(KafkaConfig kafkaConfig) + public PlainTextKafkaAdminFactory( + KafkaConfig kafkaConfig, + KafkaSecurityConfig securityConfig) { requireNonNull(kafkaConfig, "kafkaConfig is null"); + requireNonNull(securityConfig, "securityConfig is null"); nodes = kafkaConfig.getNodes(); - securityProtocol = kafkaConfig.getSecurityProtocol(); + securityProtocol = securityConfig.getSecurityProtocol(); } @Override diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaConsumerFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaConsumerFactory.java index 232e7725524d..45e4b758dfc0 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaConsumerFactory.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaConsumerFactory.java @@ -41,13 +41,16 @@ public class PlainTextKafkaConsumerFactory private final SecurityProtocol securityProtocol; @Inject - public PlainTextKafkaConsumerFactory(KafkaConfig kafkaConfig) + public PlainTextKafkaConsumerFactory( + KafkaConfig kafkaConfig, + KafkaSecurityConfig securityConfig) { requireNonNull(kafkaConfig, "kafkaConfig is null"); + requireNonNull(securityConfig, "securityConfig is null"); nodes = kafkaConfig.getNodes(); kafkaBufferSize = kafkaConfig.getKafkaBufferSize(); - securityProtocol = kafkaConfig.getSecurityProtocol(); + securityProtocol = securityConfig.getSecurityProtocol(); } @Override diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaProducerFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaProducerFactory.java index 0412fb65f0be..9f4293b105d7 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaProducerFactory.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaProducerFactory.java @@ -39,12 +39,13 @@ public class PlainTextKafkaProducerFactory private final SecurityProtocol securityProtocol; @Inject - public PlainTextKafkaProducerFactory(KafkaConfig kafkaConfig) + public PlainTextKafkaProducerFactory(KafkaConfig kafkaConfig, KafkaSecurityConfig securityConfig) { requireNonNull(kafkaConfig, "kafkaConfig is null"); + requireNonNull(securityConfig, "securityConfig is null"); nodes = kafkaConfig.getNodes(); - securityProtocol = kafkaConfig.getSecurityProtocol(); + securityProtocol = securityConfig.getSecurityProtocol(); } @Override diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSecurityModule.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSecurityModule.java index 01707c412280..91afb2d681ff 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSecurityModule.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSecurityModule.java @@ -16,7 +16,7 @@ import com.google.inject.Binder; import com.google.inject.Module; import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.trino.plugin.kafka.KafkaConfig; +import io.trino.plugin.kafka.KafkaSecurityConfig; import static io.airlift.configuration.ConditionalModule.installModuleIf; import static io.airlift.configuration.ConfigurationModule.installModules; @@ -35,7 +35,7 @@ protected void setup(Binder binder) private void bindSecurityModule(SecurityProtocol securityProtocol, Module module) { install(installModuleIf( - KafkaConfig.class, + KafkaSecurityConfig.class, config -> config.getSecurityProtocol().equals(securityProtocol), module)); } diff --git a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaConfig.java b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaConfig.java index ea95d14570f1..ffa1e0ec83ed 100644 --- a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaConfig.java +++ b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaConfig.java @@ -15,7 +15,6 @@ import com.google.common.collect.ImmutableMap; import io.trino.plugin.kafka.schema.file.FileTableDescriptionSupplier; -import io.trino.plugin.kafka.security.SecurityProtocol; import org.testng.annotations.Test; import java.util.Map; @@ -36,8 +35,7 @@ public void testDefaults() .setTableDescriptionSupplier(FileTableDescriptionSupplier.NAME) .setHideInternalColumns(true) .setMessagesPerSplit(100_000) - .setTimestampUpperBoundPushDownEnabled(false) - .setSecurityProtocol(SecurityProtocol.PLAINTEXT)); + .setTimestampUpperBoundPushDownEnabled(false)); } @Test @@ -51,7 +49,6 @@ public void testExplicitPropertyMappings() .put("kafka.hide-internal-columns", "false") .put("kafka.messages-per-split", "1") .put("kafka.timestamp-upper-bound-force-push-down-enabled", "true") - .put("kafka.security-protocol", "SSL") .build(); KafkaConfig expected = new KafkaConfig() @@ -61,8 +58,7 @@ public void testExplicitPropertyMappings() .setKafkaBufferSize("1MB") .setHideInternalColumns(false) .setMessagesPerSplit(1) - .setTimestampUpperBoundPushDownEnabled(true) - .setSecurityProtocol(SecurityProtocol.SSL); + .setTimestampUpperBoundPushDownEnabled(true); assertFullMapping(properties, expected); } diff --git a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaSecurityConfig.java b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaSecurityConfig.java new file mode 100644 index 000000000000..3545d457405a --- /dev/null +++ b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaSecurityConfig.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.kafka; + +import com.google.common.collect.ImmutableMap; +import io.trino.plugin.kafka.security.SecurityProtocol; +import org.testng.annotations.Test; + +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestKafkaSecurityConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(KafkaSecurityConfig.class) + .setSecurityProtocol(SecurityProtocol.PLAINTEXT)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("kafka.security-protocol", "SSL") + .build(); + + KafkaSecurityConfig expected = new KafkaSecurityConfig() + .setSecurityProtocol(SecurityProtocol.SSL); + + assertFullMapping(properties, expected); + } +} From c0dbffe592f827ed512fa2c9416fe50d9a5fcfe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Walkiewicz?= Date: Tue, 13 Apr 2021 22:43:14 +0200 Subject: [PATCH 120/146] Remove redundant module binding --- .../main/java/io/trino/plugin/kafka/KafkaConnectorFactory.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorFactory.java index 85d747270032..ddecd4810555 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorFactory.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorFactory.java @@ -17,7 +17,6 @@ import com.google.inject.Module; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; -import io.trino.plugin.kafka.security.KafkaSecurityModule; import io.trino.spi.NodeManager; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; @@ -60,7 +59,6 @@ public Connector create(String catalogName, Map config, Connecto Bootstrap app = new Bootstrap( new JsonModule(), new KafkaConnectorModule(), - new KafkaSecurityModule(), extension, binder -> { binder.bind(ClassLoader.class).toInstance(KafkaConnectorFactory.class.getClassLoader()); From c9b64f040c80720cd56372b69fd5c7b4989d680d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Walkiewicz?= Date: Tue, 13 Apr 2021 22:43:47 +0200 Subject: [PATCH 121/146] Use security protocol enum from Kafka library --- .../plugin/kafka/KafkaClientsModule.java | 2 +- .../plugin/kafka/KafkaSecurityConfig.java | 18 +++++++- .../kafka/PlainTextKafkaAdminFactory.java | 2 +- .../kafka/PlainTextKafkaConsumerFactory.java | 2 +- .../kafka/PlainTextKafkaProducerFactory.java | 2 +- .../kafka/security/KafkaSecurityModule.java | 1 + .../kafka/security/SecurityProtocol.java | 20 --------- .../plugin/kafka/TestKafkaSecurityConfig.java | 42 +++++++++++++++++-- 8 files changed, 60 insertions(+), 29 deletions(-) delete mode 100644 plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/SecurityProtocol.java diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaClientsModule.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaClientsModule.java index 34bfec9ef339..95c2f614703e 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaClientsModule.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaClientsModule.java @@ -18,7 +18,7 @@ import com.google.inject.Scopes; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.plugin.kafka.security.ForKafkaSsl; -import io.trino.plugin.kafka.security.SecurityProtocol; +import org.apache.kafka.common.security.auth.SecurityProtocol; import static io.airlift.configuration.ConditionalModule.installModuleIf; import static io.airlift.configuration.ConfigBinder.configBinder; diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaSecurityConfig.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaSecurityConfig.java index 0a6c11dcc49e..e806cc97002b 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaSecurityConfig.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaSecurityConfig.java @@ -15,13 +15,19 @@ import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; -import io.trino.plugin.kafka.security.SecurityProtocol; +import org.apache.kafka.common.security.auth.SecurityProtocol; +import javax.annotation.PostConstruct; import javax.validation.constraints.NotNull; +import static com.google.common.base.Preconditions.checkState; +import static java.lang.String.format; +import static org.apache.kafka.common.security.auth.SecurityProtocol.PLAINTEXT; +import static org.apache.kafka.common.security.auth.SecurityProtocol.SSL; + public class KafkaSecurityConfig { - private SecurityProtocol securityProtocol = SecurityProtocol.PLAINTEXT; + private SecurityProtocol securityProtocol = PLAINTEXT; @NotNull public SecurityProtocol getSecurityProtocol() @@ -36,4 +42,12 @@ public KafkaSecurityConfig setSecurityProtocol(SecurityProtocol securityProtocol this.securityProtocol = securityProtocol; return this; } + + @PostConstruct + public void validate() + { + checkState( + securityProtocol.equals(PLAINTEXT) || securityProtocol.equals(SSL), + format("Only %s and %s security protocols are supported", PLAINTEXT, SSL)); + } } diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaAdminFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaAdminFactory.java index b53b8dbb7308..0b710f31929e 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaAdminFactory.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaAdminFactory.java @@ -14,9 +14,9 @@ package io.trino.plugin.kafka; -import io.trino.plugin.kafka.security.SecurityProtocol; import io.trino.spi.HostAddress; import io.trino.spi.connector.ConnectorSession; +import org.apache.kafka.common.security.auth.SecurityProtocol; import javax.inject.Inject; diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaConsumerFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaConsumerFactory.java index 45e4b758dfc0..10580d4919d5 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaConsumerFactory.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaConsumerFactory.java @@ -14,9 +14,9 @@ package io.trino.plugin.kafka; import io.airlift.units.DataSize; -import io.trino.plugin.kafka.security.SecurityProtocol; import io.trino.spi.HostAddress; import io.trino.spi.connector.ConnectorSession; +import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.serialization.ByteArrayDeserializer; import javax.inject.Inject; diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaProducerFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaProducerFactory.java index 9f4293b105d7..5fb358bbc5f0 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaProducerFactory.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/PlainTextKafkaProducerFactory.java @@ -13,9 +13,9 @@ */ package io.trino.plugin.kafka; -import io.trino.plugin.kafka.security.SecurityProtocol; import io.trino.spi.HostAddress; import io.trino.spi.connector.ConnectorSession; +import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.serialization.ByteArraySerializer; import javax.inject.Inject; diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSecurityModule.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSecurityModule.java index 91afb2d681ff..d1cdd46c4982 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSecurityModule.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/KafkaSecurityModule.java @@ -17,6 +17,7 @@ import com.google.inject.Module; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.plugin.kafka.KafkaSecurityConfig; +import org.apache.kafka.common.security.auth.SecurityProtocol; import static io.airlift.configuration.ConditionalModule.installModuleIf; import static io.airlift.configuration.ConfigurationModule.installModules; diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/SecurityProtocol.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/SecurityProtocol.java deleted file mode 100644 index db4dad45a62c..000000000000 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/security/SecurityProtocol.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kafka.security; - -public enum SecurityProtocol -{ - PLAINTEXT, - SSL -} diff --git a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaSecurityConfig.java b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaSecurityConfig.java index 3545d457405a..c4ba1284b19d 100644 --- a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaSecurityConfig.java +++ b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaSecurityConfig.java @@ -14,7 +14,8 @@ package io.trino.plugin.kafka; import com.google.common.collect.ImmutableMap; -import io.trino.plugin.kafka.security.SecurityProtocol; +import org.apache.kafka.common.security.auth.SecurityProtocol; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.util.Map; @@ -22,6 +23,11 @@ import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static org.apache.kafka.common.security.auth.SecurityProtocol.PLAINTEXT; +import static org.apache.kafka.common.security.auth.SecurityProtocol.SASL_PLAINTEXT; +import static org.apache.kafka.common.security.auth.SecurityProtocol.SASL_SSL; +import static org.apache.kafka.common.security.auth.SecurityProtocol.SSL; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestKafkaSecurityConfig { @@ -29,7 +35,7 @@ public class TestKafkaSecurityConfig public void testDefaults() { assertRecordedDefaults(recordDefaults(KafkaSecurityConfig.class) - .setSecurityProtocol(SecurityProtocol.PLAINTEXT)); + .setSecurityProtocol(PLAINTEXT)); } @Test @@ -40,8 +46,38 @@ public void testExplicitPropertyMappings() .build(); KafkaSecurityConfig expected = new KafkaSecurityConfig() - .setSecurityProtocol(SecurityProtocol.SSL); + .setSecurityProtocol(SSL); assertFullMapping(properties, expected); } + + @Test(dataProvider = "validSecurityProtocols") + public void testValidSecurityProtocols(SecurityProtocol securityProtocol) + { + new KafkaSecurityConfig() + .setSecurityProtocol(securityProtocol) + .validate(); + } + + @DataProvider(name = "validSecurityProtocols") + public Object[][] validSecurityProtocols() + { + return new Object[][] {{PLAINTEXT}, {SSL}}; + } + + @Test(dataProvider = "invalidSecurityProtocols") + public void testInvalidSecurityProtocol(SecurityProtocol securityProtocol) + { + assertThatThrownBy(() -> new KafkaSecurityConfig() + .setSecurityProtocol(securityProtocol) + .validate()) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Only PLAINTEXT and SSL security protocols are supported"); + } + + @DataProvider(name = "invalidSecurityProtocols") + public Object[][] invalidSecurityProtocols() + { + return new Object[][] {{SASL_PLAINTEXT}, {SASL_SSL}}; + } } From 696838694ce17f621b89985d29a09163f6d3417c Mon Sep 17 00:00:00 2001 From: guhanjie Date: Fri, 19 Mar 2021 10:25:00 +0800 Subject: [PATCH 122/146] Add a flag for HttpRemoteTask to avoid eager but unnecessary sendUpdate It should not sendUpdate before task been started, while the needsUpdate flag in RemoteHttpTask is not enough to ensure this, which case will happen on all non-leaf stage, as 450-470 lines in SqlStageExecution.java Although task will start soon after noMoreSplits, but we should avoid this, which is unnecessary and not strictly accurate in principle. --- .../main/java/io/trino/server/remotetask/HttpRemoteTask.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/trino-main/src/main/java/io/trino/server/remotetask/HttpRemoteTask.java b/core/trino-main/src/main/java/io/trino/server/remotetask/HttpRemoteTask.java index ee09b68eca47..6318e5c67ba7 100644 --- a/core/trino-main/src/main/java/io/trino/server/remotetask/HttpRemoteTask.java +++ b/core/trino-main/src/main/java/io/trino/server/remotetask/HttpRemoteTask.java @@ -159,6 +159,7 @@ public final class HttpRemoteTask private final PartitionedSplitCountTracker partitionedSplitCountTracker; + private final AtomicBoolean started = new AtomicBoolean(false); private final AtomicBoolean aborting = new AtomicBoolean(false); public HttpRemoteTask( @@ -318,6 +319,7 @@ public void start() { try (SetThreadName ignored = new SetThreadName("HttpRemoteTask-%s", taskId)) { // to start we just need to trigger an update + started.set(true); scheduleUpdate(); dynamicFiltersFetcher.start(); @@ -513,7 +515,7 @@ private synchronized void sendUpdate() { TaskStatus taskStatus = getTaskStatus(); // don't update if the task hasn't been started yet or if it is already finished - if (!needsUpdate.get() || taskStatus.getState().isDone()) { + if (!started.get() || !needsUpdate.get() || taskStatus.getState().isDone()) { return; } From 184284c97a04e42406dc936c087cfa85aead90ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Kokosi=C5=84ski?= Date: Wed, 14 Apr 2021 12:55:08 +0200 Subject: [PATCH 123/146] Small cleanup in CachingJdbcClient - order fields - remove extra line - make things private and static where possible --- .../src/main/java/io/trino/plugin/jdbc/CachingJdbcClient.java | 3 +-- .../test/java/io/trino/plugin/jdbc/TestCachingJdbcClient.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/CachingJdbcClient.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/CachingJdbcClient.java index f58a9543aed5..9597a87c191c 100644 --- a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/CachingJdbcClient.java +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/CachingJdbcClient.java @@ -68,13 +68,13 @@ public class CachingJdbcClient private static final Duration CACHING_DISABLED = new Duration(0, MILLISECONDS); private final JdbcClient delegate; + private final List> sessionProperties; private final boolean cacheMissing; private final Cache> schemaNamesCache; private final Cache> tableNamesCache; private final Cache> tableHandleCache; private final Cache> columnsCache; - private final List> sessionProperties; private final Cache statisticsCache; @Inject @@ -89,7 +89,6 @@ public CachingJdbcClient(JdbcClient delegate, Set ses this.sessionProperties = requireNonNull(sessionPropertiesProviders, "sessionPropertiesProviders is null").stream() .flatMap(provider -> provider.getSessionProperties().stream()) .collect(toImmutableList()); - this.cacheMissing = cacheMissing; CacheBuilder cacheBuilder = CacheBuilder.newBuilder() diff --git a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestCachingJdbcClient.java b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestCachingJdbcClient.java index 00c933aca237..7687c8a6c918 100644 --- a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestCachingJdbcClient.java +++ b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestCachingJdbcClient.java @@ -65,7 +65,7 @@ public class TestCachingJdbcClient null, false)); - public static final Set SESSION_PROPERTIES_PROVIDERS = Set.of(() -> PROPERTY_METADATA); + private static final Set SESSION_PROPERTIES_PROVIDERS = Set.of(() -> PROPERTY_METADATA); private static final ConnectorSession SESSION = TestingConnectorSession.builder() .setPropertyMetadata(PROPERTY_METADATA) From e27fdf2706c58e2ccd9dcd5c06e4fb5283b9c74f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Kokosi=C5=84ski?= Date: Wed, 14 Apr 2021 12:55:09 +0200 Subject: [PATCH 124/146] Use same user key when no impersonation is used --- .../trino/plugin/jdbc/CachingJdbcClient.java | 53 ++++++--- ...dentialsBasedJdbcIdentityCacheMapping.java | 102 ++++++++++++++++++ .../plugin/jdbc/JdbcIdentityCacheMapping.java | 33 ++++++ .../plugin/jdbc/JdbcMetadataFactory.java | 7 +- .../SingletonJdbcIdentityCacheMapping.java | 42 ++++++++ .../credential/CredentialProviderModule.java | 4 + .../plugin/jdbc/TestCachingJdbcClient.java | 50 ++++++++- 7 files changed, 273 insertions(+), 18 deletions(-) create mode 100644 plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/ExtraCredentialsBasedJdbcIdentityCacheMapping.java create mode 100644 plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcIdentityCacheMapping.java create mode 100644 plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/SingletonJdbcIdentityCacheMapping.java diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/CachingJdbcClient.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/CachingJdbcClient.java index 9597a87c191c..3bbf15c84901 100644 --- a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/CachingJdbcClient.java +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/CachingJdbcClient.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.UncheckedExecutionException; import io.airlift.units.Duration; import io.trino.plugin.base.session.SessionPropertiesProvider; +import io.trino.plugin.jdbc.JdbcIdentityCacheMapping.JdbcIdentityCacheKey; import io.trino.spi.TrinoException; import io.trino.spi.connector.AggregateFunction; import io.trino.spi.connector.ColumnHandle; @@ -70,26 +71,37 @@ public class CachingJdbcClient private final JdbcClient delegate; private final List> sessionProperties; private final boolean cacheMissing; + private final JdbcIdentityCacheMapping identityMapping; - private final Cache> schemaNamesCache; + private final Cache> schemaNamesCache; private final Cache> tableNamesCache; private final Cache> tableHandleCache; private final Cache> columnsCache; private final Cache statisticsCache; @Inject - public CachingJdbcClient(@StatsCollecting JdbcClient delegate, Set sessionPropertiesProviders, BaseJdbcConfig config) + public CachingJdbcClient( + @StatsCollecting JdbcClient delegate, + Set sessionPropertiesProviders, + JdbcIdentityCacheMapping identityMapping, + BaseJdbcConfig config) { - this(delegate, sessionPropertiesProviders, config.getMetadataCacheTtl(), config.isCacheMissing()); + this(delegate, sessionPropertiesProviders, identityMapping, config.getMetadataCacheTtl(), config.isCacheMissing()); } - public CachingJdbcClient(JdbcClient delegate, Set sessionPropertiesProviders, Duration metadataCachingTtl, boolean cacheMissing) + public CachingJdbcClient( + JdbcClient delegate, + Set sessionPropertiesProviders, + JdbcIdentityCacheMapping identityMapping, + Duration metadataCachingTtl, + boolean cacheMissing) { this.delegate = requireNonNull(delegate, "delegate is null"); this.sessionProperties = requireNonNull(sessionPropertiesProviders, "sessionPropertiesProviders is null").stream() .flatMap(provider -> provider.getSessionProperties().stream()) .collect(toImmutableList()); this.cacheMissing = cacheMissing; + this.identityMapping = requireNonNull(identityMapping, "identityMapping is null"); CacheBuilder cacheBuilder = CacheBuilder.newBuilder() .expireAfterWrite(metadataCachingTtl.toMillis(), MILLISECONDS) @@ -117,14 +129,14 @@ public boolean schemaExists(ConnectorSession session, String schema) @Override public Set getSchemaNames(ConnectorSession session) { - JdbcIdentity key = JdbcIdentity.from(session); + JdbcIdentityCacheKey key = getIdentityKey(session); return get(schemaNamesCache, key, () -> delegate.getSchemaNames(session)); } @Override public List getTableNames(ConnectorSession session, Optional schema) { - TableNamesCacheKey key = new TableNamesCacheKey(JdbcIdentity.from(session), schema); + TableNamesCacheKey key = new TableNamesCacheKey(getIdentityKey(session), schema); return get(tableNamesCache, key, () -> delegate.getTableNames(session, schema)); } @@ -134,7 +146,7 @@ public List getColumns(ConnectorSession session, JdbcTableHand if (tableHandle.getColumns().isPresent()) { return tableHandle.getColumns().get(); } - ColumnsCacheKey key = new ColumnsCacheKey(JdbcIdentity.from(session), getSessionProperties(session), tableHandle.getRequiredNamedRelation().getSchemaTableName()); + ColumnsCacheKey key = new ColumnsCacheKey(getIdentityKey(session), getSessionProperties(session), tableHandle.getRequiredNamedRelation().getSchemaTableName()); return get(columnsCache, key, () -> delegate.getColumns(session, tableHandle)); } @@ -247,7 +259,7 @@ public boolean isLimitGuaranteed(ConnectorSession session) @Override public Optional getTableHandle(ConnectorSession session, SchemaTableName schemaTableName) { - TableHandleCacheKey key = new TableHandleCacheKey(JdbcIdentity.from(session), schemaTableName); + TableHandleCacheKey key = new TableHandleCacheKey(getIdentityKey(session), schemaTableName); Optional cachedTableHandle = tableHandleCache.getIfPresent(key); //noinspection OptionalAssignedToNull if (cachedTableHandle != null) { @@ -423,6 +435,11 @@ public Optional getTableScanRedirection(Conn return delegate.getTableScanRedirection(session, tableHandle); } + private JdbcIdentityCacheKey getIdentityKey(ConnectorSession session) + { + return identityMapping.getRemoteUserCacheKey(JdbcIdentity.from(session)); + } + private Map getSessionProperties(ConnectorSession session) { return sessionProperties.stream() @@ -453,6 +470,12 @@ private void invalidateColumnsCache(SchemaTableName table) invalidateCache(columnsCache, key -> key.table.equals(table)); } + @VisibleForTesting + CacheStats getTableNamesCacheStats() + { + return tableNamesCache.stats(); + } + @VisibleForTesting CacheStats getColumnsCacheStats() { @@ -476,18 +499,18 @@ private static void invalidateCache(Cache cache, Predicate filte private static final class ColumnsCacheKey { - private final JdbcIdentity identity; + private final JdbcIdentityCacheKey identity; private final SchemaTableName table; private final Map sessionProperties; - private ColumnsCacheKey(JdbcIdentity identity, Map sessionProperties, SchemaTableName table) + private ColumnsCacheKey(JdbcIdentityCacheKey identity, Map sessionProperties, SchemaTableName table) { this.identity = requireNonNull(identity, "identity is null"); this.sessionProperties = ImmutableMap.copyOf(requireNonNull(sessionProperties, "sessionProperties is null")); this.table = requireNonNull(table, "table is null"); } - public JdbcIdentity getIdentity() + public JdbcIdentityCacheKey getIdentity() { return identity; } @@ -526,10 +549,10 @@ public String toString() private static final class TableHandleCacheKey { - private final JdbcIdentity identity; + private final JdbcIdentityCacheKey identity; private final SchemaTableName tableName; - private TableHandleCacheKey(JdbcIdentity identity, SchemaTableName tableName) + private TableHandleCacheKey(JdbcIdentityCacheKey identity, SchemaTableName tableName) { this.identity = requireNonNull(identity, "identity is null"); this.tableName = requireNonNull(tableName, "tableName is null"); @@ -558,10 +581,10 @@ public int hashCode() private static final class TableNamesCacheKey { - private final JdbcIdentity identity; + private final JdbcIdentityCacheKey identity; private final Optional schemaName; - private TableNamesCacheKey(JdbcIdentity identity, Optional schemaName) + private TableNamesCacheKey(JdbcIdentityCacheKey identity, Optional schemaName) { this.identity = requireNonNull(identity, "identity is null"); this.schemaName = requireNonNull(schemaName, "schemaName is null"); diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/ExtraCredentialsBasedJdbcIdentityCacheMapping.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/ExtraCredentialsBasedJdbcIdentityCacheMapping.java new file mode 100644 index 000000000000..e87fdf987a46 --- /dev/null +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/ExtraCredentialsBasedJdbcIdentityCacheMapping.java @@ -0,0 +1,102 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.jdbc; + +import io.trino.plugin.jdbc.credential.ExtraCredentialConfig; + +import javax.inject.Inject; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; + +public final class ExtraCredentialsBasedJdbcIdentityCacheMapping + implements JdbcIdentityCacheMapping +{ + private final MessageDigest sha256; + private final Optional userCredentialName; + private final Optional passwordCredentialName; + + @Inject + public ExtraCredentialsBasedJdbcIdentityCacheMapping(ExtraCredentialConfig config) + { + try { + sha256 = MessageDigest.getInstance("SHA-256"); + } + catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + requireNonNull(config, "config is null"); + userCredentialName = config.getUserCredentialName(); + passwordCredentialName = config.getPasswordCredentialName(); + } + + @Override + public JdbcIdentityCacheKey getRemoteUserCacheKey(JdbcIdentity identity) + { + Map extraCredentials = identity.getExtraCredentials(); + return new ExtraCredentialsBasedJdbcIdentityCacheKey( + userCredentialName.map(extraCredentials::get) + .map(this::hash), + passwordCredentialName.map(extraCredentials::get) + .map(this::hash)); + } + + private byte[] hash(String value) + { + return sha256.digest(value.getBytes(UTF_8)); + } + + private static final class ExtraCredentialsBasedJdbcIdentityCacheKey + extends JdbcIdentityCacheKey + { + private static final byte[] EMPTY_BYTES = new byte[0]; + private final byte[] userHash; + private final byte[] passwordHash; + + public ExtraCredentialsBasedJdbcIdentityCacheKey(Optional userHash, Optional passwordHash) + { + this.userHash = requireNonNull(userHash, "userHash is null") + .orElse(EMPTY_BYTES); + this.passwordHash = requireNonNull(passwordHash, "passwordHash is null") + .orElse(EMPTY_BYTES); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ExtraCredentialsBasedJdbcIdentityCacheKey that = (ExtraCredentialsBasedJdbcIdentityCacheKey) o; + return Arrays.equals(userHash, that.userHash) && Arrays.equals(passwordHash, that.passwordHash); + } + + @Override + public int hashCode() + { + int result = Arrays.hashCode(userHash); + result = 31 * result + Arrays.hashCode(passwordHash); + return result; + } + } +} diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcIdentityCacheMapping.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcIdentityCacheMapping.java new file mode 100644 index 000000000000..b505d8b40684 --- /dev/null +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcIdentityCacheMapping.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.jdbc; + +public interface JdbcIdentityCacheMapping +{ + JdbcIdentityCacheKey getRemoteUserCacheKey(JdbcIdentity identity); + + /** + * This will be used as cache key for metadata. If {@link JdbcIdentity} content can influence the + * metadata then we should have {@link JdbcIdentityCacheKey} instance so + * we could cache proper metadata for given {@link JdbcIdentity}. + */ + abstract class JdbcIdentityCacheKey + { + @Override + public abstract int hashCode(); + + @Override + public abstract boolean equals(Object obj); + } +} diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadataFactory.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadataFactory.java index 1da11fa69d9d..192dafd9aa0c 100644 --- a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadataFactory.java +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcMetadataFactory.java @@ -39,6 +39,11 @@ public JdbcMetadata create() { // Session stays the same per transaction, therefore session properties don't need to // be a part of cache keys in CachingJdbcClient. - return new JdbcMetadata(new CachingJdbcClient(jdbcClient, Set.of(), new Duration(1, DAYS), true), allowDropTable); + return new JdbcMetadata(new CachingJdbcClient( + jdbcClient, + Set.of(), + new SingletonJdbcIdentityCacheMapping(), + new Duration(1, DAYS), true), + allowDropTable); } } diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/SingletonJdbcIdentityCacheMapping.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/SingletonJdbcIdentityCacheMapping.java new file mode 100644 index 000000000000..5a89060ce007 --- /dev/null +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/SingletonJdbcIdentityCacheMapping.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.jdbc; + +public final class SingletonJdbcIdentityCacheMapping + implements JdbcIdentityCacheMapping +{ + @Override + public JdbcIdentityCacheKey getRemoteUserCacheKey(JdbcIdentity identity) + { + return SingletonJdbcIdentityCacheKey.INSTANCE; + } + + private static final class SingletonJdbcIdentityCacheKey + extends JdbcIdentityCacheKey + { + private static final SingletonJdbcIdentityCacheKey INSTANCE = new SingletonJdbcIdentityCacheKey(); + + @Override + public int hashCode() + { + return 0; + } + + @Override + public boolean equals(Object obj) + { + return obj instanceof SingletonJdbcIdentityCacheMapping; + } + } +} diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/credential/CredentialProviderModule.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/credential/CredentialProviderModule.java index 2318ec2f48ac..371e5133d08c 100644 --- a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/credential/CredentialProviderModule.java +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/credential/CredentialProviderModule.java @@ -16,8 +16,11 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; +import com.google.inject.Scopes; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.configuration.ConfigurationFactory; +import io.trino.plugin.jdbc.ExtraCredentialsBasedJdbcIdentityCacheMapping; +import io.trino.plugin.jdbc.JdbcIdentityCacheMapping; import io.trino.plugin.jdbc.credential.file.ConfigFileBasedCredentialProviderConfig; import io.trino.plugin.jdbc.credential.keystore.KeyStoreBasedCredentialProviderConfig; @@ -51,6 +54,7 @@ protected void setup(Binder binder) configBinder(binder).bindConfig(ExtraCredentialConfig.class); binder.bind(CredentialProvider.class).to(ExtraCredentialProvider.class).in(SINGLETON); + binder.bind(JdbcIdentityCacheMapping.class).to(ExtraCredentialsBasedJdbcIdentityCacheMapping.class).in(Scopes.SINGLETON); } private void bindCredentialProviderModule(CredentialProviderType name, Module module) diff --git a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestCachingJdbcClient.java b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestCachingJdbcClient.java index 7687c8a6c918..73c55c7c9241 100644 --- a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestCachingJdbcClient.java +++ b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/TestCachingJdbcClient.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableSet; import io.airlift.units.Duration; import io.trino.plugin.base.session.SessionPropertiesProvider; +import io.trino.plugin.jdbc.credential.ExtraCredentialConfig; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.ConnectorSession; @@ -26,6 +27,7 @@ import io.trino.spi.connector.SchemaTableName; import io.trino.spi.connector.TableNotFoundException; import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.security.ConnectorIdentity; import io.trino.spi.session.PropertyMetadata; import io.trino.spi.statistics.Estimate; import io.trino.spi.statistics.TableStatistics; @@ -92,7 +94,7 @@ public void setUp() private CachingJdbcClient createCachingJdbcClient(Duration cacheTtl, boolean cacheMissing) { - return new CachingJdbcClient(database.getJdbcClient(), SESSION_PROPERTIES_PROVIDERS, cacheTtl, cacheMissing); + return new CachingJdbcClient(database.getJdbcClient(), SESSION_PROPERTIES_PROVIDERS, new SingletonJdbcIdentityCacheMapping(), cacheTtl, cacheMissing); } private CachingJdbcClient createCachingJdbcClient(boolean cacheMissing) @@ -441,7 +443,7 @@ public TableStatistics getTableStatistics(ConnectorSession session, JdbcTableHan return NON_EMPTY_STATS; } }; - return new CachingJdbcClient(statsAwareJdbcClient, SESSION_PROPERTIES_PROVIDERS, duration, cacheMissing); + return new CachingJdbcClient(statsAwareJdbcClient, SESSION_PROPERTIES_PROVIDERS, new SingletonJdbcIdentityCacheMapping(), duration, cacheMissing); } @Test @@ -482,6 +484,36 @@ public void testGetTableStatisticsDoNotCacheEmptyWhenCachingMissingIsDisabled() this.jdbcClient.dropTable(SESSION, table); } + @Test + public void testDifferentIdentityKeys() + { + CachingJdbcClient cachingJdbcClient = new CachingJdbcClient( + database.getJdbcClient(), + SESSION_PROPERTIES_PROVIDERS, + new ExtraCredentialsBasedJdbcIdentityCacheMapping(new ExtraCredentialConfig() + .setUserCredentialName("user") + .setPasswordCredentialName("password")), + FOREVER, + true); + ConnectorSession alice = createUserSession("alice"); + ConnectorSession bob = createUserSession("bob"); + + JdbcTableHandle table = createTable(new SchemaTableName(schema, "table")); + + assertTableNamesCache(cachingJdbcClient).loads(2).misses(2).afterRunning(() -> { + assertThat(cachingJdbcClient.getTableNames(alice, Optional.empty())).contains(table.getRequiredNamedRelation().getSchemaTableName()); + assertThat(cachingJdbcClient.getTableNames(bob, Optional.empty())).contains(table.getRequiredNamedRelation().getSchemaTableName()); + }); + + assertTableNamesCache(cachingJdbcClient).hits(2).afterRunning(() -> { + assertThat(cachingJdbcClient.getTableNames(alice, Optional.empty())).contains(table.getRequiredNamedRelation().getSchemaTableName()); + assertThat(cachingJdbcClient.getTableNames(bob, Optional.empty())).contains(table.getRequiredNamedRelation().getSchemaTableName()); + }); + + // Drop tables by not using caching jdbc client + jdbcClient.dropTable(SESSION, table); + } + private JdbcTableHandle getAnyTable(String schema) { SchemaTableName tableName = jdbcClient.getTableNames(SESSION, Optional.of(schema)) @@ -516,6 +548,15 @@ private static ConnectorSession createSession(String sessionName) .build(); } + private static ConnectorSession createUserSession(String userName) + { + return builder() + .setIdentity(ConnectorIdentity.forUser(userName) + .withExtraCredentials(ImmutableMap.of("user", userName)) + .build()) + .build(); + } + @Test public void testEverythingImplemented() { @@ -534,6 +575,11 @@ private static Set nonOverriddenMethods() } } + private static CacheStatsAssertions assertTableNamesCache(CachingJdbcClient cachingJdbcClient) + { + return new CacheStatsAssertions(cachingJdbcClient::getTableNamesCacheStats); + } + private static CacheStatsAssertions assertColumnCacheStats(CachingJdbcClient client) { return new CacheStatsAssertions(client::getColumnsCacheStats); From c7c4550283c36d499644b57e6fe8cf98671819ec Mon Sep 17 00:00:00 2001 From: Raunaq Morarka Date: Tue, 13 Apr 2021 12:15:11 +0530 Subject: [PATCH 125/146] Extract OrderingScheme#toLocalProperties --- .../io/trino/sql/planner/OrderingScheme.java | 9 ++++++ .../planner/optimizations/AddExchanges.java | 11 ++----- .../optimizations/AddLocalExchanges.java | 6 +--- .../optimizations/PropertyDerivations.java | 30 ++++--------------- 4 files changed, 18 insertions(+), 38 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/OrderingScheme.java b/core/trino-main/src/main/java/io/trino/sql/planner/OrderingScheme.java index 9c2d514d11cf..61166b90befc 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/OrderingScheme.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/OrderingScheme.java @@ -18,7 +18,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import io.trino.spi.connector.LocalProperty; import io.trino.spi.connector.SortOrder; +import io.trino.spi.connector.SortingProperty; import io.trino.sql.tree.OrderBy; import io.trino.sql.tree.SortItem; import io.trino.sql.tree.SortItem.NullOrdering; @@ -138,4 +140,11 @@ public List toSortItems() io.trino.spi.connector.SortOrder.valueOf(getOrdering(symbol).name()))) .collect(toImmutableList()); } + + public List> toLocalProperties() + { + return getOrderBy().stream() + .map(symbol -> new SortingProperty<>(symbol, getOrdering(symbol))) + .collect(toImmutableList()); + } } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddExchanges.java b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddExchanges.java index 4842bde4544c..4e194edfee30 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddExchanges.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddExchanges.java @@ -25,7 +25,6 @@ import io.trino.metadata.Metadata; import io.trino.spi.connector.GroupingProperty; import io.trino.spi.connector.LocalProperty; -import io.trino.spi.connector.SortingProperty; import io.trino.spi.type.TypeOperators; import io.trino.sql.planner.DomainTranslator; import io.trino.sql.planner.Partitioning; @@ -293,10 +292,7 @@ public PlanWithProperties visitWindow(WindowNode node, PreferredProperties prefe if (!node.getPartitionBy().isEmpty()) { desiredProperties.add(new GroupingProperty<>(node.getPartitionBy())); } - node.getOrderingScheme().ifPresent(orderingScheme -> - orderingScheme.getOrderBy().stream() - .map(symbol -> new SortingProperty<>(symbol, orderingScheme.getOrdering(symbol))) - .forEach(desiredProperties::add)); + node.getOrderingScheme().ifPresent(orderingScheme -> desiredProperties.addAll(orderingScheme.toLocalProperties())); PlanWithProperties child = planChild( node, @@ -428,10 +424,7 @@ public PlanWithProperties visitSort(SortNode node, PreferredProperties preferred // current plan so far is single node, so local properties are effectively global properties // skip the SortNode if the local properties guarantee ordering on Sort keys // TODO: This should be extracted as a separate optimizer once the planner is able to reason about the ordering of each operator - List> desiredProperties = new ArrayList<>(); - for (Symbol symbol : node.getOrderingScheme().getOrderBy()) { - desiredProperties.add(new SortingProperty<>(symbol, node.getOrderingScheme().getOrdering(symbol))); - } + List> desiredProperties = node.getOrderingScheme().toLocalProperties(); if (LocalProperties.match(child.getProperties().getLocalProperties(), desiredProperties).stream() .noneMatch(Optional::isPresent)) { diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddLocalExchanges.java b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddLocalExchanges.java index c4f270836904..c92e1e1dcee2 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddLocalExchanges.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddLocalExchanges.java @@ -22,7 +22,6 @@ import io.trino.spi.connector.ConstantProperty; import io.trino.spi.connector.GroupingProperty; import io.trino.spi.connector.LocalProperty; -import io.trino.spi.connector.SortingProperty; import io.trino.spi.type.TypeOperators; import io.trino.sql.planner.Partitioning; import io.trino.sql.planner.PartitioningScheme; @@ -393,10 +392,7 @@ public PlanWithProperties visitWindow(WindowNode node, StreamPreferredProperties if (!node.getPartitionBy().isEmpty()) { desiredProperties.add(new GroupingProperty<>(node.getPartitionBy())); } - node.getOrderingScheme().ifPresent(orderingScheme -> - orderingScheme.getOrderBy().stream() - .map(symbol -> new SortingProperty<>(symbol, orderingScheme.getOrdering(symbol))) - .forEach(desiredProperties::add)); + node.getOrderingScheme().ifPresent(orderingScheme -> desiredProperties.addAll(orderingScheme.toLocalProperties())); Iterator>> matchIterator = LocalProperties.match(child.getProperties().getLocalProperties(), desiredProperties).iterator(); Set prePartitionedInputs = ImmutableSet.of(); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/PropertyDerivations.java b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/PropertyDerivations.java index 5e5de457db80..e7b8b2004383 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/PropertyDerivations.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/PropertyDerivations.java @@ -28,7 +28,6 @@ import io.trino.spi.connector.ConstantProperty; import io.trino.spi.connector.GroupingProperty; import io.trino.spi.connector.LocalProperty; -import io.trino.spi.connector.SortingProperty; import io.trino.spi.predicate.NullableValue; import io.trino.spi.type.Type; import io.trino.spi.type.TypeOperators; @@ -278,10 +277,7 @@ public ActualProperties visitWindow(WindowNode node, List inpu localProperties.add(new GroupingProperty<>(node.getPartitionBy())); } - orderingScheme.ifPresent(scheme -> - scheme.getOrderBy().stream() - .map(column -> new SortingProperty<>(column, scheme.getOrdering(column))) - .forEach(localProperties::add)); + orderingScheme.ifPresent(ordering -> localProperties.addAll(ordering.toLocalProperties())); return ActualProperties.builderFrom(properties) .local(LocalProperties.normalizeAndPrune(localProperties.build())) @@ -334,9 +330,7 @@ public ActualProperties visitTopNRanking(TopNRankingNode node, List> localProperties = ImmutableList.builder(); localProperties.add(new GroupingProperty<>(node.getPartitionBy())); - for (Symbol column : node.getOrderingScheme().getOrderBy()) { - localProperties.add(new SortingProperty<>(column, node.getOrderingScheme().getOrdering(column))); - } + localProperties.addAll(node.getOrderingScheme().toLocalProperties()); return ActualProperties.builderFrom(properties) .local(localProperties.build()) @@ -348,12 +342,8 @@ public ActualProperties visitTopN(TopNNode node, List inputPro { ActualProperties properties = Iterables.getOnlyElement(inputProperties); - List> localProperties = node.getOrderingScheme().getOrderBy().stream() - .map(column -> new SortingProperty<>(column, node.getOrderingScheme().getOrdering(column))) - .collect(toImmutableList()); - return ActualProperties.builderFrom(properties) - .local(localProperties) + .local(node.getOrderingScheme().toLocalProperties()) .build(); } @@ -362,12 +352,8 @@ public ActualProperties visitSort(SortNode node, List inputPro { ActualProperties properties = Iterables.getOnlyElement(inputProperties); - List> localProperties = node.getOrderingScheme().getOrderBy().stream() - .map(column -> new SortingProperty<>(column, node.getOrderingScheme().getOrdering(column))) - .collect(toImmutableList()); - return ActualProperties.builderFrom(properties) - .local(localProperties) + .local(node.getOrderingScheme().toLocalProperties()) .build(); } @@ -565,12 +551,8 @@ public ActualProperties visitExchange(ExchangeNode node, List Map constants = entries.stream() .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); - ImmutableList.Builder> localProperties = ImmutableList.builder(); - if (node.getOrderingScheme().isPresent()) { - node.getOrderingScheme().get().getOrderBy().stream() - .map(column -> new SortingProperty<>(column, node.getOrderingScheme().get().getOrdering(column))) - .forEach(localProperties::add); - } + ImmutableList.Builder> localProperties = ImmutableList.builder(); + node.getOrderingScheme().ifPresent(orderingScheme -> localProperties.addAll(orderingScheme.toLocalProperties())); // Local exchanges are only created in AddLocalExchanges, at the end of optimization, and // local exchanges do not produce all global properties as represented by ActualProperties. From d28bc9f15c62874cba7e50cafe34458e32866a56 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Sun, 11 Apr 2021 15:21:38 +0200 Subject: [PATCH 126/146] Upgrade Coral to 1.0.42 This brings no user-visible changes. --- plugin/trino-hive/pom.xml | 2 +- .../main/java/io/trino/plugin/hive/ViewReaderUtil.java | 9 +++++---- pom.xml | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/plugin/trino-hive/pom.xml b/plugin/trino-hive/pom.xml index 7a79f8ddc382..8c52068cce87 100644 --- a/plugin/trino-hive/pom.xml +++ b/plugin/trino-hive/pom.xml @@ -187,7 +187,7 @@ com.linkedin.coral - coral-presto + coral-trino diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/ViewReaderUtil.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/ViewReaderUtil.java index 24b11afea73b..3d0180b5223d 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/ViewReaderUtil.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/ViewReaderUtil.java @@ -15,7 +15,7 @@ import com.linkedin.coral.hive.hive2rel.HiveMetastoreClient; import com.linkedin.coral.hive.hive2rel.HiveToRelConverter; -import com.linkedin.coral.presto.rel2presto.RelToPrestoConverter; +import com.linkedin.coral.trino.rel2trino.RelToTrinoConverter; import io.airlift.json.JsonCodec; import io.airlift.json.JsonCodecFactory; import io.airlift.json.ObjectMapperProvider; @@ -163,15 +163,16 @@ public ConnectorViewDefinition decodeViewData(String viewSql, Table table, Catal try { HiveToRelConverter hiveToRelConverter = HiveToRelConverter.create(metastoreClient); RelNode rel = hiveToRelConverter.convertView(table.getDatabaseName(), table.getTableName()); - RelToPrestoConverter rel2Presto = new RelToPrestoConverter(); - String prestoSql = rel2Presto.convert(rel); + RelToTrinoConverter relToTrino = new RelToTrinoConverter(); + String trinoSql = relToTrino.convert(rel); RelDataType rowType = rel.getRowType(); List columns = rowType.getFieldList().stream() .map(field -> new ViewColumn( field.getName(), typeManager.fromSqlType(getTypeString(field.getType())).getTypeId())) .collect(toImmutableList()); - return new ConnectorViewDefinition(prestoSql, + return new ConnectorViewDefinition( + trinoSql, Optional.of(catalogName.toString()), Optional.of(table.getDatabaseName()), columns, diff --git a/pom.xml b/pom.xml index 3abc18c1d3a8..bab987d9ff9c 100644 --- a/pom.xml +++ b/pom.xml @@ -61,7 +61,7 @@ 2.5.1 1.15.1 3.2.7 - 1.0.37 + 1.0.42 5.5.2