Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ public class ThriftHiveMetastore
private final CoalescingCounter metastoreSetDateStatisticsFailures = new CoalescingCounter(new Duration(1, SECONDS));

private static final Pattern TABLE_PARAMETER_SAFE_KEY_PATTERN = Pattern.compile("^[a-zA-Z_]+$");
private static final Pattern TABLE_PARAMETER_SAFE_VALUE_PATTERN = Pattern.compile("^[a-zA-Z0-9]*$");
private static final Pattern TABLE_PARAMETER_SAFE_VALUE_PATTERN = Pattern.compile("^[a-zA-Z0-9\\s]*$");
private final boolean assumeCanonicalPartitionKeys;

@Inject
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ public class IcebergMetadata
implements ConnectorMetadata
{
private static final Logger log = Logger.get(IcebergMetadata.class);
private static final String ICEBERG_MATERIALIZED_VIEW_COMMENT = "Presto Materialized View";
public static final String DEPENDS_ON_TABLES = "dependsOnTables";

private final CatalogName catalogName;
Expand Down Expand Up @@ -408,7 +409,7 @@ public List<SchemaTableName> listTables(ConnectorSession session, Optional<Strin

schemaName.map(Collections::singletonList)
.orElseGet(metastore::getAllDatabases).stream()
.flatMap(schema -> metastore.getAllViews(schema).stream()
.flatMap(schema -> metastore.getTablesWithParameter(schema, TABLE_COMMENT, ICEBERG_MATERIALIZED_VIEW_COMMENT).stream()
.map(table -> new SchemaTableName(schema, table)))
.forEach(tablesListBuilder::add);
return tablesListBuilder.build();
Expand Down Expand Up @@ -896,7 +897,7 @@ public void createMaterializedView(ConnectorSession session, SchemaTableName vie
.put(PRESTO_QUERY_ID_NAME, session.getQueryId())
.put(STORAGE_TABLE, storageTableName)
.put(PRESTO_VIEW_FLAG, "true")
.put(TABLE_COMMENT, "Presto Materialized View")
.put(TABLE_COMMENT, ICEBERG_MATERIALIZED_VIEW_COMMENT)
.build();

Column dummyColumn = new Column("dummy", HIVE_STRING, Optional.empty());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import io.trino.Session;
import io.trino.metadata.QualifiedObjectName;
import io.trino.plugin.hive.HdfsConfig;
import io.trino.plugin.hive.HdfsConfiguration;
import io.trino.plugin.hive.HdfsConfigurationInitializer;
Expand All @@ -28,10 +29,14 @@
import io.trino.plugin.hive.metastore.MetastoreConfig;
import io.trino.plugin.hive.metastore.file.FileHiveMetastore;
import io.trino.plugin.hive.metastore.file.FileHiveMetastoreConfig;
import io.trino.spi.connector.ConnectorMaterializedViewDefinition;
import io.trino.spi.connector.SchemaTableName;
import io.trino.spi.security.Identity;
import io.trino.spi.security.SelectedRole;
import io.trino.testing.AbstractTestQueryFramework;
import io.trino.testing.DistributedQueryRunner;
import io.trino.transaction.TransactionId;
import io.trino.transaction.TransactionManager;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
Expand All @@ -41,12 +46,13 @@

import static io.trino.spi.security.SelectedRole.Type.ROLE;
import static io.trino.testing.TestingSession.testSessionBuilder;
import static org.testng.Assert.assertEquals;
import static org.assertj.core.api.Assertions.assertThat;

public class TestIcebergMetadataListing
extends AbstractTestQueryFramework
{
private HiveMetastore metastore;
private SchemaTableName storageTable;

@Override
protected DistributedQueryRunner createQueryRunner()
Expand Down Expand Up @@ -87,14 +93,20 @@ public void setUp()
assertQuerySucceeds("CREATE SCHEMA hive.test_schema");
assertQuerySucceeds("CREATE TABLE iceberg.test_schema.iceberg_table1 (_string VARCHAR, _integer INTEGER)");
assertQuerySucceeds("CREATE TABLE iceberg.test_schema.iceberg_table2 (_double DOUBLE) WITH (partitioning = ARRAY['_double'])");
assertQuerySucceeds("CREATE MATERIALIZED VIEW iceberg.test_schema.iceberg_materialized_view AS " +
"SELECT * FROM iceberg.test_schema.iceberg_table1");
storageTable = getStorageTable("iceberg", "test_schema", "iceberg_materialized_view");

assertQuerySucceeds("CREATE TABLE hive.test_schema.hive_table (_double DOUBLE)");
assertEquals(ImmutableSet.copyOf(metastore.getAllTables("test_schema")), ImmutableSet.of("iceberg_table1", "iceberg_table2", "hive_table"));
assertQuerySucceeds("CREATE VIEW hive.test_schema.hive_view AS SELECT * FROM hive.test_schema.hive_table");
}

@AfterClass(alwaysRun = true)
public void tearDown()
{
assertQuerySucceeds("DROP TABLE IF EXISTS hive.test_schema.hive_table");
assertQuerySucceeds("DROP VIEW IF EXISTS hive.test_schema.hive_view");
assertQuerySucceeds("DROP MATERIALIZED VIEW IF EXISTS iceberg.test_schema.iceberg_materialized_view");
assertQuerySucceeds("DROP TABLE IF EXISTS iceberg.test_schema.iceberg_table2");
assertQuerySucceeds("DROP TABLE IF EXISTS iceberg.test_schema.iceberg_table1");
assertQuerySucceeds("DROP SCHEMA IF EXISTS hive.test_schema");
Expand All @@ -103,15 +115,37 @@ public void tearDown()
@Test
public void testTableListing()
{
assertQuery("SHOW TABLES FROM iceberg.test_schema", "VALUES 'iceberg_table1', 'iceberg_table2'");
assertThat(metastore.getAllTables("test_schema"))
.containsExactlyInAnyOrder(
"iceberg_table1",
"iceberg_table2",
"iceberg_materialized_view",
storageTable.getTableName(),
"hive_table",
"hive_view");

assertQuery(
"SHOW TABLES FROM iceberg.test_schema",
"VALUES " +
"'iceberg_table1', " +
"'iceberg_table2', " +
"'iceberg_materialized_view', " +
"'" + storageTable.getTableName() + "'");
}

@Test
public void testTableColumnListing()
{
// Verify information_schema.columns does not include columns from non-Iceberg tables
assertQuery("SELECT table_name, column_name FROM iceberg.information_schema.columns WHERE table_schema = 'test_schema'",
"VALUES ('iceberg_table1', '_string'), ('iceberg_table1', '_integer'), ('iceberg_table2', '_double')");
// TODO this should include columns from the materialized view as well
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please create an issue in Github and reference it here

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should get solved by #8113

assertQuery(
"SELECT table_name, column_name FROM iceberg.information_schema.columns WHERE table_schema = 'test_schema'",
"VALUES " +
"('iceberg_table1', '_string'), " +
"('iceberg_table1', '_integer'), " +
"('iceberg_table2', '_double'), " +
"('" + storageTable.getTableName() + "', '_string'), " +
"('" + storageTable.getTableName() + "', '_integer')");
}

@Test
Expand All @@ -126,4 +160,15 @@ public void testTableValidation()
assertQuerySucceeds("SELECT * FROM iceberg.test_schema.iceberg_table1");
assertQueryFails("SELECT * FROM iceberg.test_schema.hive_table", "Not an Iceberg table: test_schema.hive_table");
}

private SchemaTableName getStorageTable(String catalogName, String schemaName, String objectName)
{
TransactionManager transactionManager = getQueryRunner().getTransactionManager();
TransactionId transactionId = transactionManager.beginTransaction(false);
Session session = getSession().beginTransactionId(transactionId, transactionManager, getQueryRunner().getAccessControl());
Optional<ConnectorMaterializedViewDefinition> materializedView = getQueryRunner().getMetadata()
.getMaterializedView(session, new QualifiedObjectName(catalogName, schemaName, objectName));
assertThat(materializedView).isPresent();
return materializedView.get().getStorageTable().get().getSchemaTableName();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.tests.product.iceberg;

import io.trino.tempto.AfterTestWithContext;
import io.trino.tempto.BeforeTestWithContext;
import io.trino.tempto.ProductTest;
import org.testng.annotations.Test;

import static com.google.common.collect.Iterators.getOnlyElement;
import static io.trino.tempto.assertions.QueryAssert.Row.row;
import static io.trino.tempto.assertions.QueryAssert.assertThat;
import static io.trino.tests.product.TestGroups.ICEBERG;
import static io.trino.tests.product.TestGroups.STORAGE_FORMATS;
import static io.trino.tests.product.utils.QueryExecutors.onTrino;

public class TestIcebergHiveMetadataListing
extends ProductTest
{
private String storageTable;

@BeforeTestWithContext
public void setUp()
{
onTrino().executeQuery("CREATE TABLE iceberg.default.iceberg_table1 (_string VARCHAR, _integer INTEGER)");
onTrino().executeQuery("CREATE MATERIALIZED VIEW iceberg.default.iceberg_materialized_view AS " +
"SELECT * FROM iceberg.default.iceberg_table1");
storageTable = getOnlyElement(onTrino().executeQuery("SHOW TABLES FROM iceberg.default")
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is storage table created immediately?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, in iceberg CREATE MV we create the storage table even before storing the MV definition in HMS

.column(1).stream()
.map(String.class::cast)
.filter(name -> name.startsWith("st_"))
.iterator());

onTrino().executeQuery("CREATE TABLE hive.default.hive_table (_double DOUBLE)");
onTrino().executeQuery("CREATE VIEW hive.default.hive_view AS SELECT * FROM hive.default.hive_table");
}

@AfterTestWithContext
public void cleanUp()
{
onTrino().executeQuery("DROP TABLE IF EXISTS hive.default.hive_table");
onTrino().executeQuery("DROP VIEW IF EXISTS hive.default.hive_view");
onTrino().executeQuery("DROP MATERIALIZED VIEW IF EXISTS iceberg.default.iceberg_materialized_view");
onTrino().executeQuery("DROP TABLE IF EXISTS iceberg.default.iceberg_table1");
}

@Test(groups = {ICEBERG, STORAGE_FORMATS})
public void testTableListing()
{
assertThat(onTrino().executeQuery("SHOW TABLES FROM iceberg.default"))
.containsOnly(
row("iceberg_table1"),
row("iceberg_materialized_view"),
row(storageTable));
}

@Test(groups = {ICEBERG, STORAGE_FORMATS})
public void testColumnListing()
{
assertThat(onTrino().executeQuery(
"SELECT table_name, column_name FROM iceberg.information_schema.columns " +
"WHERE table_catalog = 'iceberg' AND table_schema = 'default'"))
.containsOnly(
row("iceberg_table1", "_string"),
row("iceberg_table1", "_integer"),
row(storageTable, "_string"),
row(storageTable, "_integer"));
}
}