diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java index 19a7ec827a09..2c6fc91d1ae9 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java @@ -241,7 +241,6 @@ public class DeltaLakeMetadata private final long defaultCheckpointInterval; private final boolean ignoreCheckpointWriteFailures; private final int domainCompactionThreshold; - private final boolean hideNonDeltaLakeTables; private final boolean unsafeWritesEnabled; private final JsonCodec dataFileInfoCodec; private final JsonCodec updateResultJsonCodec; @@ -259,7 +258,6 @@ public DeltaLakeMetadata( TypeManager typeManager, AccessControlMetadata accessControlMetadata, int domainCompactionThreshold, - boolean hideNonDeltaLakeTables, boolean unsafeWritesEnabled, JsonCodec dataFileInfoCodec, JsonCodec updateResultJsonCodec, @@ -277,7 +275,6 @@ public DeltaLakeMetadata( this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.accessControlMetadata = requireNonNull(accessControlMetadata, "accessControlMetadata is null"); this.domainCompactionThreshold = domainCompactionThreshold; - this.hideNonDeltaLakeTables = hideNonDeltaLakeTables; this.unsafeWritesEnabled = unsafeWritesEnabled; this.dataFileInfoCodec = requireNonNull(dataFileInfoCodec, "dataFileInfoCodec is null"); this.updateResultJsonCodec = requireNonNull(updateResultJsonCodec, "updateResultJsonCodec is null"); @@ -500,9 +497,6 @@ public Stream streamTableColumns(ConnectorSession session, }); } catch (NotADeltaLakeTableException e) { - if (!hideNonDeltaLakeTables) { - throw e; - } return Stream.empty(); } catch (RuntimeException e) { diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java index e71d12dc90b8..3f2f2df86e4f 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java @@ -49,7 +49,6 @@ public class DeltaLakeMetadataFactory private final DeltaLakeRedirectionsProvider deltaLakeRedirectionsProvider; private final CachingExtendedStatisticsAccess statisticsAccess; private final int domainCompactionThreshold; - private final boolean hideNonDeltaLakeTables; private final boolean unsafeWritesEnabled; private final long checkpointWritingInterval; private final boolean ignoreCheckpointWriteFailures; @@ -64,7 +63,6 @@ public DeltaLakeMetadataFactory( TypeManager typeManager, DeltaLakeAccessControlMetadataFactory accessControlMetadataFactory, DeltaLakeConfig deltaLakeConfig, - @HideNonDeltaLakeTables boolean hideNonDeltaLakeTables, JsonCodec dataFileInfoCodec, JsonCodec updateResultJsonCodec, TransactionLogWriterFactory transactionLogWriterFactory, @@ -88,7 +86,6 @@ public DeltaLakeMetadataFactory( this.statisticsAccess = requireNonNull(statisticsAccess, "statisticsAccess is null"); requireNonNull(deltaLakeConfig, "deltaLakeConfig is null"); this.domainCompactionThreshold = deltaLakeConfig.getDomainCompactionThreshold(); - this.hideNonDeltaLakeTables = hideNonDeltaLakeTables; this.unsafeWritesEnabled = deltaLakeConfig.getUnsafeWritesEnabled(); this.checkpointWritingInterval = deltaLakeConfig.getDefaultCheckpointWritingInterval(); this.ignoreCheckpointWriteFailures = deltaLakeConfig.isIgnoreCheckpointWriteFailures(); @@ -113,7 +110,6 @@ public DeltaLakeMetadata create(ConnectorIdentity identity) typeManager, accessControlMetadataFactory.create(cachingHiveMetastore), domainCompactionThreshold, - hideNonDeltaLakeTables, unsafeWritesEnabled, dataFileInfoCodec, updateResultJsonCodec, diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/HideNonDeltaLakeTables.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/HideNonDeltaLakeTables.java deleted file mode 100644 index 07527b9b8685..000000000000 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/HideNonDeltaLakeTables.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.deltalake; - -import javax.inject.Qualifier; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -import static java.lang.annotation.ElementType.FIELD; -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.ElementType.PARAMETER; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -@Retention(RUNTIME) -@Target({FIELD, PARAMETER, METHOD}) -@Qualifier -public @interface HideNonDeltaLakeTables {} diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/file/DeltaLakeFileMetastoreModule.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/file/DeltaLakeFileMetastoreModule.java index 4e648574d442..225b8b578435 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/file/DeltaLakeFileMetastoreModule.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/file/DeltaLakeFileMetastoreModule.java @@ -14,10 +14,7 @@ package io.trino.plugin.deltalake.metastore.file; import com.google.inject.Binder; -import com.google.inject.Provides; -import com.google.inject.Singleton; import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.trino.plugin.deltalake.HideNonDeltaLakeTables; import io.trino.plugin.hive.metastore.file.FileMetastoreModule; public class DeltaLakeFileMetastoreModule @@ -28,12 +25,4 @@ protected void setup(Binder binder) { install(new FileMetastoreModule()); } - - @Provides - @Singleton - @HideNonDeltaLakeTables - public boolean provideHideNonDeltaLakeTables() - { - return false; - } } diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/DeltaLakeGlueMetastoreModule.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/DeltaLakeGlueMetastoreModule.java index 88ccdca1d4ab..6dbd56d63fb3 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/DeltaLakeGlueMetastoreModule.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/DeltaLakeGlueMetastoreModule.java @@ -16,11 +16,8 @@ import com.amazonaws.services.glue.model.Table; import com.google.inject.Binder; import com.google.inject.Key; -import com.google.inject.Provides; -import com.google.inject.Singleton; import com.google.inject.TypeLiteral; import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.trino.plugin.deltalake.HideNonDeltaLakeTables; import io.trino.plugin.hive.metastore.glue.ForGlueHiveMetastore; import io.trino.plugin.hive.metastore.glue.GlueMetastoreModule; @@ -42,12 +39,4 @@ protected void setup(Binder binder) install(new GlueMetastoreModule()); } - - @Provides - @Singleton - @HideNonDeltaLakeTables - public boolean provideHideNonDeltaLakeTables(DeltaLakeGlueMetastoreConfig glueMetastoreConfig) - { - return glueMetastoreConfig.isHideNonDeltaLakeTables(); - } } diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/thrift/DeltaLakeThriftMetastoreModule.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/thrift/DeltaLakeThriftMetastoreModule.java index bf2e9aafde6a..dd967be1d780 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/thrift/DeltaLakeThriftMetastoreModule.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/thrift/DeltaLakeThriftMetastoreModule.java @@ -14,10 +14,7 @@ package io.trino.plugin.deltalake.metastore.thrift; import com.google.inject.Binder; -import com.google.inject.Provides; -import com.google.inject.Singleton; import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.trino.plugin.deltalake.HideNonDeltaLakeTables; import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreModule; public class DeltaLakeThriftMetastoreModule @@ -28,13 +25,4 @@ protected void setup(Binder binder) { install(new ThriftMetastoreModule()); } - - // TODO support delta.hide-non-delta-lake-tables with thrift metastore (https://github.com/trinodb/trino/issues/12014) - @Provides - @Singleton - @HideNonDeltaLakeTables - public boolean provideHideNonDeltaLakeTables() - { - return false; - } } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePerTransactionMetastoreCache.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePerTransactionMetastoreCache.java index 9c75df5348ac..04d4ae35e12c 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePerTransactionMetastoreCache.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePerTransactionMetastoreCache.java @@ -110,7 +110,6 @@ protected void setup(Binder binder) binder.bind(ThriftMetastore.class).to(ThriftHiveMetastore.class).in(Scopes.SINGLETON); newExporter(binder).export(ThriftMetastore.class).as((generator) -> generator.generatedNameOf(ThriftHiveMetastore.class)); install(new ThriftMetastoreAuthenticationModule()); - binder.bind(Boolean.class).annotatedWith(HideNonDeltaLakeTables.class).toInstance(false); binder.bind(BridgingHiveMetastoreFactory.class).in(Scopes.SINGLETON); } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java new file mode 100644 index 000000000000..305a6e6b29bf --- /dev/null +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedHiveMetastoreWithViews.java @@ -0,0 +1,125 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.deltalake; + +import com.google.common.collect.ImmutableMap; +import io.trino.plugin.deltalake.util.DockerizedMinioDataLake; +import io.trino.plugin.hive.TestingHivePlugin; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.QueryRunner; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.Optional; + +import static io.trino.plugin.deltalake.DeltaLakeDockerizedMinioDataLake.createDockerizedMinioDataLakeForDeltaLake; +import static io.trino.plugin.deltalake.util.MinioContainer.MINIO_ACCESS_KEY; +import static io.trino.plugin.deltalake.util.MinioContainer.MINIO_SECRET_KEY; +import static io.trino.testing.sql.TestTable.randomTableSuffix; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestDeltaLakeSharedHiveMetastoreWithViews + extends AbstractTestQueryFramework +{ + protected final String schema = "test_shared_schema_with_hive_views_" + randomTableSuffix(); + private final String bucketName = "delta-lake-shared-hive-with-views-" + randomTableSuffix(); + + private DockerizedMinioDataLake dockerizedMinioDataLake; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + this.dockerizedMinioDataLake = closeAfterClass(createDockerizedMinioDataLakeForDeltaLake(bucketName, Optional.empty())); + + DistributedQueryRunner queryRunner = DeltaLakeQueryRunner.createS3DeltaLakeQueryRunner( + "delta", + schema, + ImmutableMap.builder() + .put("delta.enable-non-concurrent-writes", "true") + .buildOrThrow(), + dockerizedMinioDataLake.getMinioAddress(), + dockerizedMinioDataLake.getTestingHadoop()); + queryRunner.execute("CREATE SCHEMA " + schema + " WITH (location = 's3://" + bucketName + "/" + schema + "')"); + + queryRunner.installPlugin(new TestingHivePlugin()); + Map s3Properties = ImmutableMap.builder() + .put("hive.s3.aws-access-key", MINIO_ACCESS_KEY) + .put("hive.s3.aws-secret-key", MINIO_SECRET_KEY) + .put("hive.s3.endpoint", dockerizedMinioDataLake.getMinioAddress()) + .put("hive.s3.path-style-access", "true") + .buildOrThrow(); + queryRunner.createCatalog( + "hive", + "hive", + ImmutableMap.builder() + .put("hive.metastore.uri", dockerizedMinioDataLake.getTestingHadoop().getMetastoreAddress()) + .put("hive.allow-drop-table", "true") + .putAll(s3Properties) + .buildOrThrow()); + + queryRunner.execute("CREATE TABLE hive." + schema + ".hive_table (a_integer integer)"); + dockerizedMinioDataLake.getTestingHadoop().runOnHive("CREATE VIEW " + schema + ".hive_view AS SELECT * FROM " + schema + ".hive_table"); + queryRunner.execute("CREATE TABLE delta." + schema + ".delta_table (a_varchar varchar)"); + + return queryRunner; + } + + @AfterClass(alwaysRun = true) + public void cleanup() + { + assertQuerySucceeds("DROP TABLE IF EXISTS hive." + schema + ".hive_table"); + dockerizedMinioDataLake.getTestingHadoop().runOnHive("DROP VIEW IF EXISTS " + schema + ".hive_view"); + assertQuerySucceeds("DROP TABLE IF EXISTS delta." + schema + ".delta_table"); + assertQuerySucceeds("DROP SCHEMA IF EXISTS hive." + schema); + } + + @Test + public void testReadInformationSchema() + { + assertThat(query("SELECT table_schema FROM hive.information_schema.tables WHERE table_name = 'hive_table' AND table_schema='" + schema + "'")) + .skippingTypesCheck() + .containsAll("VALUES '" + schema + "'"); + assertThat(query("SELECT table_schema FROM delta.information_schema.tables WHERE table_name = 'delta_table' AND table_schema='" + schema + "'")) + .skippingTypesCheck() + .containsAll("VALUES '" + schema + "'"); + + assertQuery("SELECT table_name, column_name from hive.information_schema.columns WHERE table_schema = '" + schema + "'", + "VALUES ('hive_table', 'a_integer')"); + assertQuery("SELECT table_name, column_name from delta.information_schema.columns WHERE table_schema = '" + schema + "'", + "VALUES ('delta_table', 'a_varchar')"); + } + + @Test + public void testShowTables() + { + assertQuery("SHOW TABLES FROM delta." + schema, "VALUES 'hive_table', 'hive_view', 'delta_table'"); + assertQuery("SHOW TABLES FROM hive." + schema, "VALUES 'hive_table', 'hive_view', 'delta_table'"); + + assertThatThrownBy(() -> query("SHOW CREATE TABLE delta." + schema + ".hive_table")) + .hasMessageContaining("not a Delta Lake table"); + assertThatThrownBy(() -> query("SHOW CREATE TABLE delta." + schema + ".hive_view")) + .hasMessageContaining("not a Delta Lake table"); + assertThatThrownBy(() -> query("SHOW CREATE TABLE hive." + schema + ".delta_table")) + .hasMessageContaining("Cannot query Delta Lake table"); + + assertThatThrownBy(() -> query("DESCRIBE delta." + schema + ".hive_table")) + .hasMessageContaining("not a Delta Lake table"); + assertThatThrownBy(() -> query("DESCRIBE hive." + schema + ".delta_table")) + .hasMessageContaining("Cannot query Delta Lake table"); + } +} diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java index 576a51d192b1..81af2a754bbd 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java @@ -21,7 +21,6 @@ import com.google.inject.Binder; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.Session; -import io.trino.plugin.deltalake.HideNonDeltaLakeTables; import io.trino.plugin.deltalake.TestingDeltaLakePlugin; import io.trino.plugin.hive.HdfsConfig; import io.trino.plugin.hive.HdfsConfiguration; @@ -109,7 +108,6 @@ public CountingAccessMetastoreModule(CountingAccessHiveMetastore metastore) protected void setup(Binder binder) { binder.bind(HiveMetastoreFactory.class).annotatedWith(RawHiveMetastoreFactory.class).toInstance(HiveMetastoreFactory.ofInstance(metastore)); - binder.bind(Boolean.class).annotatedWith(HideNonDeltaLakeTables.class).toInstance(false); } }