diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6517e6a5009..3ad67e2aa33b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -528,6 +528,7 @@ jobs: runs-on: ubuntu-latest outputs: have_azure_secrets: ${{ steps.check-secrets.outputs.have_azure_secrets }} + have_databricks_secrets: ${{ steps.check-databricks-secrets.outputs.have_databricks_secrets }} steps: - uses: actions/checkout@v2 with: @@ -553,6 +554,17 @@ jobs: echo "::set-output name=have_azure_secrets::false" fi id: check-secrets + - name: Check Delta Databricks secrets + id: check-databricks-secrets + run: | + if [[ "${{ secrets.DATABRICKS_TOKEN }}" != "" ]]; \ + then + echo "Secrets to run Delta Databricks product tests were configured in the repo" + echo "::set-output name=have_databricks_secrets::true" + else + echo "Secrets to run Delta Databricks product tests were not configured in the repo" + echo "::set-output name=have_databricks_secrets::false" + fi - name: Maven Install run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" @@ -598,6 +610,7 @@ jobs: # suite-4 does not exist - suite-5 - suite-azure + - suite-delta-lake-databricks jdk: - 11 exclude: @@ -623,6 +636,14 @@ jobs: ignore exclusion if: >- ${{ needs.build-pt.outputs.have_azure_secrets == 'true' }} + - suite: suite-delta-lake-databricks + config: cdh5 + - suite: suite-delta-lake-databricks + config: hdp3 + - suite: suite-delta-lake-databricks + ignore exclusion if: >- + ${{ needs.build-pt.outputs.have_databricks_secrets == 'true' }} + ignore exclusion if: # Do not use this property outside of the matrix configuration. # @@ -681,7 +702,7 @@ jobs: jdk: 11 # this suite is not meant to be run with different configs - config: default - suite: suite-delta-lake + suite: suite-delta-lake-oss jdk: 11 # PT Launcher's timeout defaults to 2h, add some margin timeout-minutes: 130 @@ -720,6 +741,14 @@ jobs: ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }} ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }} ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }} + S3_BUCKET: trino-ci-test + AWS_REGION: us-east-2 + DATABRICKS_AWS_ACCESS_KEY_ID: ${{ secrets.DATABRICKS_AWS_ACCESS_KEY_ID }} + DATABRICKS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DATABRICKS_AWS_SECRET_ACCESS_KEY }} + DATABRICKS_73_JDBC_URL: ${{ secrets.DATABRICKS_73_JDBC_URL }} + DATABRICKS_91_JDBC_URL: ${{ secrets.DATABRICKS_91_JDBC_URL }} + DATABRICKS_LOGIN: token + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} run: | testing/bin/ptl suite run \ --suite ${{ matrix.suite }} \ diff --git a/plugin/trino-delta-lake/README.md b/plugin/trino-delta-lake/README.md new file mode 100644 index 000000000000..abc842f64ba6 --- /dev/null +++ b/plugin/trino-delta-lake/README.md @@ -0,0 +1,178 @@ +# Delta Lake Connector Developer Notes + +The Delta Lake connector can be used to interact with [Delta Lake](https://delta.io/) tables. + +Trino has product tests in place for testing its compatibility with the +following Delta Lake implementations: + +- Delta Lake OSS +- Delta Lake Databricks + + +## Delta Lake OSS Product tests + +Testing against Delta Lake OSS is quite straightforward by simply spinning up +the corresponding product test environment: + +``` +testing/bin/ptl env up --environment singlenode-delta-lake-oss +``` + + +## Delta Lake Databricks Product tests + +At the time of this writing, Databricks Delta Lake and OSS Delta Lake differ in functionality provided. + +In order to setup a Databricks testing environment there are several steps to be performed. + +### Delta Lake Databricks on AWS + +Start by setting up a Databricks account via https://databricks.com/try-databricks and after +filling your contact details, choose *AWS* as preferred cloud provider. + +Create an AWS S3 bucket to be used for storing the content of the Delta Lake tables managed +by the Databricks runtime. + +Follow the guideline [Secure access to S3 buckets using instance profiles](https://docs.databricks.com/administration-guide/cloud-configurations/aws/instance-profiles.html) +for allowing the Databricks cluster to access the AWS S3 bucket on which the Delta Lake tables are stored or AWS Glue +table metastore. + +In order to make sure that the setup has been done correctly, proceed via Databricks Web UI to +create a notebook on which a simple table could be created: + +``` +%sql +CREATE TABLE default.test1 ( + a_bigint BIGINT) +USING DELTA LOCATION 's3://my-s3-bucket/test1' +``` + +### Use AWS Glue Data Catalog as the metastore for Databricks Runtime + +[AWS Glue](https://aws.amazon.com/glue) is the metastore of choice for Databricks Delta Lake product tests +on Trino because it is a managed solution which allows connectivity to the metastore backing the +Databricks runtime from Trino as well while executing the product tests. + +Follow the guideline [Use AWS Glue Data Catalog as the metastore for Databricks Runtime](https://docs.databricks.com/data/metastores/aws-glue-metastore.html#configure-glue-data-catalog-as-the-metastore) +for performing the setup of Glue as Data Catalog on your Databricks Cluster. + +After performing successfully this step you should be able to perform any of the statements: + +``` +show databases; + +show tables; +``` + +The output of the previously mentioned statements should be the same as the one seen on the +AWS Glue administration Web UI. + + +### Create AWS user to be used by Trino for managing Delta Lake tables + +Trino needs a set of security credentials for successfully connecting to the AWS infrastructure +in order to perform create/drop tables on AWS Glue and read/modify table content on AWS S3. + +Create via AWS IAM a user which has the appropriate policies for interacting with AWS. +Below are presented a set of simplistic permission policies which can be configured on this +user: + +`GlueAccess` + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "GrantCatalogAccessToGlue", + "Effect": "Allow", + "Action": [ + "glue:BatchCreatePartition", + "glue:BatchDeletePartition", + "glue:BatchGetPartition", + "glue:CreateDatabase", + "glue:CreateTable", + "glue:CreateUserDefinedFunction", + "glue:DeleteDatabase", + "glue:DeletePartition", + "glue:DeleteTable", + "glue:DeleteUserDefinedFunction", + "glue:GetDatabase", + "glue:GetDatabases", + "glue:GetPartition", + "glue:GetPartitions", + "glue:GetTable", + "glue:GetTables", + "glue:GetUserDefinedFunction", + "glue:GetUserDefinedFunctions", + "glue:UpdateDatabase", + "glue:UpdatePartition", + "glue:UpdateTable", + "glue:UpdateUserDefinedFunction" + ], + "Resource": [ + "*" + ] + } + ] +} +``` + +`S3Access` + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::my-s3-bucket" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:PutObjectAcl" + ], + "Resource": [ + "arn:aws:s3:::my-s3-bucket/*" + ] + } + ] +} +``` + +For the newly created AWS IAM user, make sure to retrieve the security credentials because they +are to be used by Trino in communicating with AWS. + + +### Setup token authentication on the Databricks cluster + +Follow the guideline [Authentication using Databricks personal access tokens](https://docs.databricks.com/dev-tools/api/latest/authentication.html) +for setting up your Databricks personal access token. + + +### Test the functionality of the Databricks Delta Lake product test environment + + +Run the following command for spinning up the Databricks 9.1 Delta Lake product test +environment for Trino: + +``` +env S3_BUCKET=my-s3-bucket \ + AWS_REGION=us-east-2 \ + AWS_SECRET_ACCESS_KEY=xxx \ + AWS_ACCESS_KEY_ID=xxx \ + DATABRICKS_91_JDBC_URL='xxx' \ + DATABRICKS_LOGIN=token \ + DATABRICKS_TOKEN=xxx \ + testing/bin/ptl env up --environment singlenode-delta-lake-databricks91 +``` diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java index ec37af15e687..26d48be4e95c 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/AbstractSinglenodeDeltaLakeDatabricks.java @@ -47,18 +47,17 @@ public AbstractSinglenodeDeltaLakeDatabricks(Standard standard, DockerFiles dock public void extendEnvironment(Environment.Builder builder) { String databricksTestJdbcUrl = databricksTestJdbcUrl(); - String databricksTestJdbcDriverClass = requireNonNull(System.getenv("DATABRICKS_TEST_JDBC_DRIVER_CLASS"), "Environment DATABRICKS_TEST_JDBC_DRIVER_CLASS was not set"); - String databricksTestLogin = requireNonNull(System.getenv("DATABRICKS_TEST_LOGIN"), "Environment DATABRICKS_TEST_LOGIN was not set"); - String databricksTestToken = requireNonNull(System.getenv("DATABRICKS_TEST_TOKEN"), "Environment DATABRICKS_TEST_TOKEN was not set"); - String hiveMetastoreUri = requireNonNull(System.getenv("HIVE_METASTORE_URI"), "Environment HIVE_METASTORE_URI was not set"); + String databricksTestLogin = requireNonNull(System.getenv("DATABRICKS_LOGIN"), "Environment DATABRICKS_LOGIN was not set"); + String databricksTestToken = requireNonNull(System.getenv("DATABRICKS_TOKEN"), "Environment DATABRICKS_TOKEN was not set"); + String awsRegion = requireNonNull(System.getenv("AWS_REGION"), "Environment AWS_REGION was not set"); String s3Bucket = requireNonNull(System.getenv("S3_BUCKET"), "Environment S3_BUCKET was not set"); DockerFiles.ResourceProvider configDir = dockerFiles.getDockerFilesHostDirectory("conf/environment/singlenode-delta-lake-databricks"); builder.configureContainer(COORDINATOR, dockerContainer -> exportAWSCredentials(dockerContainer) - .withEnv("HIVE_METASTORE_URI", hiveMetastoreUri) - .withEnv("DATABRICKS_TEST_JDBC_URL", databricksTestJdbcUrl) - .withEnv("DATABRICKS_TEST_LOGIN", databricksTestLogin) - .withEnv("DATABRICKS_TEST_TOKEN", databricksTestToken)); + .withEnv("AWS_REGION", awsRegion) + .withEnv("DATABRICKS_JDBC_URL", databricksTestJdbcUrl) + .withEnv("DATABRICKS_LOGIN", databricksTestLogin) + .withEnv("DATABRICKS_TOKEN", databricksTestToken)); builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); builder.addConnector( "delta-lake", @@ -67,23 +66,22 @@ public void extendEnvironment(Environment.Builder builder) builder.configureContainer(TESTS, container -> exportAWSCredentials(container) .withEnv("S3_BUCKET", s3Bucket) - .withEnv("DATABRICKS_TEST_JDBC_DRIVER_CLASS", databricksTestJdbcDriverClass) - .withEnv("DATABRICKS_TEST_JDBC_URL", databricksTestJdbcUrl) - .withEnv("DATABRICKS_TEST_LOGIN", databricksTestLogin) - .withEnv("DATABRICKS_TEST_TOKEN", databricksTestToken) - .withEnv("HIVE_METASTORE_URI", hiveMetastoreUri)); + .withEnv("AWS_REGION", awsRegion) + .withEnv("DATABRICKS_JDBC_URL", databricksTestJdbcUrl) + .withEnv("DATABRICKS_LOGIN", databricksTestLogin) + .withEnv("DATABRICKS_TOKEN", databricksTestToken)); configureTempto(builder, configDir); } private DockerContainer exportAWSCredentials(DockerContainer container) { - container = exportAWSCredential(container, "AWS_ACCESS_KEY_ID", true); - container = exportAWSCredential(container, "AWS_SECRET_ACCESS_KEY", true); - return exportAWSCredential(container, "AWS_SESSION_TOKEN", false); + container = exportAWSCredential(container, "DATABRICKS_AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY_ID", true); + container = exportAWSCredential(container, "DATABRICKS_AWS_SECRET_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", true); + return exportAWSCredential(container, "DATABRICKS_AWS_SESSION_TOKEN", "AWS_SESSION_TOKEN", false); } - private DockerContainer exportAWSCredential(DockerContainer container, String credentialEnvVariable, boolean required) + private DockerContainer exportAWSCredential(DockerContainer container, String credentialEnvVariable, String containerEnvVariable, boolean required) { String credentialValue = System.getenv(credentialEnvVariable); if (credentialValue == null) { @@ -92,6 +90,6 @@ private DockerContainer exportAWSCredential(DockerContainer container, String cr } return container; } - return container.withEnv(credentialEnvVariable, credentialValue); + return container.withEnv(containerEnvVariable, credentialValue); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks73.java similarity index 80% rename from testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks.java rename to testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks73.java index 29b3a5cc9784..ca6450dcfcc5 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks73.java @@ -21,12 +21,12 @@ import static java.util.Objects.requireNonNull; @TestsEnvironment -public class EnvSinglenodeDeltaLakeDatabricks +public class EnvSinglenodeDeltaLakeDatabricks73 extends AbstractSinglenodeDeltaLakeDatabricks { @Inject - public EnvSinglenodeDeltaLakeDatabricks(Standard standard, DockerFiles dockerFiles) + public EnvSinglenodeDeltaLakeDatabricks73(Standard standard, DockerFiles dockerFiles) { super(standard, dockerFiles); } @@ -34,6 +34,6 @@ public EnvSinglenodeDeltaLakeDatabricks(Standard standard, DockerFiles dockerFil @Override String databricksTestJdbcUrl() { - return requireNonNull(System.getenv("DATABRICKS_TEST_JDBC_URL"), "Environment DATABRICKS_TEST_JDBC_URL was not set"); + return requireNonNull(System.getenv("DATABRICKS_73_JDBC_URL"), "Environment DATABRICKS_73_JDBC_URL was not set"); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks91.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks91.java index 551d17d8a805..0f9b61613109 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks91.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks91.java @@ -33,6 +33,6 @@ public EnvSinglenodeDeltaLakeDatabricks91(Standard standard, DockerFiles dockerF @Override String databricksTestJdbcUrl() { - return requireNonNull(System.getenv("DATABRICKS_91_TEST_JDBC_URL"), "Environment DATABRICKS_91_TEST_JDBC_URL was not set"); + return requireNonNull(System.getenv("DATABRICKS_91_JDBC_URL"), "Environment DATABRICKS_91_JDBC_URL was not set"); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks.java new file mode 100644 index 000000000000..9613b89ce97a --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks.java @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.suite.suites; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.env.EnvironmentConfig; +import io.trino.tests.product.launcher.env.environment.EnvSinglenodeDeltaLakeDatabricks73; +import io.trino.tests.product.launcher.env.environment.EnvSinglenodeDeltaLakeDatabricks91; +import io.trino.tests.product.launcher.suite.Suite; +import io.trino.tests.product.launcher.suite.SuiteTestRun; + +import java.util.List; + +import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment; + +public class SuiteDeltaLakeDatabricks + extends Suite +{ + @Override + public List getTestRuns(EnvironmentConfig config) + { + String[] excludedTests = { + // AWS Glue does not support table comments + "io.trino.tests.product.deltalake.TestHiveAndDeltaLakeRedirect.testDeltaToHiveCommentTable", + "io.trino.tests.product.deltalake.TestHiveAndDeltaLakeRedirect.testHiveToDeltaCommentTable", + "io.trino.tests.product.deltalake.TestDeltaLakeAlterTableCompatibility.testCommentOnTableUnsupportedWriterVersion", + // AWS Glue does not support column comments + "io.trino.tests.product.deltalake.TestHiveAndDeltaLakeRedirect.testDeltaToHiveCommentColumn", + "io.trino.tests.product.deltalake.TestHiveAndDeltaLakeRedirect.testHiveToDeltaCommentColumn", + "io.trino.tests.product.deltalake.TestDeltaLakeAlterTableCompatibility.testCommentOnColumnUnsupportedWriterVersion", + // AWS Glue does not support table renames + "io.trino.tests.product.deltalake.TestHiveAndDeltaLakeRedirect.testDeltaToHiveAlterTable", + "io.trino.tests.product.deltalake.TestHiveAndDeltaLakeRedirect.testHiveToDeltaAlterTable", + // TODO https://github.com/trinodb/trino/issues/13017 + "io.trino.tests.product.deltalake.TestDeltaLakeDropTableCompatibility.testCreateManagedTableInDeltaDropTableInTrino" + }; + return ImmutableList.of( + testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks73.class) + .withGroups("configured_features", "delta-lake-databricks") + .withExcludedGroups("delta-lake-exclude-73") + .withExcludedTests(excludedTests) + .build(), + + testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks91.class) + .withGroups("configured_features", "delta-lake-databricks") + .withExcludedGroups("delta-lake-exclude-91") + .withExcludedTests(excludedTests) + .build()); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLake.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeOss.java similarity index 78% rename from testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLake.java rename to testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeOss.java index f18a3e272d67..490f27a337d3 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLake.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeOss.java @@ -25,7 +25,7 @@ import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment; -public class SuiteDeltaLake +public class SuiteDeltaLakeOss extends Suite { @Override @@ -39,15 +39,6 @@ public List getTestRuns(EnvironmentConfig config) testOnEnvironment(EnvSinglenodeDeltaLakeKerberizedHdfs.class) .withGroups("configured_features", "delta-lake-hdfs") .build(), - //TODO enable the product tests against Databricks testing environment -// testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks.class) -// .withGroups("configured_features", "delta-lake-databricks") -// .withExcludedGroups("delta-lake-exclude-73") -// .build(), -// -// testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks91.class) -// .withGroups("configured_features", "delta-lake-databricks") -// .build(), testOnEnvironment(EnvSinglenodeDeltaLakeOss.class) // TODO: make the list of tests run here as close to those run on SinglenodeDeltaLakeDatabricks diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/delta.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/delta.properties index d84e99d50230..f8726fc898ff 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/delta.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/delta.properties @@ -1,5 +1,6 @@ connector.name=delta-lake -hive.metastore.uri=${ENV:HIVE_METASTORE_URI} +hive.metastore=glue +hive.metastore.glue.region=${ENV:AWS_REGION} # We need to give access to bucket owner (the AWS account integrated with Databricks), otherwise files won't be readable from Databricks hive.s3.upload-acl-type=BUCKET_OWNER_FULL_CONTROL delta.enable-non-concurrent-writes=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/hive.properties index f90f06222b54..1a207291a5ed 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/hive.properties @@ -1,5 +1,6 @@ connector.name=hive -hive.metastore.uri=${ENV:HIVE_METASTORE_URI} +hive.metastore=glue +hive.metastore.glue.region=${ENV:AWS_REGION} # We need to give access to bucket owner (the AWS account integrated with Databricks), otherwise files won't be readable from Databricks hive.s3.upload-acl-type=BUCKET_OWNER_FULL_CONTROL hive.allow-drop-table=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/tempto-configuration.yaml b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/tempto-configuration.yaml index 44988322b17d..82a093453c3d 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/tempto-configuration.yaml +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/tempto-configuration.yaml @@ -2,14 +2,14 @@ databases: presto: jdbc_user: root delta: - jdbc_driver_class: ${DATABRICKS_TEST_JDBC_DRIVER_CLASS} + jdbc_driver_class: com.databricks.client.jdbc.Driver schema: default prepare_statement: - USE ${databases.delta.schema} table_manager_type: jdbc - jdbc_url: ${DATABRICKS_TEST_JDBC_URL} - jdbc_user: ${DATABRICKS_TEST_LOGIN} - jdbc_password: ${DATABRICKS_TEST_TOKEN} + jdbc_url: ${DATABRICKS_JDBC_URL} + jdbc_user: ${DATABRICKS_LOGIN} + jdbc_password: ${DATABRICKS_TOKEN} s3: server_type: aws diff --git a/testing/trino-product-tests/pom.xml b/testing/trino-product-tests/pom.xml index 597fc521886a..82ecbd194b8a 100644 --- a/testing/trino-product-tests/pom.xml +++ b/testing/trino-product-tests/pom.xml @@ -126,6 +126,11 @@ aws-java-sdk-core + + com.amazonaws + aws-java-sdk-glue + + com.amazonaws aws-java-sdk-s3 diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java index 1550e25c241c..09e87f14c90a 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java @@ -78,6 +78,7 @@ public final class TestGroups public static final String DELTA_LAKE_MINIO = "delta-lake-minio"; public static final String DELTA_LAKE_DATABRICKS = "delta-lake-databricks"; public static final String DELTA_LAKE_EXCLUDE_73 = "delta-lake-exclude-73"; + public static final String DELTA_LAKE_EXCLUDE_91 = "delta-lake-exclude-91"; private TestGroups() {} } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDatabricksCompatibilityCleanUp.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDatabricksWithGlueMetastoreCleanUp.java similarity index 68% rename from testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDatabricksCompatibilityCleanUp.java rename to testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDatabricksWithGlueMetastoreCleanUp.java index 5e7afc0d3d3f..a54875b9326a 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDatabricksCompatibilityCleanUp.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDatabricksWithGlueMetastoreCleanUp.java @@ -13,21 +13,20 @@ */ package io.trino.tests.product.deltalake; +import com.amazonaws.services.glue.AWSGlueAsync; +import com.amazonaws.services.glue.AWSGlueAsyncClientBuilder; +import com.amazonaws.services.glue.model.GetTableRequest; +import com.amazonaws.services.glue.model.Table; import com.google.common.collect.ImmutableSet; -import com.google.common.net.HostAndPort; import io.airlift.log.Logger; -import io.trino.plugin.hive.metastore.thrift.NoHiveMetastoreAuthentication; -import io.trino.plugin.hive.metastore.thrift.ThriftHiveMetastoreClient; -import io.trino.plugin.hive.metastore.thrift.Transport; import io.trino.tempto.ProductTest; import io.trino.tempto.query.QueryResult; -import org.apache.hadoop.hive.metastore.api.Table; import org.testng.annotations.Test; -import java.net.URI; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Locale; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -36,53 +35,41 @@ import static io.trino.tests.product.utils.QueryExecutors.onTrino; import static java.lang.String.format; import static java.lang.System.currentTimeMillis; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.TimeUnit.DAYS; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.stream.Collectors.toUnmodifiableList; -public class TestDatabricksCompatibilityCleanUp +public class TestDatabricksWithGlueMetastoreCleanUp extends ProductTest { - private static final Logger log = Logger.get(TestDatabricksCompatibilityCleanUp.class); - private static final long SCHEMA_CLEANUP_THRESHOLD_SECONDS = (currentTimeMillis() / 1000) - DAYS.toSeconds(7); + private static final Logger log = Logger.get(TestDatabricksWithGlueMetastoreCleanUp.class); + private static final Instant SCHEMA_CLEANUP_THRESHOLD = Instant.now().minus(7, ChronoUnit.DAYS); private static final long MAX_JOB_TIME_MILLIS = MINUTES.toMillis(5); @Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS}) public void testCleanUpOldTablesUsingDelta() - throws Exception { - String hiveMetastoreUri = requireNonNull(System.getenv("HIVE_METASTORE_URI"), "Environment HIVE_METASTORE_URI was not set"); - URI metastoreUri = URI.create(hiveMetastoreUri); + AWSGlueAsync glueClient = AWSGlueAsyncClientBuilder.standard().build(); long startTime = currentTimeMillis(); List schemas = onTrino().executeQuery("SELECT DISTINCT(table_schema) FROM information_schema.tables") .rows().stream() .map(row -> (String) row.get(0)) .filter(schema -> schema.toLowerCase(Locale.ROOT).startsWith("test") || schema.equals("default")) .collect(toUnmodifiableList()); - try (ThriftHiveMetastoreClient thriftHiveMetastoreClient = new ThriftHiveMetastoreClient( - Transport.create(HostAndPort.fromParts(metastoreUri.getHost(), metastoreUri.getPort()), - Optional.empty(), - Optional.empty(), - 100000, - new NoHiveMetastoreAuthentication(), - Optional.empty()), - metastoreUri.getHost())) { - // this is needed to make deletion of some views possible - onTrino().executeQuery("SET SESSION hive.hive_views_legacy_translation = true"); - schemas.forEach(schema -> cleanSchema(schema, startTime, thriftHiveMetastoreClient)); - } + + // this is needed to make deletion of some views possible + onTrino().executeQuery("SET SESSION hive.hive_views_legacy_translation = true"); + schemas.forEach(schema -> cleanSchema(schema, startTime, glueClient)); } - private void cleanSchema(String schema, long startTime, ThriftHiveMetastoreClient thriftHiveMetastoreClient) + private void cleanSchema(String schema, long startTime, AWSGlueAsync glueClient) { Set allTableNames = findAllTestTablesInSchema(schema); int droppedTablesCount = 0; for (String tableName : allTableNames) { try { - Table table = thriftHiveMetastoreClient.getTable(schema, tableName); - int createTime = table.getCreateTime(); - if (createTime <= SCHEMA_CLEANUP_THRESHOLD_SECONDS) { + Table table = glueClient.getTable(new GetTableRequest().withDatabaseName(schema).withName(tableName)).getTable(); + Instant createTime = table.getCreateTime().toInstant(); + if (createTime.isBefore(SCHEMA_CLEANUP_THRESHOLD)) { if (table.getTableType() != null && table.getTableType().contains("VIEW")) { onTrino().executeQuery(format("DROP VIEW IF EXISTS %s.%s", schema, tableName)); log.info("Dropped view %s.%s", schema, tableName); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java index e2d32b69aa95..ae426acb2e45 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java @@ -19,6 +19,7 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.tempto.assertions.QueryAssert.assertThat; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_73; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getColumnCommentOnDelta; @@ -54,7 +55,7 @@ public void testAddColumnWithCommentOnTrino() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_73, PROFILE_SPECIFIC_TESTS}) public void testAddColumnUnsupportedWriterVersion() { String tableName = "test_dl_add_column_unsupported_writer_" + randomTableSuffix(); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java index 1b86c6c76793..e6bc5b61416c 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java @@ -23,6 +23,8 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.tempto.assertions.QueryAssert.assertThat; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_73; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_91; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.hive.util.TemporaryHiveTable.randomTableSuffix; @@ -34,7 +36,7 @@ public class TestDeltaLakeColumnMappingMode { // TODO: Add test with 'delta.columnMapping.mode'='id' table property. This requires Databricks runtime version 10.2 or higher version. - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_73, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}) public void testColumnMappingModeNone() { String tableName = "test_dl_column_mapping_mode_none" + randomTableSuffix(); @@ -62,7 +64,7 @@ public void testColumnMappingModeNone() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_73, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}) public void testColumnMappingModeName() { String tableName = "test_dl_column_mapping_mode_name_" + randomTableSuffix(); @@ -117,7 +119,7 @@ public void testColumnMappingModeName() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_73, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}) public void testColumnMappingModeNameWithNonLowerCaseColumnName() { String tableName = "test_dl_column_mapping_mode_name_non_loewr_case_" + randomTableSuffix(); @@ -152,7 +154,7 @@ public void testColumnMappingModeNameWithNonLowerCaseColumnName() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_73, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}) public void testShowStatsFromJsonForColumnMappingModeName() { String tableName = "test_dl_show_stats_json_for_column_mapping_mode_" + randomTableSuffix(); @@ -184,7 +186,7 @@ public void testShowStatsFromJsonForColumnMappingModeName() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_73, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}) public void testShowStatsFromParquetForColumnMappingModeName() { String tableName = "test_dl_show_parquet_stats_parquet_for_column_mapping_mode_" + randomTableSuffix(); @@ -219,7 +221,7 @@ public void testShowStatsFromParquetForColumnMappingModeName() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_73, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}) public void testShowStatsOnPartitionedForColumnMappingModeName() { String tableName = "test_dl_show_stats_partitioned_for_column_mapping_mode_" + randomTableSuffix(); @@ -251,7 +253,7 @@ public void testShowStatsOnPartitionedForColumnMappingModeName() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_73, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS}) public void testUnsupportedOperationsColumnMappingModeName() { String tableName = "test_dl_unsupported_column_mapping_mode_" + randomTableSuffix(); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksUpdates.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksUpdates.java index 005dbdd22c11..98fad6faedfc 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksUpdates.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksUpdates.java @@ -74,26 +74,23 @@ public void testUpdatesFromDatabricks() QueryResult prestoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\" ORDER BY id", tableName)); assertThat(databricksResult).containsExactly(toRows(prestoResult)); - assertThat(onDelta().executeQuery(format("UPDATE default.%s SET value = 'France' WHERE id = 2", tableName))) - .containsOnly(row(-1)); + onDelta().executeQuery(format("UPDATE default.%s SET value = 'France' WHERE id = 2", tableName)); databricksResult = onDelta().executeQuery(format("SELECT * FROM default.%s ORDER BY id", tableName)); prestoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\" ORDER BY id", tableName)); assertThat(databricksResult).containsExactly(toRows(prestoResult)); - assertThat(onDelta().executeQuery(format("UPDATE default.%s SET value = 'Spain' WHERE id = 2", tableName))) - .containsOnly(row(-1)); + onDelta().executeQuery(format("UPDATE default.%s SET value = 'Spain' WHERE id = 2", tableName)); databricksResult = onDelta().executeQuery(format("SELECT * FROM default.%s ORDER BY id", tableName)); prestoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\" ORDER BY id", tableName)); assertThat(databricksResult).containsExactly(toRows(prestoResult)); - assertThat(onDelta().executeQuery(format("UPDATE default.%s SET value = 'Portugal' WHERE id = 2", tableName))) - .containsOnly(row(-1)); + onDelta().executeQuery(format("UPDATE default.%s SET value = 'Portugal' WHERE id = 2", tableName)); databricksResult = onDelta().executeQuery(format("SELECT * FROM default.%s ORDER BY id", tableName)); prestoResult = onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\" ORDER BY id", tableName)); assertThat(databricksResult).containsExactly(toRows(prestoResult)); } finally { - assertThat(onDelta().executeQuery("DROP TABLE default." + tableName)).containsExactly(row(-1)); + onDelta().executeQuery("DROP TABLE default." + tableName); } } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDropTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDropTableCompatibility.java index ca2435cf443c..ab26919c5416 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDropTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDropTableCompatibility.java @@ -60,18 +60,29 @@ public static Object[][] engineConfigurations() {TRINO, DELTA, true}, {TRINO, DELTA, false}, {DELTA, TRINO, true}, - {DELTA, TRINO, false}, {DELTA, DELTA, true}, {DELTA, DELTA, false}, }; } @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "engineConfigurations") - public void testDatabricksManagedTableDroppedFromTrino(Engine creator, Engine dropper, boolean explicitLocation) + public void testDropTable(Engine creator, Engine dropper, boolean explicitLocation) { - String schemaName = "schema_with_location_" + randomTableSuffix(); + testDropTableAccuracy(creator, dropper, explicitLocation); + } + + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + public void testCreateManagedTableInDeltaDropTableInTrino() + { + //TODO Integrate this method into `engineConfigurations()` data provider method after dealing with https://github.com/trinodb/trino/issues/13017 + testDropTableAccuracy(DELTA, TRINO, false); + } + + private void testDropTableAccuracy(Engine creator, Engine dropper, boolean explicitLocation) + { + String schemaName = "test_schema_with_location_" + randomTableSuffix(); String schemaLocation = format("s3://%s/databricks-compatibility-test-%s", bucketName, schemaName); - String tableName = explicitLocation ? "external_table" : "managed_table"; + String tableName = explicitLocation ? "test_external_table" : "test_managed_table"; Optional tableLocation = explicitLocation ? Optional.of(format("s3://" + bucketName + "/databricks-compatibility-test-%s/%s", schemaName, tableName)) : Optional.empty(); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestHiveAndDeltaLakeCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestHiveAndDeltaLakeCompatibility.java index 19bd9d153d51..9cd338db0a4d 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestHiveAndDeltaLakeCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestHiveAndDeltaLakeCompatibility.java @@ -31,7 +31,7 @@ public class TestHiveAndDeltaLakeCompatibility public void testInformationSchemaColumnsOnPresenceOfHiveView() { // use dedicated schema so we control the number and shape of tables - String schemaName = "redirect_to_delta_information_schema_columns_schema_" + randomTableSuffix(); + String schemaName = "test_redirect_to_delta_information_schema_columns_schema_" + randomTableSuffix(); onTrino().executeQuery("CREATE SCHEMA IF NOT EXISTS hive." + schemaName); String hiveViewName = "delta_schema_columns_hive_view_" + randomTableSuffix(); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestHiveAndDeltaLakeRedirect.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestHiveAndDeltaLakeRedirect.java index 5d01490541ed..4d22c4cd5483 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestHiveAndDeltaLakeRedirect.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestHiveAndDeltaLakeRedirect.java @@ -15,6 +15,7 @@ import com.google.common.collect.ImmutableList; import io.trino.tempto.assertions.QueryAssert.Row; +import io.trino.tempto.query.QueryExecutionException; import io.trino.tempto.query.QueryResult; import org.assertj.core.api.AbstractStringAssert; import org.assertj.core.api.Assertions; @@ -49,9 +50,9 @@ public class TestHiveAndDeltaLakeRedirect @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testHiveToDeltaRedirect() { - String tableName = "redirect_to_delta_" + randomTableSuffix(); + String tableName = "test_redirect_to_delta_" + randomTableSuffix(); - onDelta().executeQuery(createTableInDatabricks(tableName, false)); + onDelta().executeQuery(createTableOnDelta(tableName, false)); try { QueryResult databricksResult = onDelta().executeQuery("SELECT * FROM " + tableName); @@ -68,34 +69,37 @@ public void testHiveToDeltaRedirect() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testHiveToDeltaNonDefaultSchemaRedirect() { - String tableName = "redirect_to_delta_non_default_schema_" + randomTableSuffix(); + String schemaName = "test_extraordinary_" + randomTableSuffix(); + String tableName = "test_redirect_to_delta_non_default_schema_" + randomTableSuffix(); - onDelta().executeQuery("CREATE SCHEMA IF NOT EXISTS extraordinary"); - onDelta().executeQuery(createTableInDatabricks("extraordinary", tableName, false)); + String schemaLocation = format("s3://%s/delta-redirect-test-%s", bucketName, schemaName); + onDelta().executeQuery(format("CREATE SCHEMA IF NOT EXISTS %s LOCATION \"%s\"", schemaName, schemaLocation)); + onDelta().executeQuery(createTableOnDelta(schemaName, tableName, false)); try { - QueryResult databricksResult = onDelta().executeQuery("SELECT * FROM extraordinary." + tableName); - QueryResult hiveResult = onTrino().executeQuery(format("SELECT * FROM hive.extraordinary.\"%s\"", tableName)); + QueryResult databricksResult = onDelta().executeQuery(format("SELECT * FROM %s.%s", schemaName, tableName)); + QueryResult hiveResult = onTrino().executeQuery(format("SELECT * FROM hive.%s.\"%s\"", schemaName, tableName)); assertThat(databricksResult).containsOnly(hiveResult.rows().stream() .map(Row::new) .collect(toImmutableList())); } finally { - onDelta().executeQuery("DROP TABLE extraordinary." + tableName); + onDelta().executeQuery(format("DROP TABLE %s.%s", schemaName, tableName)); + onDelta().executeQuery("DROP SCHEMA " + schemaName); } } @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testHiveToNonexistentDeltaCatalogRedirectFailure() { - String tableName = "redirect_to_nonexistent_delta_" + randomTableSuffix(); + String tableName = "test_redirect_to_nonexistent_delta_" + randomTableSuffix(); try { - onDelta().executeQuery(createTableInDatabricks(tableName, false)); + onDelta().executeQuery(createTableOnDelta(tableName, false)); onTrino().executeQuery("SET SESSION hive.delta_lake_catalog_name = 'epsilon'"); assertQueryFailure(() -> onTrino().executeQuery(format("SELECT * FROM hive.default.\"%s\"", tableName))) - .hasMessageMatching(".*Table 'hive.default.redirect_to_nonexistent_delta_.*' redirected to 'epsilon.default.redirect_to_nonexistent_delta_.*', but the target catalog 'epsilon' does not exist"); + .hasMessageMatching(".*Table 'hive.default.test_redirect_to_nonexistent_delta_.*' redirected to 'epsilon.default.test_redirect_to_nonexistent_delta_.*', but the target catalog 'epsilon' does not exist"); } finally { onDelta().executeQuery("DROP TABLE " + tableName); @@ -106,9 +110,9 @@ public void testHiveToNonexistentDeltaCatalogRedirectFailure() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testHiveToDeltaRedirectWithDefaultSchemaInSession() { - String tableName = "redirect_to_delta_with_use_" + randomTableSuffix(); + String tableName = "test_redirect_to_delta_with_use_" + randomTableSuffix(); - onDelta().executeQuery(createTableInDatabricks(tableName, false)); + onDelta().executeQuery(createTableOnDelta(tableName, false)); try { onTrino().executeQuery("USE hive.default"); @@ -127,14 +131,14 @@ public void testHiveToDeltaRedirectWithDefaultSchemaInSession() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testHiveToUnpartitionedDeltaPartitionsRedirectFailure() { - String tableName = "delta_lake_unpartitioned_table_" + randomTableSuffix(); + String tableName = "test_delta_lake_unpartitioned_table_" + randomTableSuffix(); - onDelta().executeQuery(createTableInDatabricks(tableName, false)); + onDelta().executeQuery(createTableOnDelta(tableName, false)); try { assertQueryFailure(() -> onTrino().executeQuery(format("SELECT * FROM hive.default.\"%s$partitions\"", tableName))) - .hasMessageMatching(".*Table 'hive.default.delta_lake_unpartitioned_table_.*\\$partitions' redirected to 'delta.default.delta_lake_unpartitioned_table_.*\\$partitions', " + - "but the target table 'delta.default.delta_lake_unpartitioned_table_.*\\$partitions' does not exist"); + .hasMessageMatching(".*Table 'hive.default.test_delta_lake_unpartitioned_table_.*\\$partitions' redirected to 'delta.default.test_delta_lake_unpartitioned_table_.*\\$partitions', " + + "but the target table 'delta.default.test_delta_lake_unpartitioned_table_.*\\$partitions' does not exist"); } finally { onDelta().executeQuery("DROP TABLE " + tableName); @@ -144,14 +148,14 @@ public void testHiveToUnpartitionedDeltaPartitionsRedirectFailure() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testHiveToPartitionedDeltaPartitionsRedirectFailure() { - String tableName = "delta_lake_partitioned_table_" + randomTableSuffix(); + String tableName = "test_delta_lake_partitioned_table_" + randomTableSuffix(); - onDelta().executeQuery(createTableInDatabricks(tableName, true)); + onDelta().executeQuery(createTableOnDelta(tableName, true)); try { assertQueryFailure(() -> onTrino().executeQuery(format("SELECT * FROM hive.default.\"%s$partitions\"", tableName))) - .hasMessageMatching(".*Table 'hive.default.delta_lake_partitioned_table_.*\\$partitions' redirected to 'delta.default.delta_lake_partitioned_table_.*\\$partitions', " + - "but the target table 'delta.default.delta_lake_partitioned_table_.*\\$partitions' does not exist"); + .hasMessageMatching(".*Table 'hive.default.test_delta_lake_partitioned_table_.*\\$partitions' redirected to 'delta.default.test_delta_lake_partitioned_table_.*\\$partitions', " + + "but the target table 'delta.default.test_delta_lake_partitioned_table_.*\\$partitions' does not exist"); } finally { onDelta().executeQuery("DROP TABLE " + tableName); @@ -161,7 +165,7 @@ public void testHiveToPartitionedDeltaPartitionsRedirectFailure() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testDeltaToHiveRedirect() { - String tableName = "redirect_to_hive_" + randomTableSuffix(); + String tableName = "test_redirect_to_hive_" + randomTableSuffix(); onTrino().executeQuery(createTableInHiveConnector("default", tableName, false)); @@ -185,11 +189,13 @@ public void testDeltaToHiveRedirect() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testDeltaToHiveNonDefaultSchemaRedirect() { - String tableName = "redirect_to_hive_non_default_schema_" + randomTableSuffix(); + String schemaName = "test_extraordinary" + randomTableSuffix(); + String schemaLocation = format("s3://%s/delta-redirect-test-%s", bucketName, schemaName); + String tableName = "test_redirect_to_hive_non_default_schema_" + randomTableSuffix(); - onTrino().executeQuery("CREATE SCHEMA IF NOT EXISTS hive.extraordinary"); + onTrino().executeQuery(format("CREATE SCHEMA IF NOT EXISTS hive.%s WITH (location='%s')", schemaName, schemaLocation)); - onTrino().executeQuery(createTableInHiveConnector("extraordinary", tableName, false)); + onTrino().executeQuery(createTableInHiveConnector(schemaName, tableName, false)); try { List expectedResults = ImmutableList.of( @@ -198,20 +204,21 @@ public void testDeltaToHiveNonDefaultSchemaRedirect() row(3, false, 0), row(4, false, 1), row(5, true, 37)); - QueryResult deltaResult = onTrino().executeQuery(format("SELECT * FROM delta.extraordinary.\"%s\"", tableName)); - QueryResult hiveResult = onTrino().executeQuery(format("SELECT * FROM hive.extraordinary.\"%s\"", tableName)); + QueryResult deltaResult = onTrino().executeQuery(format("SELECT * FROM delta.%s.\"%s\"", schemaName, tableName)); + QueryResult hiveResult = onTrino().executeQuery(format("SELECT * FROM hive.%s.\"%s\"", schemaName, tableName)); assertThat(deltaResult).containsOnly(expectedResults); assertThat(hiveResult).containsOnly(expectedResults); } finally { - onTrino().executeQuery("DROP TABLE hive.extraordinary." + tableName); + onTrino().executeQuery(format("DROP TABLE hive.%s.%s", schemaName, tableName)); + onTrino().executeQuery("DROP SCHEMA " + schemaName); } } @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testDeltaToNonexistentHiveCatalogRedirectFailure() { - String tableName = "redirect_to_nonexistent_hive_" + randomTableSuffix(); + String tableName = "test_redirect_to_nonexistent_hive_" + randomTableSuffix(); onTrino().executeQuery(createTableInHiveConnector("default", tableName, false)); @@ -219,7 +226,7 @@ public void testDeltaToNonexistentHiveCatalogRedirectFailure() onTrino().executeQuery("SET SESSION delta.hive_catalog_name = 'spark'"); assertQueryFailure(() -> onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s\"", tableName))) - .hasMessageMatching(".*Table 'delta.default.redirect_to_nonexistent_hive_.*' redirected to 'spark.default.redirect_to_nonexistent_hive_.*', but the target catalog 'spark' does not exist"); + .hasMessageMatching(".*Table 'delta.default.test_redirect_to_nonexistent_hive_.*' redirected to 'spark.default.test_redirect_to_nonexistent_hive_.*', but the target catalog 'spark' does not exist"); } finally { onTrino().executeQuery("DROP TABLE hive.default." + tableName); @@ -230,7 +237,7 @@ public void testDeltaToNonexistentHiveCatalogRedirectFailure() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testDeltaToHiveRedirectWithDefaultSchemaInSession() { - String tableName = "redirect_to_hive_with_use_" + randomTableSuffix(); + String tableName = "test_redirect_to_hive_with_use_" + randomTableSuffix(); onTrino().executeQuery("USE hive.default"); @@ -256,7 +263,7 @@ public void testDeltaToHiveRedirectWithDefaultSchemaInSession() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testDeltaToPartitionedHivePartitionsRedirect() { - String tableName = "hive_partitioned_table_" + randomTableSuffix(); + String tableName = "test_hive_partitioned_table_" + randomTableSuffix(); onTrino().executeQuery(createTableInHiveConnector("default", tableName, true)); @@ -280,13 +287,13 @@ public void testDeltaToPartitionedHivePartitionsRedirect() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testDeltaToUnpartitionedHivePartitionsRedirectFailure() { - String tableName = "hive_unpartitioned_table_" + randomTableSuffix(); + String tableName = "test_hive_unpartitioned_table_" + randomTableSuffix(); onTrino().executeQuery(createTableInHiveConnector("default", tableName, false)); try { assertQueryFailure(() -> onTrino().executeQuery(format("SELECT * FROM delta.default.\"%s$partitions\"", tableName))) - .hasMessageMatching(".*Table 'delta.default.hive_unpartitioned_table.*partitions' does not exist"); + .hasMessageMatching(".*Table 'delta.default.test_hive_unpartitioned_table.*partitions' does not exist"); } finally { onTrino().executeQuery("DROP TABLE hive.default." + tableName); @@ -296,7 +303,7 @@ public void testDeltaToUnpartitionedHivePartitionsRedirectFailure() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testDeltaToHiveInsert() { - String tableName = "hive_insert_by_delta_" + randomTableSuffix(); + String tableName = "test_hive_insert_by_delta_" + randomTableSuffix(); onTrino().executeQuery(createTableInHiveConnector("default", tableName, true)); @@ -325,9 +332,9 @@ public void testDeltaToHiveInsert() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testHiveToDeltaInsert() { - String tableName = "delta_insert_by_hive_" + randomTableSuffix(); + String tableName = "test_delta_insert_by_hive_" + randomTableSuffix(); - onDelta().executeQuery(createTableInDatabricks(tableName, true)); + onDelta().executeQuery(createTableOnDelta(tableName, true)); try { onTrino().executeQuery(format("INSERT INTO hive.default.\"%s\" VALUES (1234567890, 'San Escobar', 5, 'If I had a world of my own, everything would be nonsense')", tableName)); @@ -343,7 +350,7 @@ public void testHiveToDeltaInsert() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testDeltaToHiveDescribe() { - String tableName = "hive_describe_by_delta_" + randomTableSuffix(); + String tableName = "test_hive_describe_by_delta_" + randomTableSuffix(); onTrino().executeQuery(createTableInHiveConnector("default", tableName, true)); @@ -363,9 +370,9 @@ public void testDeltaToHiveDescribe() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testHiveToDeltaDescribe() { - String tableName = "delta_describe_by_hive_" + randomTableSuffix(); + String tableName = "test_delta_describe_by_hive_" + randomTableSuffix(); - onDelta().executeQuery(createTableInDatabricks(tableName, true)); + onDelta().executeQuery(createTableOnDelta(tableName, true)); try { List expectedResults = ImmutableList.of( @@ -384,7 +391,7 @@ public void testHiveToDeltaDescribe() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testDeltaToHiveShowCreateTable() { - String tableName = "hive_show_create_table_by_delta_" + randomTableSuffix(); + String tableName = "test_hive_show_create_table_by_delta_" + randomTableSuffix(); onTrino().executeQuery(createTableInHiveConnector("default", tableName, true)); @@ -400,9 +407,9 @@ public void testDeltaToHiveShowCreateTable() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testHiveToDeltaShowCreateTable() { - String tableName = "delta_show_create_table_by_hive_" + randomTableSuffix(); + String tableName = "test_delta_show_create_table_by_hive_" + randomTableSuffix(); - onDelta().executeQuery(createTableInDatabricks(tableName, true)); + onDelta().executeQuery(createTableOnDelta(tableName, true)); try { assertThat(onTrino().executeQuery(format("SHOW CREATE TABLE hive.default.\"%s\"", tableName))) @@ -416,12 +423,19 @@ public void testHiveToDeltaShowCreateTable() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testDeltaToHiveAlterTable() { - String tableName = "hive_alter_table_by_delta_" + randomTableSuffix(); + String tableName = "test_hive_alter_table_by_delta_" + randomTableSuffix(); // TODO set the partitioning for the table to `true` after the fix of https://github.com/trinodb/trino/issues/11826 onTrino().executeQuery(createTableInHiveConnector("default", tableName, false)); String newTableName = tableName + "_new"; try { onTrino().executeQuery("ALTER TABLE delta.default." + tableName + " RENAME TO " + newTableName); + } + catch (QueryExecutionException e) { + onTrino().executeQuery("DROP TABLE hive.default." + tableName); + throw e; + } + + try { assertResultsEqual( onTrino().executeQuery("TABLE hive.default." + newTableName), onTrino().executeQuery("TABLE delta.default." + newTableName)); @@ -434,23 +448,25 @@ public void testDeltaToHiveAlterTable() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testHiveToDeltaAlterTable() { - String tableName = "delta_alter_table_by_hive_" + randomTableSuffix(); + String tableName = "test_delta_alter_table_by_hive_" + randomTableSuffix(); String newTableName = tableName + "_new"; - onDelta().executeQuery(createTableInDatabricks(tableName, true)); + onDelta().executeQuery(createTableOnDelta(tableName, true)); try { onTrino().executeQuery("ALTER TABLE hive.default.\"" + tableName + "\" RENAME TO \"" + newTableName + "\""); - } - finally { onDelta().executeQuery("DROP TABLE " + newTableName); } + catch (QueryExecutionException e) { + onDelta().executeQuery("DROP TABLE " + tableName); + throw e; + } } @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testDeltaToHiveCommentTable() { - String tableName = "hive_comment_table_by_delta_" + randomTableSuffix(); + String tableName = "test_hive_comment_table_by_delta_" + randomTableSuffix(); onTrino().executeQuery(createTableInHiveConnector("default", tableName, true)); try { @@ -470,9 +486,9 @@ public void testDeltaToHiveCommentTable() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testHiveToDeltaCommentTable() { - String tableName = "delta_comment_table_by_hive_" + randomTableSuffix(); + String tableName = "test_delta_comment_table_by_hive_" + randomTableSuffix(); - onDelta().executeQuery(createTableInDatabricks(tableName, true)); + onDelta().executeQuery(createTableOnDelta(tableName, true)); try { assertThat(onTrino().executeQuery("SELECT comment FROM system.metadata.table_comments WHERE catalog_name = 'delta' AND schema_name = 'default' AND table_name = '" + tableName + "'")) @@ -491,7 +507,7 @@ public void testHiveToDeltaCommentTable() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testDeltaToHiveCommentColumn() { - String tableName = "hive_comment_column_by_delta_" + randomTableSuffix(); + String tableName = "test_hive_comment_column_by_delta_" + randomTableSuffix(); String columnName = "id"; onTrino().executeQuery(createTableInHiveConnector("default", tableName, true)); @@ -513,10 +529,10 @@ public void testDeltaToHiveCommentColumn() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testHiveToDeltaCommentColumn() { - String tableName = "delta_comment_column_by_hive_" + randomTableSuffix(); + String tableName = "test_delta_comment_column_by_hive_" + randomTableSuffix(); String columnName = "nationkey"; - onDelta().executeQuery(createTableInDatabricks(tableName, true)); + onDelta().executeQuery(createTableOnDelta(tableName, true)); try { assertColumnComment("hive", "default", tableName, columnName).isNull(); @@ -536,11 +552,12 @@ public void testHiveToDeltaCommentColumn() @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) public void testInsertIntoDeltaTableFromHiveNonDefaultSchemaRedirect() { - String destSchema = "extraordinary"; - String destTableName = "create_delta_table_from_hive_non_default_schema_" + randomTableSuffix(); + String destSchema = "test_extraordinary_" + randomTableSuffix(); + String destTableName = "test_create_delta_table_from_hive_non_default_schema_" + randomTableSuffix(); - onDelta().executeQuery("CREATE SCHEMA IF NOT EXISTS extraordinary"); - onDelta().executeQuery(createTableInDatabricks(destSchema, destTableName, false)); + String schemaLocation = format("s3://%s/delta-redirect-test-%s", bucketName, destSchema); + onDelta().executeQuery(format("CREATE SCHEMA IF NOT EXISTS %s LOCATION \"%s\"", destSchema, schemaLocation)); + onDelta().executeQuery(createTableOnDelta(destSchema, destTableName, false)); try { onTrino().executeQuery(format("INSERT INTO hive.%s.\"%s\" (nationkey, name, regionkey) VALUES (26, 'POLAND', 3)", destSchema, destTableName)); @@ -562,7 +579,8 @@ public void testInsertIntoDeltaTableFromHiveNonDefaultSchemaRedirect() .containsOnly(expectedDestinationTableRows); } finally { - onDelta().executeQuery("DROP TABLE extraordinary." + destTableName); + onDelta().executeQuery(format("DROP TABLE %s.%s", destSchema, destTableName)); + onTrino().executeQuery("DROP SCHEMA " + destSchema); } } @@ -570,12 +588,13 @@ public void testInsertIntoDeltaTableFromHiveNonDefaultSchemaRedirect() public void testInformationSchemaColumnsHiveToDeltaRedirect() { // use dedicated schema so we control the number and shape of tables - String schemaName = "redirect_to_delta_information_schema_columns_schema_" + randomTableSuffix(); - onTrino().executeQuery("CREATE SCHEMA IF NOT EXISTS hive." + schemaName); + String schemaName = "test_redirect_to_delta_information_schema_columns_schema_" + randomTableSuffix(); + String schemaLocation = format("s3://%s/delta-redirect-test-%s", bucketName, schemaName); + onTrino().executeQuery(format("CREATE SCHEMA IF NOT EXISTS hive.%s WITH (location = '%s')", schemaName, schemaLocation)); String tableName = "redirect_to_delta_information_schema_columns_table_" + randomTableSuffix(); try { - onDelta().executeQuery(createTableInDatabricks(schemaName, tableName, false)); + onDelta().executeQuery(createTableOnDelta(schemaName, tableName, false)); // via redirection with table filter assertThat(onTrino().executeQuery( @@ -614,10 +633,11 @@ public void testInformationSchemaColumnsHiveToDeltaRedirect() public void testInformationSchemaColumnsDeltaToHiveRedirect() { // use dedicated schema so we control the number and shape of tables - String schemaName = "redirect_to_hive_information_schema_columns_schema_" + randomTableSuffix(); - onTrino().executeQuery("CREATE SCHEMA IF NOT EXISTS hive." + schemaName); + String schemaName = "test_redirect_to_hive_information_schema_columns_schema_" + randomTableSuffix(); + String schemaLocation = format("s3://%s/delta-redirect-test-%s", bucketName, schemaName); + onTrino().executeQuery(format("CREATE SCHEMA IF NOT EXISTS hive.%s WITH (location='%s')", schemaName, schemaLocation)); - String tableName = "redirect_to_hive_information_schema_columns_table_" + randomTableSuffix(); + String tableName = "test_redirect_to_hive_information_schema_columns_table_" + randomTableSuffix(); try { onTrino().executeQuery(createTableInHiveConnector(schemaName, tableName, false)); @@ -655,12 +675,13 @@ public void testInformationSchemaColumnsDeltaToHiveRedirect() public void testSystemJdbcColumnsHiveToDeltaRedirect() { // use dedicated schema so we control the number and shape of tables - String schemaName = "redirect_to_delta_system_jdbc_columns_schema_" + randomTableSuffix(); - onTrino().executeQuery("CREATE SCHEMA IF NOT EXISTS hive." + schemaName); + String schemaName = "test_redirect_to_delta_system_jdbc_columns_schema_" + randomTableSuffix(); + String schemaLocation = format("s3://%s/delta-redirect-test-%s", bucketName, schemaName); + onTrino().executeQuery(format("CREATE SCHEMA IF NOT EXISTS hive.%s WITH (location='%s')", schemaName, schemaLocation)); - String tableName = "redirect_to_delta_system_jdbc_columns_table_" + randomTableSuffix(); + String tableName = "test_redirect_to_delta_system_jdbc_columns_table_" + randomTableSuffix(); try { - onDelta().executeQuery(createTableInDatabricks(schemaName, tableName, false)); + onDelta().executeQuery(createTableOnDelta(schemaName, tableName, false)); // via redirection with table filter assertThat(onTrino().executeQuery( @@ -700,10 +721,11 @@ public void testSystemJdbcColumnsHiveToDeltaRedirect() public void testSystemJdbcColumnsDeltaToHiveRedirect() { // use dedicated schema so we control the number and shape of tables - String schemaName = "redirect_to_hive_system_jdbc_columns_schema_" + randomTableSuffix(); - onTrino().executeQuery("CREATE SCHEMA IF NOT EXISTS hive." + schemaName); + String schemaName = "test_redirect_to_hive_system_jdbc_columns_schema_" + randomTableSuffix(); + String schemaLocation = format("s3://%s/delta-redirect-test-%s", bucketName, schemaName); + onTrino().executeQuery(format("CREATE SCHEMA IF NOT EXISTS hive.%s WITH (location='%s')", schemaName, schemaLocation)); - String tableName = "redirect_to_hive_system_jdbc_columns_table_" + randomTableSuffix(); + String tableName = "test_redirect_to_hive_system_jdbc_columns_table_" + randomTableSuffix(); try { onTrino().executeQuery(createTableInHiveConnector(schemaName, tableName, false)); @@ -828,13 +850,13 @@ public Object[][] trueFalse() return new Object[][] {{true}, {false}}; } - private String createTableInDatabricks(String tableName, boolean partitioned) + private String createTableOnDelta(String tableName, boolean partitioned) { - return createTableInDatabricks("default", tableName, partitioned); + return createTableOnDelta("default", tableName, partitioned); } @Language("SQL") - private String createTableInDatabricks(String schema, String tableName, boolean partitioned) + private String createTableOnDelta(String schema, String tableName, boolean partitioned) { return "CREATE TABLE " + schema + "." + tableName + " " + "USING DELTA " +