diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7087522eee63..a7159b3cd7b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -883,7 +883,6 @@ jobs: - suite-7-non-generic - suite-hive-transactional - suite-azure - - suite-delta-lake-databricks104 - suite-delta-lake-databricks113 - suite-delta-lake-databricks122 - suite-delta-lake-databricks133 @@ -923,9 +922,6 @@ jobs: ignore exclusion if: >- ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.GCP_CREDENTIALS_KEY != '' }} - - suite: suite-delta-lake-databricks104 - ignore exclusion if: >- - ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} - suite: suite-delta-lake-databricks113 ignore exclusion if: >- ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} @@ -990,7 +986,6 @@ jobs: AWS_REGION: "" TRINO_AWS_ACCESS_KEY_ID: "" TRINO_AWS_SECRET_ACCESS_KEY: "" - DATABRICKS_104_JDBC_URL: "" DATABRICKS_113_JDBC_URL: "" DATABRICKS_122_JDBC_URL: "" DATABRICKS_133_JDBC_URL: "" @@ -1067,7 +1062,6 @@ jobs: AWS_REGION: ${{ vars.TRINO_AWS_REGION }} TRINO_AWS_ACCESS_KEY_ID: ${{ vars.TRINO_AWS_ACCESS_KEY_ID }} TRINO_AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }} - DATABRICKS_104_JDBC_URL: ${{ vars.DATABRICKS_104_JDBC_URL }} DATABRICKS_113_JDBC_URL: ${{ vars.DATABRICKS_113_JDBC_URL }} DATABRICKS_122_JDBC_URL: ${{ vars.DATABRICKS_122_JDBC_URL }} DATABRICKS_133_JDBC_URL: ${{ vars.DATABRICKS_133_JDBC_URL }} diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java index 8c3da6fcc062..a7eb09f73bbd 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java @@ -125,6 +125,7 @@ public class TestDeltaLakeBasic new ResourceTable("no_column_stats", "databricks73/no_column_stats"), new ResourceTable("liquid_clustering", "deltalake/liquid_clustering"), new ResourceTable("region_91_lts", "databricks91/region"), + new ResourceTable("region_104_lts", "databricks104/region"), new ResourceTable("timestamp_ntz", "databricks131/timestamp_ntz"), new ResourceTable("timestamp_ntz_partition", "databricks131/timestamp_ntz_partition"), new ResourceTable("uniform_hudi", "deltalake/uniform_hudi"), @@ -218,6 +219,14 @@ void testDatabricks91() .matches("SELECT * FROM tpch.tiny.region"); } + @Test + void testDatabricks104() + { + assertThat(query("SELECT * FROM region_104_lts")) + .skippingTypesCheck() // name and comment columns are unbounded varchar in Delta Lake and bounded varchar in TPCH + .matches("SELECT * FROM tpch.tiny.region"); + } + @Test public void testNoColumnStats() { diff --git a/plugin/trino-delta-lake/src/test/resources/databricks104/region/README.md b/plugin/trino-delta-lake/src/test/resources/databricks104/region/README.md new file mode 100644 index 000000000000..094c62593cb0 --- /dev/null +++ b/plugin/trino-delta-lake/src/test/resources/databricks104/region/README.md @@ -0,0 +1,13 @@ +Data generated using Databricks 10.4: + +```sql +CREATE TABLE default.region (regionkey bigint, name string, comment string) +USING DELTA LOCATION 's3://trino-ci-test/default/region'; + +INSERT INTO default.region VALUES +(0, 'AFRICA', 'lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to '), +(1, 'AMERICA', 'hs use ironic, even requests. s'), +(2, 'ASIA', 'ges. thinly even pinto beans ca'), +(3, 'EUROPE', 'ly final courts cajole furiously final excuse'), +(4, 'MIDDLE EAST', 'uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl'); +``` diff --git a/plugin/trino-delta-lake/src/test/resources/databricks104/region/_delta_log/00000000000000000000.json b/plugin/trino-delta-lake/src/test/resources/databricks104/region/_delta_log/00000000000000000000.json new file mode 100644 index 000000000000..4c880ca0b7eb --- /dev/null +++ b/plugin/trino-delta-lake/src/test/resources/databricks104/region/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1738882384750,"userId":"7853186923043731","userName":"yuya.ebihara@starburstdata.com","operation":"CREATE TABLE","operationParameters":{"isManaged":"false","description":null,"partitionBy":"[]","properties":"{}"},"notebook":{"notebookId":"1841155838656679"},"clusterId":"0705-100936-r65whg38","isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"38b7030f-ef5e-4fa7-a511-5687aa66081b"}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"fe62a9c4-7768-4a9a-a498-9706284fc221","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"regionkey\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"comment\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1738882384385}} diff --git a/plugin/trino-delta-lake/src/test/resources/databricks104/region/_delta_log/00000000000000000001.json b/plugin/trino-delta-lake/src/test/resources/databricks104/region/_delta_log/00000000000000000001.json new file mode 100644 index 000000000000..1a0f607bc2a1 --- /dev/null +++ b/plugin/trino-delta-lake/src/test/resources/databricks104/region/_delta_log/00000000000000000001.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1738882405690,"userId":"7853186923043731","userName":"yuya.ebihara@starburstdata.com","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1841155838656679"},"clusterId":"0705-100936-r65whg38","readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"5","numOutputBytes":"3497"},"engineInfo":"Databricks-Runtime/10.4.x-scala2.12","txnId":"c6563eac-ecba-40e4-b2c8-0820ba2b9594"}} +{"add":{"path":"part-00000-c5f42889-aac9-4899-8ca6-f9db1e07d400-c000.snappy.parquet","partitionValues":{},"size":1725,"modificationTime":1738882406000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"regionkey\":0,\"name\":\"AFRICA\",\"comment\":\"hs use ironic, even requests. s\"},\"maxValues\":{\"regionkey\":1,\"name\":\"AMERICA\",\"comment\":\"lar deposits. blithely final pac�\"},\"nullCount\":{\"regionkey\":0,\"name\":0,\"comment\":0}}","tags":{"INSERTION_TIME":"1738882406000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} +{"add":{"path":"part-00001-76423f81-c703-421f-bb1a-c227a8c914b3-c000.snappy.parquet","partitionValues":{},"size":1772,"modificationTime":1738882406000,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"regionkey\":2,\"name\":\"ASIA\",\"comment\":\"ges. thinly even pinto beans ca\"},\"maxValues\":{\"regionkey\":4,\"name\":\"MIDDLE EAST\",\"comment\":\"uickly special accounts cajole c�\"},\"nullCount\":{\"regionkey\":0,\"name\":0,\"comment\":0}}","tags":{"INSERTION_TIME":"1738882406000001","OPTIMIZE_TARGET_SIZE":"268435456"}}} diff --git a/plugin/trino-delta-lake/src/test/resources/databricks104/region/part-00000-c5f42889-aac9-4899-8ca6-f9db1e07d400-c000.snappy.parquet b/plugin/trino-delta-lake/src/test/resources/databricks104/region/part-00000-c5f42889-aac9-4899-8ca6-f9db1e07d400-c000.snappy.parquet new file mode 100644 index 000000000000..c26f976710af Binary files /dev/null and b/plugin/trino-delta-lake/src/test/resources/databricks104/region/part-00000-c5f42889-aac9-4899-8ca6-f9db1e07d400-c000.snappy.parquet differ diff --git a/plugin/trino-delta-lake/src/test/resources/databricks104/region/part-00001-76423f81-c703-421f-bb1a-c227a8c914b3-c000.snappy.parquet b/plugin/trino-delta-lake/src/test/resources/databricks104/region/part-00001-76423f81-c703-421f-bb1a-c227a8c914b3-c000.snappy.parquet new file mode 100644 index 000000000000..d0699765f352 Binary files /dev/null and b/plugin/trino-delta-lake/src/test/resources/databricks104/region/part-00001-76423f81-c703-421f-bb1a-c227a8c914b3-c000.snappy.parquet differ diff --git a/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java b/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java index 11c13365cdb9..4c9057bd9b5a 100644 --- a/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java +++ b/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java @@ -96,12 +96,11 @@ public final class TestGroups public static final String DELTA_LAKE_AZURE = "delta-lake-azure"; public static final String DELTA_LAKE_GCS = "delta-lake-gcs"; public static final String DELTA_LAKE_DATABRICKS = "delta-lake-databricks"; - public static final String DELTA_LAKE_DATABRICKS_113 = "delta-lake-databricks-113"; public static final String DELTA_LAKE_DATABRICKS_122 = "delta-lake-databricks-122"; public static final String DELTA_LAKE_DATABRICKS_133 = "delta-lake-databricks-133"; public static final String DELTA_LAKE_DATABRICKS_143 = "delta-lake-databricks-143"; public static final String DATABRICKS_UNITY_HTTP_HMS = "databricks-unity-http-hms"; - public static final String DELTA_LAKE_EXCLUDE_104 = "delta-lake-exclude-104"; + public static final String DELTA_LAKE_EXCLUDE_113 = "delta-lake-exclude-113"; public static final String DELTA_LAKE_ALLUXIO_CACHING = "delta-lake-alluxio-caching"; public static final String HUDI = "hudi"; public static final String PARQUET = "parquet"; diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks104.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks104.java deleted file mode 100644 index cf204d8d18ab..000000000000 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks104.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.tests.product.launcher.env.environment; - -import com.google.inject.Inject; -import io.trino.tests.product.launcher.docker.DockerFiles; -import io.trino.tests.product.launcher.env.common.Standard; -import io.trino.tests.product.launcher.env.common.TestsEnvironment; - -import static io.trino.testing.SystemEnvironmentUtils.requireEnv; - -@TestsEnvironment -public class EnvSinglenodeDeltaLakeDatabricks104 - extends AbstractSinglenodeDeltaLakeDatabricks -{ - @Inject - public EnvSinglenodeDeltaLakeDatabricks104(Standard standard, DockerFiles dockerFiles) - { - super(standard, dockerFiles); - } - - @Override - String databricksTestJdbcUrl() - { - return requireEnv("DATABRICKS_104_JDBC_URL"); - } -} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks104.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks104.java deleted file mode 100644 index decb318190d4..000000000000 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks104.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.tests.product.launcher.suite.suites; - -import com.google.common.collect.ImmutableList; -import io.trino.tests.product.launcher.env.EnvironmentConfig; -import io.trino.tests.product.launcher.env.environment.EnvSinglenodeDeltaLakeDatabricks104; -import io.trino.tests.product.launcher.suite.SuiteDeltaLakeDatabricks; -import io.trino.tests.product.launcher.suite.SuiteTestRun; - -import java.util.List; - -import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; -import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment; - -public class SuiteDeltaLakeDatabricks104 - extends SuiteDeltaLakeDatabricks -{ - @Override - public List getTestRuns(EnvironmentConfig config) - { - return ImmutableList.of( - testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks104.class) - .withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS) - .withExcludedGroups(DELTA_LAKE_EXCLUDE_104) - .withExcludedTests(getExcludedTests()) - .build()); - } -} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks113.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks113.java index dc6c51530ea4..ae9f51cfb3ec 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks113.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks113.java @@ -22,7 +22,8 @@ import java.util.List; import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_113; import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment; public class SuiteDeltaLakeDatabricks113 @@ -33,7 +34,8 @@ public List getTestRuns(EnvironmentConfig config) { return ImmutableList.of( testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks113.class) - .withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS_113) + .withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS) + .withExcludedGroups(DELTA_LAKE_EXCLUDE_113) .withExcludedTests(getExcludedTests()) .build()); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java index bc8d7edd1f24..3b90e0a1fd2c 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java @@ -25,11 +25,10 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_143_RUNTIME_VERSION; -import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_91_RUNTIME_VERSION; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.dropDeltaTableWithRetry; @@ -244,7 +243,7 @@ public void testTrinoPreservesReaderAndWriterVersions() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_104, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_113, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTrinoPreservesTableFeature() { @@ -297,7 +296,7 @@ public void testTrinoAlterTablePreservesGeneratedColumn() onTrino().executeQuery("ALTER TABLE delta.default." + tableName + " ADD COLUMN c INT"); assertThat((String) onDelta().executeQuery("SHOW CREATE TABLE default." + tableName).getOnlyValue()) - .contains((getDatabricksRuntimeVersion().orElseThrow().equals(DATABRICKS_91_RUNTIME_VERSION) ? "`b`" : "b") + " INT GENERATED ALWAYS AS ( a * 2 )"); + .contains("b INT GENERATED ALWAYS AS ( a * 2 )"); onDelta().executeQuery("INSERT INTO default." + tableName + " (a, c) VALUES (1, 3)"); assertThat(onTrino().executeQuery("SELECT * FROM delta.default." + tableName)) .containsOnly(row(1, 2, 3)); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java index 30971019b12a..679937044cde 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java @@ -24,11 +24,8 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_143_RUNTIME_VERSION; @@ -75,7 +72,7 @@ public void testNonLowercaseColumnNames() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testNonLowercaseFieldNames() { @@ -190,8 +187,7 @@ public void testGeneratedColumnWithNonLowerCaseColumnName() } } - // Exclude 10.4 because it throws MISSING_COLUMN when executing INSERT statement - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_104, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testIdentityColumnWithNonLowerCaseColumnName() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java index fca2c054dbcf..74b210918459 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java @@ -31,11 +31,10 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; @@ -99,7 +98,7 @@ public void testUpdateTableWithCdf(String columnMappingMode) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_104, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_113, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testUpdateTableWithChangeDataFeedWriterFeature() { @@ -532,7 +531,7 @@ public void testMergeDeleteIntoTableWithCdfEnabled(String columnMappingMode) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testMergeMixedDeleteAndUpdateIntoTableWithCdfEnabled() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java index 530a409e2402..8a50e0a77fa1 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java @@ -24,7 +24,6 @@ import io.trino.tempto.query.QueryResult; import io.trino.testng.services.Flaky; import io.trino.tests.product.deltalake.util.DatabricksVersion; -import org.testng.SkipException; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -38,7 +37,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; @@ -46,8 +44,7 @@ import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.TransactionLogAssertions.assertLastEntryIsCheckpointed; import static io.trino.tests.product.deltalake.TransactionLogAssertions.assertTransactionLogVersion; -import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_104_RUNTIME_VERSION; -import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_91_RUNTIME_VERSION; +import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_113_RUNTIME_VERSION; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.dropDeltaTableWithRetry; @@ -56,7 +53,6 @@ import static io.trino.tests.product.utils.QueryExecutors.onTrino; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestDeltaLakeCheckpointsCompatibility extends BaseTestDeltaLakeS3Storage @@ -278,7 +274,7 @@ private void trinoUsesCheckpointInterval(String deltaTableProperties) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDatabricksUsesCheckpointInterval() { @@ -370,7 +366,7 @@ private void testCheckpointMinMaxStatisticsForRowType(Consumer sqlExecut // Assert min/max queries can be computed from just metadata String explainSelectMax = getOnlyElement(onDelta().executeQuery("EXPLAIN SELECT max(root.entry_one) FROM default." + tableName).column(1)); - String column = databricksRuntimeVersion.orElseThrow().isAtLeast(DATABRICKS_104_RUNTIME_VERSION) ? "root.entry_one" : "root.entry_one AS `entry_one`"; + String column = databricksRuntimeVersion.orElseThrow().isAtLeast(DATABRICKS_113_RUNTIME_VERSION) ? "root.entry_one" : "root.entry_one AS `entry_one`"; assertThat(explainSelectMax).matches("== Physical Plan ==\\s*LocalTableScan \\[max\\(" + column + "\\).*]\\s*"); // check both engines can read both tables @@ -436,7 +432,7 @@ private void testCheckpointNullStatisticsForRowType(Consumer sqlExecutor // Assert counting non null entries can be computed from just metadata String explainCountNotNull = getOnlyElement(onDelta().executeQuery("EXPLAIN SELECT count(root.entry_two) FROM default." + tableName).column(1)); - String column = databricksRuntimeVersion.orElseThrow().isAtLeast(DATABRICKS_104_RUNTIME_VERSION) ? "root.entry_two" : "root.entry_two AS `entry_two`"; + String column = databricksRuntimeVersion.orElseThrow().isAtLeast(DATABRICKS_113_RUNTIME_VERSION) ? "root.entry_two" : "root.entry_two AS `entry_two`"; assertThat(explainCountNotNull).matches("== Physical Plan ==\\s*LocalTableScan \\[count\\(" + column + "\\).*]\\s*"); // check both engines can read both tables @@ -562,11 +558,6 @@ private void testWriteStatsAsJsonEnabled(Consumer sqlExecutor, String ta " delta.checkpoint.writeStatsAsStruct = true)", tableName, type, bucketName); - if (databricksRuntimeVersion.isPresent() && databricksRuntimeVersion.get().equals(DATABRICKS_91_RUNTIME_VERSION) && type.equals("struct")) { - assertThatThrownBy(() -> onDelta().executeQuery(createTableSql)).hasStackTraceContaining("ParseException"); - throw new SkipException("New runtime version covers the type"); - } - onDelta().executeQuery(createTableSql); try { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCloneTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCloneTableCompatibility.java index d773c66a6547..70296c15d6ec 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCloneTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCloneTableCompatibility.java @@ -28,7 +28,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; @@ -241,7 +240,7 @@ public void testReadFromSchemaChangedShallowCloneTable() testReadSchemaChangedCloneTable("SHALLOW", false); } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_104, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testReadFromSchemaChangedDeepCloneTable() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java index 9037f8c833a9..aa284125c55d 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java @@ -27,7 +27,6 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; @@ -53,7 +52,7 @@ public class TestDeltaLakeColumnMappingMode extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testColumnMappingModeNone() { @@ -226,7 +225,7 @@ private void testColumnMappingModeReaderAndWriterVersion(Consumer create onTrino().executeQuery("DROP TABLE delta.default." + tableName); } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTrinoColumnMappingMode(String mode) { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java index a6845c3492f5..230c614c721d 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java @@ -25,7 +25,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; @@ -43,7 +42,7 @@ public class TestDeltaLakeCreateTableAsSelectCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTrinoTypesWithDatabricks() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java index 09dad521a2d9..07b8f3abdd89 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java @@ -26,7 +26,6 @@ import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; -import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_104_RUNTIME_VERSION; import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_113_RUNTIME_VERSION; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH; @@ -69,7 +68,7 @@ public void testDatabricksCanReadInitialCreateTable() assertThat(onDelta().executeQuery("SHOW TABLES FROM default LIKE '" + tableName + "'")).contains(row("default", tableName, false)); assertThat(onDelta().executeQuery("SELECT count(*) FROM default." + tableName)).contains(row(0)); String showCreateTable; - if (databricksRuntimeVersion.isAtLeast(DATABRICKS_104_RUNTIME_VERSION)) { + if (databricksRuntimeVersion.isAtLeast(DATABRICKS_113_RUNTIME_VERSION)) { showCreateTable = format( "CREATE TABLE spark_catalog.default.%s (\n integer INT,\n string STRING,\n timetz TIMESTAMP)\nUSING delta\nLOCATION 's3://%s/%s'\n%s", tableName, @@ -112,7 +111,7 @@ public void testDatabricksCanReadInitialCreatePartitionedTable() assertThat(onDelta().executeQuery("SHOW TABLES LIKE '" + tableName + "'")).contains(row("default", tableName, false)); assertThat(onDelta().executeQuery("SELECT count(*) FROM " + tableName)).contains(row(0)); String showCreateTable; - if (databricksRuntimeVersion.isAtLeast(DATABRICKS_104_RUNTIME_VERSION)) { + if (databricksRuntimeVersion.isAtLeast(DATABRICKS_113_RUNTIME_VERSION)) { showCreateTable = format( "CREATE TABLE spark_catalog.default.%s (\n integer INT,\n string STRING,\n timetz TIMESTAMP)\nUSING delta\n" + "PARTITIONED BY (string)\nLOCATION 's3://%s/%s'\n%s", @@ -155,7 +154,7 @@ public void testDatabricksCanReadInitialCreateTableAs() assertThat(onDelta().executeQuery("SHOW TABLES FROM default LIKE '" + tableName + "'")).contains(row("default", tableName, false)); assertThat(onDelta().executeQuery("SELECT count(*) FROM default." + tableName)).contains(row(3)); String showCreateTable; - if (databricksRuntimeVersion.isAtLeast(DATABRICKS_104_RUNTIME_VERSION)) { + if (databricksRuntimeVersion.isAtLeast(DATABRICKS_113_RUNTIME_VERSION)) { showCreateTable = format( "CREATE TABLE spark_catalog.default.%s (\n integer INT,\n string STRING,\n timetz TIMESTAMP)\nUSING delta\nLOCATION 's3://%s/%s'\n%s", tableName, @@ -201,7 +200,7 @@ public void testDatabricksCanReadInitialCreatePartitionedTableAs() assertThat(onDelta().executeQuery("SHOW TABLES LIKE '" + tableName + "'")).contains(row("default", tableName, false)); assertThat(onDelta().executeQuery("SELECT count(*) FROM " + tableName)).contains(row(3)); String showCreateTable; - if (databricksRuntimeVersion.isAtLeast(DATABRICKS_104_RUNTIME_VERSION)) { + if (databricksRuntimeVersion.isAtLeast(DATABRICKS_113_RUNTIME_VERSION)) { showCreateTable = format( "CREATE TABLE spark_catalog.default.%s (\n integer INT,\n string STRING,\n timetz TIMESTAMP)\nUSING delta\n" + "PARTITIONED BY (string)\nLOCATION 's3://%s/%s'\n%s", @@ -401,13 +400,6 @@ public void testCreateTableWithAllPartitionColumns() private String getDatabricksDefaultTableProperties() { - if (databricksRuntimeVersion.equals(DATABRICKS_104_RUNTIME_VERSION)) { - return "TBLPROPERTIES (\n" + - " 'Type' = 'EXTERNAL',\n" + - " 'delta.enableDeletionVectors' = 'false',\n" + - " 'delta.minReaderVersion' = '1',\n" + - " 'delta.minWriterVersion' = '2')\n"; - } if (databricksRuntimeVersion.isAtLeast(DATABRICKS_113_RUNTIME_VERSION)) { return "TBLPROPERTIES (\n" + " 'delta.enableDeletionVectors' = 'false',\n" + diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDeleteCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDeleteCompatibility.java index ef24c834f3a8..639900df2f1a 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDeleteCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDeleteCompatibility.java @@ -31,7 +31,7 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; @@ -208,7 +208,7 @@ public void testTrinoDeletionVectors() } // Databricks 12.1 and OSS Delta 2.4.0 added support for deletion vectors - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_104, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingModeDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_113, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingModeDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDeletionVectors(String mode) { @@ -495,7 +495,7 @@ public void testDeletionVectorsAcrossAddFile(boolean partitioned) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_104, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_113, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDeletionVectorsTruncateTable() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeIdentityColumnCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeIdentityColumnCompatibility.java index a0cf50cd03e4..eb7d60a1d6af 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeIdentityColumnCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeIdentityColumnCompatibility.java @@ -21,7 +21,7 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH; @@ -37,7 +37,7 @@ public class TestDeltaLakeIdentityColumnCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS_113, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testIdentityColumn() { @@ -77,7 +77,7 @@ public void testIdentityColumn() } } - @Test(groups = {DELTA_LAKE_DATABRICKS_113, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testRenameIdentityColumn(String mode) { @@ -114,7 +114,7 @@ public void testRenameIdentityColumn(String mode) } } - @Test(groups = {DELTA_LAKE_DATABRICKS_113, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDropIdentityColumn(String mode) { @@ -152,7 +152,7 @@ public void testDropIdentityColumn(String mode) } } - @Test(groups = {DELTA_LAKE_DATABRICKS_113, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testVacuumProcedureWithIdentityColumn() { @@ -183,7 +183,7 @@ public void testVacuumProcedureWithIdentityColumn() } } - @Test(groups = {DELTA_LAKE_DATABRICKS_113, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testIdentityColumnCheckpointInterval() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java index b7290e0839b0..d080a1f6f848 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java @@ -31,13 +31,12 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; -import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_104_RUNTIME_VERSION; +import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_113_RUNTIME_VERSION; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.dropDeltaTableWithRetry; @@ -58,7 +57,7 @@ public void setup() databricksRuntimeVersion = getDatabricksRuntimeVersion(); } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testInsertCompatibility() { @@ -94,7 +93,7 @@ public void testInsertCompatibility() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testPartitionedInsertCompatibility() { @@ -414,7 +413,7 @@ public void testCompression(String compressionCodec) assertThat(onTrino().executeQuery("SELECT * FROM " + trinoTableName)) .containsOnly(expected); - if ("ZSTD".equals(compressionCodec) && databricksRuntimeVersion.orElseThrow().isOlderThan(DATABRICKS_104_RUNTIME_VERSION)) { + if ("ZSTD".equals(compressionCodec) && databricksRuntimeVersion.orElseThrow().isOlderThan(DATABRICKS_113_RUNTIME_VERSION)) { assertQueryFailure(() -> onDelta().executeQuery("SELECT * FROM default." + tableName)) .hasMessageContaining("java.lang.ClassNotFoundException: org.apache.hadoop.io.compress.ZStandardCodec"); } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSelectCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSelectCompatibility.java index 3a5a65f57132..e8e7ac5e03d0 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSelectCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSelectCompatibility.java @@ -24,7 +24,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; @@ -41,7 +40,7 @@ public class TestDeltaLakeSelectCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testPartitionedSelectSpecialCharacters() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSystemTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSystemTableCompatibility.java index 842737d9ed3c..6f7abf4ea8ab 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSystemTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSystemTableCompatibility.java @@ -24,7 +24,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_104; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; @@ -38,7 +37,7 @@ public class TestDeltaLakeSystemTableCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_104, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTablePropertiesCaseSensitivity() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java index 918507de550e..e7a7b48f69a5 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java @@ -24,7 +24,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; @@ -38,7 +37,7 @@ public class TestDeltaLakeUpdateCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testUpdatesFromDatabricks() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeWriteDatabricksCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeWriteDatabricksCompatibility.java index 6e08f9a20a9d..1787a1de9ff2 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeWriteDatabricksCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeWriteDatabricksCompatibility.java @@ -35,7 +35,7 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_122_RUNTIME_VERSION; @@ -323,7 +323,7 @@ public void testInsertingIntoDatabricksTableWithAddedNotNullConstraint() } } - @Test(groups = {DELTA_LAKE_DATABRICKS_113, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTrinoVacuumRemoveChangeDataFeedFiles() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/util/DatabricksVersion.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/util/DatabricksVersion.java index 6d3c2ba8d72b..2433a5ccd23d 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/util/DatabricksVersion.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/util/DatabricksVersion.java @@ -24,8 +24,6 @@ public record DatabricksVersion(int majorVersion, int minorVersion) public static final DatabricksVersion DATABRICKS_143_RUNTIME_VERSION = new DatabricksVersion(14, 3); public static final DatabricksVersion DATABRICKS_122_RUNTIME_VERSION = new DatabricksVersion(12, 2); public static final DatabricksVersion DATABRICKS_113_RUNTIME_VERSION = new DatabricksVersion(11, 3); - public static final DatabricksVersion DATABRICKS_104_RUNTIME_VERSION = new DatabricksVersion(10, 4); - public static final DatabricksVersion DATABRICKS_91_RUNTIME_VERSION = new DatabricksVersion(9, 1); private static final Pattern DATABRICKS_VERSION_PATTERN = Pattern.compile("(\\d+)\\.(\\d+)");