Skip to content

Commit b4b2d9a

Browse files
chenjian2664ebyhr
chenjian2664
authored and
ebyhr
committed
Add support for Databricks 16.4 LTS environment to product tests
1 parent feb4cfe commit b4b2d9a

File tree

13 files changed

+111
-12
lines changed

13 files changed

+111
-12
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -880,6 +880,7 @@ jobs:
880880
- suite-delta-lake-databricks133
881881
- suite-delta-lake-databricks143
882882
- suite-delta-lake-databricks154
883+
- suite-delta-lake-databricks164
883884
- suite-ranger
884885
- suite-gcs
885886
- suite-clients
@@ -928,6 +929,9 @@ jobs:
928929
- suite: suite-delta-lake-databricks154
929930
ignore exclusion if: >-
930931
${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
932+
- suite: suite-delta-lake-databricks164
933+
ignore exclusion if: >-
934+
${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
931935
- suite: suite-snowflake
932936
ignore exclusion if: >-
933937
${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.SNOWFLAKE_PASSWORD != '' }}
@@ -977,6 +981,7 @@ jobs:
977981
DATABRICKS_133_JDBC_URL: ""
978982
DATABRICKS_143_JDBC_URL: ""
979983
DATABRICKS_154_JDBC_URL: ""
984+
DATABRICKS_164_JDBC_URL: ""
980985
DATABRICKS_LOGIN: ""
981986
DATABRICKS_TOKEN: ""
982987
GCP_CREDENTIALS_KEY: ""
@@ -1049,6 +1054,7 @@ jobs:
10491054
DATABRICKS_133_JDBC_URL: ${{ vars.DATABRICKS_133_JDBC_URL }}
10501055
DATABRICKS_143_JDBC_URL: ${{ vars.DATABRICKS_143_JDBC_URL }}
10511056
DATABRICKS_154_JDBC_URL: ${{ vars.DATABRICKS_154_JDBC_URL }}
1057+
DATABRICKS_164_JDBC_URL: ${{ vars.DATABRICKS_164_JDBC_URL }}
10521058
DATABRICKS_LOGIN: token
10531059
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
10541060
GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}

‎docs/src/main/sphinx/connector/delta-lake.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ data.
1313
To connect to Databricks Delta Lake, you need:
1414

1515
- Tables written by Databricks Runtime 7.3 LTS, 9.1 LTS, 10.4 LTS, 11.3 LTS,
16-
12.2 LTS, 13.3 LTS, 14.3 LTS and 15.4 LTS are supported.
16+
12.2 LTS, 13.3 LTS, 14.3 LTS, 15.4 LTS and 16.4 LTS are supported.
1717
- Deployments using AWS, HDFS, Azure Storage, and Google Cloud Storage (GCS) are
1818
fully supported.
1919
- Network access from the coordinator and workers to the Delta Lake storage.

‎testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,10 @@ public final class TestGroups
9797
public static final String DELTA_LAKE_DATABRICKS_122 = "delta-lake-databricks-122";
9898
public static final String DELTA_LAKE_DATABRICKS_133 = "delta-lake-databricks-133";
9999
public static final String DELTA_LAKE_DATABRICKS_143 = "delta-lake-databricks-143";
100+
public static final String DELTA_LAKE_DATABRICKS_154 = "delta-lake-databricks-154";
100101
public static final String DELTA_LAKE_EXCLUDE_113 = "delta-lake-exclude-113";
102+
// TODO: Remove it once we support generatedColumns, particularly for writes in Delta Lake
103+
public static final String DELTA_LAKE_EXCLUDE_164 = "delta-lake-exclude-164";
101104
public static final String DELTA_LAKE_ALLUXIO_CACHING = "delta-lake-alluxio-caching";
102105
public static final String HUDI = "hudi";
103106
public static final String PARQUET = "parquet";

‎testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks164.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.tests.product.launcher.env.environment;
15+
16+
import com.google.inject.Inject;
17+
import io.trino.tests.product.launcher.docker.DockerFiles;
18+
import io.trino.tests.product.launcher.env.common.Standard;
19+
import io.trino.tests.product.launcher.env.common.TestsEnvironment;
20+
21+
import static io.trino.testing.SystemEnvironmentUtils.requireEnv;
22+
23+
@TestsEnvironment
24+
public class EnvSinglenodeDeltaLakeDatabricks164
25+
extends AbstractSinglenodeDeltaLakeDatabricks
26+
{
27+
@Inject
28+
public EnvSinglenodeDeltaLakeDatabricks164(Standard standard, DockerFiles dockerFiles)
29+
{
30+
super(standard, dockerFiles);
31+
}
32+
33+
@Override
34+
String databricksTestJdbcUrl()
35+
{
36+
return requireEnv("DATABRICKS_164_JDBC_URL");
37+
}
38+
}

‎testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks154.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import java.util.List;
2323

2424
import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES;
25-
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS;
25+
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_154;
2626
import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment;
2727

2828
public class SuiteDeltaLakeDatabricks154
@@ -33,7 +33,7 @@ public List<SuiteTestRun> getTestRuns(EnvironmentConfig config)
3333
{
3434
return ImmutableList.of(
3535
testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks154.class)
36-
.withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS)
36+
.withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS_154)
3737
.withExcludedTests(getExcludedTests())
3838
.build());
3939
}

‎testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks164.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.tests.product.launcher.suite.suites;
15+
16+
import com.google.common.collect.ImmutableList;
17+
import io.trino.tests.product.launcher.env.EnvironmentConfig;
18+
import io.trino.tests.product.launcher.env.environment.EnvSinglenodeDeltaLakeDatabricks164;
19+
import io.trino.tests.product.launcher.suite.SuiteDeltaLakeDatabricks;
20+
import io.trino.tests.product.launcher.suite.SuiteTestRun;
21+
22+
import java.util.List;
23+
24+
import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES;
25+
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS;
26+
import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_164;
27+
import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment;
28+
29+
public class SuiteDeltaLakeDatabricks164
30+
extends SuiteDeltaLakeDatabricks
31+
{
32+
@Override
33+
public List<SuiteTestRun> getTestRuns(EnvironmentConfig config)
34+
{
35+
return ImmutableList.of(
36+
testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks164.class)
37+
.withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS)
38+
.withExcludedGroups(DELTA_LAKE_EXCLUDE_164)
39+
.withExcludedTests(getExcludedTests())
40+
.build());
41+
}
42+
}

‎testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS;
2727
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133;
2828
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143;
29+
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_154;
2930
import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS;
3031
import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS;
3132
import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_143_RUNTIME_VERSION;
@@ -72,7 +73,7 @@ public void testNonLowercaseColumnNames()
7273
}
7374
}
7475

75-
@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS})
76+
@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS})
7677
@Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH)
7778
public void testNonLowercaseFieldNames()
7879
{

‎testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122;
3535
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133;
3636
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143;
37+
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_154;
3738
import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_113;
3839
import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS;
3940
import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS;
@@ -531,7 +532,7 @@ public void testMergeDeleteIntoTableWithCdfEnabled(String columnMappingMode)
531532
}
532533
}
533534

534-
@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS})
535+
@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS})
535536
@Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH)
536537
public void testMergeMixedDeleteAndUpdateIntoTableWithCdfEnabled()
537538
{

‎testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122;
4141
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133;
4242
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143;
43+
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_154;
4344
import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS;
4445
import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS;
4546
import static io.trino.tests.product.deltalake.TransactionLogAssertions.assertLastEntryIsCheckpointed;
@@ -274,7 +275,7 @@ private void trinoUsesCheckpointInterval(String deltaTableProperties)
274275
}
275276
}
276277

277-
@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, PROFILE_SPECIFIC_TESTS})
278+
@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, PROFILE_SPECIFIC_TESTS})
278279
@Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH)
279280
public void testDatabricksUsesCheckpointInterval()
280281
{

‎testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122;
3131
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133;
3232
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143;
33+
import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_154;
34+
import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_164;
3335
import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS;
3436
import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS;
3537
import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE;
@@ -52,7 +54,7 @@
5254
public class TestDeltaLakeColumnMappingMode
5355
extends BaseTestDeltaLakeS3Storage
5456
{
55-
@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS})
57+
@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS})
5658
@Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH)
5759
public void testColumnMappingModeNone()
5860
{
@@ -239,7 +241,9 @@ public void testTrinoColumnMappingMode(String mode)
239241
")"));
240242
}
241243

242-
@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider")
244+
// When setting the table property `delta.columnMapping.mode` on Databricks 16.4, it will enable the `delta.feature.generatedColumns`
245+
// feature, which is not supported by Trino.
246+
@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_164, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider")
243247
@Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH)
244248
public void testDeltaColumnMappingMode(String mode)
245249
{

0 commit comments

Comments
 (0)