Skip to content

Add test for CHECK constraint and id column mapping in Delta Lake #13403

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ebyhr merged 3 commits into from
Jul 29, 2022
Merged

Add test for CHECK constraint and id column mapping in Delta Lake #13403

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f86f024
Remove left over comment from CheckpointEntryIterator
ebyhr Jul 29, 2022
1ad8ea0
Add test for CHECK constraint in Delta Lake
ebyhr Jul 27, 2022
47e1316
Add test for delta.columnMapping.mode=id in Delta Lake
ebyhr Jul 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...ain/java/io/trino/plugin/deltalake/transactionlog/checkpoint/CheckpointEntryIterator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -405,7 +405,7 @@ else if (!addEntryBlock.isNull(5)) {
return DeltaLakeTransactionLogEntry.addFileEntry(result);
}

private DeltaLakeParquetFileStatistics parseStatisticsFromParquet(Block statsRowBlock) // TODO: Fix
private DeltaLakeParquetFileStatistics parseStatisticsFromParquet(Block statsRowBlock)
{
if (metadataEntry == null) {
throw new TrinoException(DELTA_LAKE_BAD_DATA, "Checkpoint file found without metadata entry");
Expand Down
23 changes: 21 additions & 2 deletions ...-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@
public class TestDeltaLakeColumnMappingMode
extends BaseTestDeltaLakeS3Storage
{
// TODO: Add test with 'delta.columnMapping.mode'='id' table property. This requires Databricks runtime version 10.2 or higher version.

@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_73, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS})
public void testColumnMappingModeNone()
{
Expand Down Expand Up @@ -64,6 +62,27 @@ public void testColumnMappingModeNone()
}
}

@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_73, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS})
public void testColumnMappingModeId()
{
String tableName = "test_dl_column_mapping_mode_id" + randomTableSuffix();

onDelta().executeQuery("" +
"CREATE TABLE default." + tableName +
" (a_number INT)" +
" USING delta " +
" LOCATION 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'" +
" TBLPROPERTIES ('delta.columnMapping.mode'='id')");

try {
assertQueryFailure(() -> onTrino().executeQuery("SELECT * FROM delta.default." + tableName))
.hasMessageContaining("Only 'name' or 'none' is supported for the 'delta.columnMapping.mode' table property");
}
finally {
onDelta().executeQuery("DROP TABLE default." + tableName);
}
}

@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_73, DELTA_LAKE_EXCLUDE_91, PROFILE_SPECIFIC_TESTS})
public void testColumnMappingModeName()
{
Expand Down
33 changes: 33 additions & 0 deletions ...ain/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksInsertCompatibility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,39 @@ public void testDeleteCompatibility()
}
}

@Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, DELTA_LAKE_EXCLUDE_73, PROFILE_SPECIFIC_TESTS})
public void testCheckConstraintsCompatibility()
Comment thread
findepi marked this conversation as resolved.
Outdated
{
// CHECK constraint is not supported by Trino
String tableName = "test_check_constraint_not_supported_" + randomTableSuffix();

onDelta().executeQuery("CREATE TABLE default." + tableName +
"(id INT, a_number INT) " +
"USING DELTA " +
"LOCATION 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'");
onDelta().executeQuery("ALTER TABLE default." + tableName + " ADD CONSTRAINT id_constraint CHECK (id < 100)");

try {
onDelta().executeQuery("INSERT INTO default." + tableName + " (id, a_number) VALUES (1, 1)");

assertThat(onTrino().executeQuery("SELECT id, a_number FROM " + tableName))
.containsOnly(row(1, 1));

assertQueryFailure(() -> onTrino().executeQuery("INSERT INTO delta.default." + tableName + " VALUES (2, 2)"))
.hasMessageMatching(".*Table default." + tableName + " requires Delta Lake writer version 3 which is not supported");
assertQueryFailure(() -> onTrino().executeQuery("DELETE FROM delta.default." + tableName + " WHERE a_number = 1"))
.hasMessageMatching(".*Table default." + tableName + " requires Delta Lake writer version 3 which is not supported");
assertQueryFailure(() -> onTrino().executeQuery("UPDATE delta.default." + tableName + " SET a_number = 10 WHERE id = 1"))
.hasMessageMatching(".*Table default." + tableName + " requires Delta Lake writer version 3 which is not supported");

assertThat(onTrino().executeQuery("SELECT id, a_number FROM " + tableName))
.containsOnly(row(1, 1));
}
finally {
onDelta().executeQuery("DROP TABLE default." + tableName);
}
}

@Test(groups = {DELTA_LAKE_EXCLUDE_73, PROFILE_SPECIFIC_TESTS})
public void testGeneratedColumnsCompatibility()
{
Expand Down