Skip to content

Test: Increase max writer version to 4 for delta lake #14589

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ebyhr wants to merge 3 commits into from
Closed

Test: Increase max writer version to 4 for delta lake #14589

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
90e40d1
Increase max writer version to 4 for delta lake
homar Oct 11, 2022
3112cb8
Verify if delta lake insert is compatible with CDF
homar Oct 12, 2022
e631917
Merge testGeneratedColumnsCompatibility and testWritesToTableWithGene…
homar Oct 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ public class DeltaLakeMetadata
// The required writer version used by tables created by Trino
private static final int WRITER_VERSION = 2;
// The highest writer version Trino supports writing to
private static final int MAX_WRITER_VERSION = 3;
private static final int MAX_WRITER_VERSION = 4;
// This constant should be used only for a new table
private static final ProtocolEntry DEFAULT_PROTOCOL = new ProtocolEntry(READER_VERSION, WRITER_VERSION);
// Matches the dummy column Databricks stores in the metastore
Expand Down
12 changes: 6 additions & 6 deletions .../src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,14 @@ public void testAddColumnUnsupportedWriterVersion()
onDelta().executeQuery(format("" +
"CREATE TABLE default.%s (col int) " +
"USING DELTA LOCATION 's3://%s/%s'" +
"TBLPROPERTIES ('delta.minWriterVersion'='4')",
"TBLPROPERTIES ('delta.minWriterVersion'='5')",
tableName,
bucketName,
tableDirectory));

try {
assertQueryFailure(() -> onTrino().executeQuery("ALTER TABLE delta.default." + tableName + " ADD COLUMN new_col int"))
.hasMessageMatching(".* Table .* requires Delta Lake writer version 4 which is not supported");
.hasMessageMatching(".* Table .* requires Delta Lake writer version 5 which is not supported");
}
finally {
onDelta().executeQuery("DROP TABLE default." + tableName);
Expand Down Expand Up @@ -180,14 +180,14 @@ public void testCommentOnTableUnsupportedWriterVersion()
onDelta().executeQuery(format("" +
"CREATE TABLE default.%s (col int) " +
"USING DELTA LOCATION 's3://%s/%s'" +
"TBLPROPERTIES ('delta.minWriterVersion'='4')",
"TBLPROPERTIES ('delta.minWriterVersion'='5')",
tableName,
bucketName,
tableDirectory));

try {
assertQueryFailure(() -> onTrino().executeQuery("COMMENT ON TABLE delta.default." + tableName + " IS 'test comment'"))
.hasMessageMatching(".* Table .* requires Delta Lake writer version 4 which is not supported");
.hasMessageMatching(".* Table .* requires Delta Lake writer version 5 which is not supported");
}
finally {
onTrino().executeQuery("DROP TABLE delta.default." + tableName);
Expand Down Expand Up @@ -224,14 +224,14 @@ public void testCommentOnColumnUnsupportedWriterVersion()
onDelta().executeQuery(format("" +
"CREATE TABLE default.%s (col int) " +
"USING DELTA LOCATION 's3://%s/%s'" +
"TBLPROPERTIES ('delta.minWriterVersion'='4')",
"TBLPROPERTIES ('delta.minWriterVersion'='5')",
tableName,
bucketName,
tableDirectory));

try {
assertQueryFailure(() -> onTrino().executeQuery("COMMENT ON COLUMN delta.default." + tableName + ".col IS 'test column comment'"))
.hasMessageMatching(".* Table .* requires Delta Lake writer version 4 which is not supported");
.hasMessageMatching(".* Table .* requires Delta Lake writer version 5 which is not supported");
}
finally {
onTrino().executeQuery("DROP TABLE delta.default." + tableName);
Expand Down
43 changes: 10 additions & 33 deletions ...ain/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksInsertCompatibility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -348,37 +348,6 @@ public void testCheckConstraintsCompatibility()
}
}

@Test(groups = {DELTA_LAKE_EXCLUDE_73, PROFILE_SPECIFIC_TESTS})
public void testGeneratedColumnsCompatibility()
{
// Generated columns require writer version 4, which is not supported by Trino
String tableName = "test_generated_columns_not_supported_" + randomTableSuffix();

onDelta().executeQuery("CREATE TABLE default." + tableName + "( " +
" id INT, " +
" a_number INT, " +
" a_number_times_two INT GENERATED ALWAYS AS (a_number * 2)) " +
"USING DELTA " +
"LOCATION 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'");

try {
onDelta().executeQuery("INSERT INTO default." + tableName + " (id, a_number) VALUES (1, 1), (2, 2), (3, 3)");

assertThat(onTrino().executeQuery("SELECT a_number, a_number_times_two FROM " + tableName))
.containsOnly(row(1, 2), row(2, 4), row(3, 6));

assertQueryFailure(() -> onTrino().executeQuery("INSERT INTO delta.default." + tableName + " VALUES (1, 2, 4)"))
.hasMessageMatching(".*Table default." + tableName + " requires Delta Lake writer version 4 which is not supported");
assertQueryFailure(() -> onTrino().executeQuery("DELETE FROM delta.default." + tableName + " WHERE a_number = 1"))
.hasMessageMatching(".*Table default." + tableName + " requires Delta Lake writer version 4 which is not supported");
assertQueryFailure(() -> onTrino().executeQuery("UPDATE delta.default." + tableName + " SET a_number_times_two = 5 WHERE a_number = 2"))
.hasMessageMatching(".*Table default." + tableName + " requires Delta Lake writer version 4 which is not supported");
}
finally {
onDelta().executeQuery("DROP TABLE default." + tableName);
}
}

/**
* Smoke test compression when writing to a Delta table. It's verified that writer doesn't fail and reads succeed,
* but it's not verified that compression actually takes place.
Expand Down Expand Up @@ -526,6 +495,11 @@ public void testWritesToTableWithGeneratedColumnFails()
"USING DELTA " +
"LOCATION 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'");

onDelta().executeQuery("INSERT INTO default." + tableName + " (a) VALUES (1), (2), (3)");

assertThat(onTrino().executeQuery("SELECT a, b FROM " + tableName))
.containsOnly(row(1, true), row(2, true), row(3, true));

// Disallowing all statements just in case though some statements may not unrelated to generated columns
assertQueryFailure(() -> onTrino().executeQuery("INSERT INTO delta.default." + tableName + " VALUES (1, false)"))
.hasMessageContaining("Writing to tables with generated columns is not supported");
Expand All @@ -552,15 +526,18 @@ public void testWritesToTableWithCDFFails()
"LOCATION 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'" +
"TBLPROPERTIES (delta.enableChangeDataFeed = true)");

assertQueryFailure(() -> onTrino().executeQuery("INSERT INTO delta.default." + tableName + " VALUES (1, 2)"))
.hasMessageMatching(".* Table .* requires Delta Lake writer version 4 which is not supported");
onTrino().executeQuery("INSERT INTO delta.default." + tableName + " VALUES (1, 2)");
assertQueryFailure(() -> onTrino().executeQuery("UPDATE delta.default." + tableName + " SET a = 3 WHERE b = 3"))
.hasMessageContaining("Writing to tables with Change Data Feed enabled is not supported");
assertQueryFailure(() -> onTrino().executeQuery("DELETE FROM delta.default." + tableName + " WHERE a = 3"))
.hasMessageContaining("Writing to tables with Change Data Feed enabled is not supported");
assertQueryFailure(() -> onTrino().executeQuery("MERGE INTO delta.default." + tableName + " t USING delta.default." + tableName + " s " +
"ON (t.a = s.a) WHEN MATCHED THEN UPDATE SET b = 42"))
.hasMessageContaining("Writing to tables with Change Data Feed enabled is not supported");
assertThat(onTrino().executeQuery("SELECT * FROM delta.default." + tableName))
.containsOnly(row(1, 2));
assertThat(onDelta().executeQuery("SELECT a, b, _change_type, _commit_version FROM table_changes('default." + tableName + "', 0)"))
.containsOnly(row(1, 2, "insert", 1L));
}
finally {
onDelta().executeQuery("DROP TABLE IF EXISTS default." + tableName);
Expand Down