Skip to content

Simplify assertion of SHOW CREATE TABLE result in Delta Lake tests #18904

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ebyhr merged 1 commit into from
Sep 4, 2023
Merged

Simplify assertion of SHOW CREATE TABLE result in Delta Lake tests #18904

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 4 additions & 62 deletions ...ava/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCheckpointsCompatibility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
import static io.trino.tests.product.deltalake.TransactionLogAssertions.assertLastEntryIsCheckpointed;
import static io.trino.tests.product.deltalake.TransactionLogAssertions.assertTransactionLogVersion;
import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_104_RUNTIME_VERSION;
import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_113_RUNTIME_VERSION;
import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_91_RUNTIME_VERSION;
import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE;
import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH;
Expand Down Expand Up @@ -148,20 +147,8 @@ public void testTrinoUsesCheckpointInterval()

try {
// validate that we can see the checkpoint interval
String showCreateTable = format(
"CREATE TABLE delta.default.%s (\n" +
" a_number integer,\n" +
" a_string varchar\n" +
")\n" +
"WITH (\n" +
" checkpoint_interval = 5,\n" +
" location = 's3://%s/%s',\n" +
" partitioned_by = ARRAY['a_number']\n" +
")",
tableName,
bucketName,
tableDirectory);
assertThat(onTrino().executeQuery("SHOW CREATE TABLE delta.default." + tableName)).containsExactlyInOrder(row(showCreateTable));
assertThat((String) onTrino().executeQuery("SHOW CREATE TABLE delta.default." + tableName).getOnlyValue())
.contains("checkpoint_interval = 5");

// sanity check
fillWithInserts("delta.default." + tableName, "(1, 'trino')", 4);
Expand Down Expand Up @@ -206,35 +193,8 @@ public void testDatabricksUsesCheckpointInterval()

try {
// validate that Databricks can see the checkpoint interval
String showCreateTable;
if (databricksRuntimeVersion.isAtLeast(DATABRICKS_104_RUNTIME_VERSION)) {
showCreateTable = format(
"CREATE TABLE spark_catalog.default.%s (\n" +
" a_number BIGINT,\n" +
" a_string STRING)\n" +
"USING delta\n" +
"PARTITIONED BY (a_number)\n" +
"LOCATION 's3://%s/%s'\n%s",
tableName,
bucketName,
tableDirectory,
getDatabricksTablePropertiesWithCheckpointInterval());
}
else {
showCreateTable = format(
"CREATE TABLE `default`.`%s` (\n" +
" `a_number` BIGINT,\n" +
" `a_string` STRING)\n" +
"USING DELTA\n" +
"PARTITIONED BY (a_number)\n" +
"LOCATION 's3://%s/%s'\n" +
"TBLPROPERTIES (\n" +
" 'delta.checkpointInterval' = '3')\n",
tableName,
bucketName,
tableDirectory);
}
assertThat(onDelta().executeQuery("SHOW CREATE TABLE default." + tableName)).containsExactlyInOrder(row(showCreateTable));
assertThat((String) onDelta().executeQuery("SHOW CREATE TABLE default." + tableName).getOnlyValue())
.contains("'delta.checkpointInterval' = '3'");

// sanity check
onDelta().executeQuery("INSERT INTO default." + tableName + " VALUES (1, 'databricks')");
Expand All @@ -258,24 +218,6 @@ public void testDatabricksUsesCheckpointInterval()
}
}

private String getDatabricksTablePropertiesWithCheckpointInterval()
{
if (databricksRuntimeVersion.isAtLeast(DATABRICKS_113_RUNTIME_VERSION)) {
return "TBLPROPERTIES (\n" +
" 'delta.checkpointInterval' = '3',\n" +
" 'delta.minReaderVersion' = '1',\n" +
" 'delta.minWriterVersion' = '2')\n";
}
if (databricksRuntimeVersion.equals(DATABRICKS_104_RUNTIME_VERSION)) {
return "TBLPROPERTIES (\n" +
" 'Type' = 'EXTERNAL',\n" +
" 'delta.checkpointInterval' = '3',\n" +
" 'delta.minReaderVersion' = '1',\n" +
" 'delta.minWriterVersion' = '2')\n";
}
throw new IllegalArgumentException("Unsupported databricks runtime version: " + databricksRuntimeVersion);
}

@Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS})
@Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH)
public void testTrinoCheckpointMinMaxStatisticsForRowType()
Expand Down