Skip to content

Update Iceberg from 1.3.1 to 1.4.1 #21230

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ChunxuTang merged 1 commit into from
Nov 1, 2023
Merged

Update Iceberg from 1.3.1 to 1.4.1 #21230

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions presto-iceberg/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@

<properties>
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
<dep.iceberg.version>1.3.1</dep.iceberg.version>
<dep.nessie.version>0.59.0</dep.nessie.version>
<dep.iceberg.version>1.4.1</dep.iceberg.version>
<dep.nessie.version>0.71.0</dep.nessie.version>
</properties>

<dependencies>
Expand Down
18 changes: 9 additions & 9 deletions ...to-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedSmokeTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ public void testShowCreateTable()
")\n" +
"WITH (\n" +
" format = 'PARQUET',\n" +
" format_version = '1',\n" +
" format_version = '2',\n" +
" location = '%s'\n" +
")", getLocation("tpch", "orders")));
}
Expand Down Expand Up @@ -369,7 +369,7 @@ private void testCreatePartitionedTableAs(Session session, FileFormat fileFormat
")\n" +
"WITH (\n" +
" format = '" + fileFormat + "',\n" +
" format_version = '1',\n" +
" format_version = '2',\n" +
" location = '%s',\n" +
" partitioning = ARRAY['order_status','ship_priority','bucket(order_key, 9)']\n" +
")",
Expand Down Expand Up @@ -415,7 +415,7 @@ public void testTableComments()
"COMMENT '%s'\n" +
"WITH (\n" +
" format = 'ORC',\n" +
" format_version = '1'\n" +
" format_version = '2'\n" +
")";

assertUpdate(format(createTable, "test table comment"));
Expand All @@ -427,7 +427,7 @@ public void testTableComments()
"COMMENT '%s'\n" +
"WITH (\n" +
" format = 'ORC',\n" +
" format_version = '1',\n" +
" format_version = '2',\n" +
" location = '%s'\n" +
")";
String createTableSql = format(createTableTemplate, "test table comment", getLocation("tpch", "test_table_comments"));
Expand Down Expand Up @@ -514,7 +514,7 @@ private void testCreateTableLike()
assertUpdate(session, "CREATE TABLE test_create_table_like_original (col1 INTEGER, aDate DATE) WITH(format = 'PARQUET', partitioning = ARRAY['aDate'])");
assertEquals(getTablePropertiesString("test_create_table_like_original"), format("WITH (\n" +
" format = 'PARQUET',\n" +
" format_version = '1',\n" +
" format_version = '2',\n" +
" location = '%s',\n" +
" partitioning = ARRAY['adate']\n" +
")", getLocation("tpch", "test_create_table_like_original")));
Expand All @@ -527,23 +527,23 @@ private void testCreateTableLike()
assertUpdate(session, "CREATE TABLE test_create_table_like_copy1 (LIKE test_create_table_like_original)");
assertEquals(getTablePropertiesString("test_create_table_like_copy1"), format("WITH (\n" +
" format = 'PARQUET',\n" +
" format_version = '1',\n" +
" format_version = '2',\n" +
" location = '%s'\n" +
")", getLocation("tpch", "test_create_table_like_copy1")));
dropTable(session, "test_create_table_like_copy1");

assertUpdate(session, "CREATE TABLE test_create_table_like_copy2 (LIKE test_create_table_like_original EXCLUDING PROPERTIES)");
assertEquals(getTablePropertiesString("test_create_table_like_copy2"), format("WITH (\n" +
" format = 'PARQUET',\n" +
" format_version = '1',\n" +
" format_version = '2',\n" +
" location = '%s'\n" +
")", getLocation("tpch", "test_create_table_like_copy2")));
dropTable(session, "test_create_table_like_copy2");

assertUpdate(session, "CREATE TABLE test_create_table_like_copy3 (LIKE test_create_table_like_original INCLUDING PROPERTIES)");
assertEquals(getTablePropertiesString("test_create_table_like_copy3"), format("WITH (\n" +
" format = 'PARQUET',\n" +
" format_version = '1',\n" +
" format_version = '2',\n" +
" location = '%s',\n" +
" partitioning = ARRAY['adate']\n" +
")", catalogType.equals(CatalogType.HIVE) ?
Expand All @@ -554,7 +554,7 @@ private void testCreateTableLike()
assertUpdate(session, "CREATE TABLE test_create_table_like_copy4 (LIKE test_create_table_like_original INCLUDING PROPERTIES) WITH (format = 'ORC')");
assertEquals(getTablePropertiesString("test_create_table_like_copy4"), format("WITH (\n" +
" format = 'ORC',\n" +
" format_version = '1',\n" +
" format_version = '2',\n" +
" location = '%s',\n" +
" partitioning = ARRAY['adate']\n" +
")", catalogType.equals(CatalogType.HIVE) ?
Expand Down
14 changes: 12 additions & 2 deletions presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergSystemTables.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,14 @@

import java.nio.file.Path;
import java.time.LocalDate;
import java.util.List;
import java.util.Map;
import java.util.function.Function;

import static com.facebook.presto.iceberg.IcebergQueryRunner.ICEBERG_CATALOG;
import static com.facebook.presto.testing.TestingSession.testSessionBuilder;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static org.assertj.core.api.Assertions.assertThat;
import static org.testng.Assert.assertEquals;

public class TestIcebergSystemTables
Expand Down Expand Up @@ -189,8 +191,16 @@ public void testPropertiesTable()
{
assertQuery("SHOW COLUMNS FROM test_schema.\"test_table$properties\"",
"VALUES ('key', 'varchar', '', '')," + "('value', 'varchar', '', '')");
assertQuery("SELECT COUNT(*) FROM test_schema.\"test_table$properties\"", "VALUES 1");
assertQuery("SELECT * FROM test_schema.\"test_table$properties\"", "VALUES ('write.format.default', 'PARQUET')");
assertQuery("SELECT COUNT(*) FROM test_schema.\"test_table$properties\"", "VALUES 2");
List<MaterializedRow> materializedRows = computeActual(getSession(),
"SELECT * FROM test_schema.\"test_table$properties\"").getMaterializedRows();

assertThat(materializedRows).hasSize(2);
assertThat(materializedRows)
.anySatisfy(row -> assertThat(row)
.isEqualTo(new MaterializedRow(MaterializedResult.DEFAULT_PRECISION, "write.format.default", "PARQUET")))
.anySatisfy(row -> assertThat(row)
.isEqualTo(new MaterializedRow(MaterializedResult.DEFAULT_PRECISION, "write.parquet.compression-codec", "zstd")));
}

@Test
Expand Down
8 changes: 4 additions & 4 deletions presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestPartitionFields.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,10 @@ public void testParse()
assertInvalid("bucket()", "Invalid partition field declaration: bucket()");
assertInvalid("abc", "Cannot find source column: abc");
assertInvalid("notes", "Cannot partition by non-primitive source field: list<string>");
assertInvalid("bucket(price, 42)", "Cannot bucket by type: double");
assertInvalid("bucket(notes, 88)", "Cannot bucket by type: list<string>");
assertInvalid("truncate(ts, 13)", "Cannot truncate type: timestamp");
assertInvalid("year(order_key)", "Cannot partition type long by year");
assertInvalid("bucket(price, 42)", "Invalid source type double for transform: bucket[42]");
assertInvalid("bucket(notes, 88)", "Cannot partition by non-primitive source field: list<string>");
assertInvalid("truncate(ts, 13)", "Invalid source type timestamp for transform: truncate[13]");
assertInvalid("year(order_key)", "Invalid source type long for transform: year");
}

private static void assertParse(String value, PartitionSpec expected)
Expand Down
10 changes: 7 additions & 3 deletions ...eberg/src/test/java/com/facebook/presto/iceberg/nessie/TestIcebergSystemTablesNessie.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,16 +88,20 @@ public void testPropertiesTable()
{
assertQuery("SHOW COLUMNS FROM test_schema.\"test_table$properties\"",
"VALUES ('key', 'varchar', '', '')," + "('value', 'varchar', '', '')");
assertQuery("SELECT COUNT(*) FROM test_schema.\"test_table$properties\"", "VALUES 3");
assertQuery("SELECT COUNT(*) FROM test_schema.\"test_table$properties\"", "VALUES 5");
List<MaterializedRow> materializedRows = computeActual(getSession(),
"SELECT * FROM test_schema.\"test_table$properties\"").getMaterializedRows();

// nessie writes a "nessie.commit.id" + "gc.enabled=false" to the table properties
assertThat(materializedRows).hasSize(3);
assertThat(materializedRows).hasSize(5);
assertThat(materializedRows)
.anySatisfy(row -> assertThat(row)
.isEqualTo(new MaterializedRow(MaterializedResult.DEFAULT_PRECISION, "write.format.default", "PARQUET")))
.anySatisfy(row -> assertThat(row.getField(0)).isEqualTo("nessie.commit.id"))
.anySatisfy(row -> assertThat(row).isEqualTo(new MaterializedRow(MaterializedResult.DEFAULT_PRECISION, "gc.enabled", "false")));
.anySatisfy(row -> assertThat(row).isEqualTo(new MaterializedRow(MaterializedResult.DEFAULT_PRECISION, "gc.enabled", "false")))
.anySatisfy(row -> assertThat(row)
.isEqualTo(new MaterializedRow(MaterializedResult.DEFAULT_PRECISION, "write.parquet.compression-codec", "zstd")))
.anySatisfy(row -> assertThat(row)
.isEqualTo(new MaterializedRow(MaterializedResult.DEFAULT_PRECISION, "write.metadata.delete-after-commit.enabled", "false")));
}
}
2 changes: 1 addition & 1 deletion ...-testing-docker/src/main/java/com/facebook/presto/testing/containers/NessieContainer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public class NessieContainer
{
private static final Logger log = Logger.get(NessieContainer.class);

public static final String DEFAULT_IMAGE = "projectnessie/nessie:0.59.0";
public static final String DEFAULT_IMAGE = "projectnessie/nessie:0.71.0";
public static final String DEFAULT_HOST_NAME = "nessie";
public static final String VERSION_STORE_TYPE = "INMEMORY";

Expand Down