Skip to content

Add Presto iceberg connector #15836

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
beinan merged 14 commits into from
May 20, 2021
Merged

Add Presto iceberg connector #15836

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
064c724
upgrade parquet to 1.11.0
Mar 5, 2021
d09a67d
Add iceberg connector
May 7, 2021
47d002c
Set up query tests
Mar 15, 2021
573c62e
Include iceberg connector in the root POM
Mar 15, 2021
7aa5740
Fix parquet version conflict
Mar 17, 2021
d480372
Refactor HdfsFileIo names
Apr 14, 2021
fcc9058
Upgrade Presto to 0.252
Apr 21, 2021
f78301b
Fix review comments
May 7, 2021
839c81a
Adjust the usage of Hive Metastore with MetastoreContext
May 7, 2021
209fbe6
Add MetastoreContext in the iceberg HiveTableOperations
May 7, 2021
6d6f53e
Remove unnecessary comments
May 14, 2021
a99ca25
Include presto-iceberg in example config
May 14, 2021
f1c2088
Adjust the test change of AbstractTestDistributedQueries
May 14, 2021
a55462e
Upgrade to 0.254
May 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@
<dep.testng.version>6.10</dep.testng.version>
<dep.assertj-core.version>3.8.0</dep.assertj-core.version>
<dep.logback.version>1.2.3</dep.logback.version>
<dep.parquet.version>1.10.0</dep.parquet.version>
<dep.parquet.version>1.11.0</dep.parquet.version>
<dep.nexus-staging-plugin.version>1.6.8</dep.nexus-staging-plugin.version>
<dep.asm.version>9.0</dep.asm.version>
<dep.gcs.version>1.9.17</dep.gcs.version>
Expand Down Expand Up @@ -106,6 +106,7 @@
<module>presto-hive-common</module>
<module>presto-hive-hadoop2</module>
<module>presto-hive-metastore</module>
<module>presto-iceberg</module>
<module>presto-i18n-functions</module>
<module>presto-teradata-functions</module>
<module>presto-example-http</module>
Expand Down Expand Up @@ -283,6 +284,18 @@
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-iceberg</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
<version>1.3.2</version>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-example-http</artifactId>
Expand Down
4 changes: 4 additions & 0 deletions presto-geospatial/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,10 @@
<ignoredResourcePattern>parquet.thrift</ignoredResourcePattern>
<ignoredResourcePattern>about.html</ignoredResourcePattern>
</ignoredResourcePatterns>
<ignoredClassPatterns>
<ignoredClassPattern>shaded.parquet.it.unimi.dsi.fastutil.*</ignoredClassPattern>
<ignoredClassPattern>module-info</ignoredClassPattern>
</ignoredClassPatterns>
</configuration>
</plugin>
</plugins>
Expand Down
4 changes: 4 additions & 0 deletions presto-hive-hadoop2/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,10 @@
<ignoredResourcePattern>parquet.thrift</ignoredResourcePattern>
<ignoredResourcePattern>about.html</ignoredResourcePattern>
</ignoredResourcePatterns>
<ignoredClassPatterns>
<ignoredClassPattern>shaded.parquet.it.unimi.dsi.fastutil.*</ignoredClassPattern>
<ignoredClassPattern>module-info</ignoredClassPattern>
</ignoredClassPatterns>
</configuration>
</plugin>
</plugins>
Expand Down
14 changes: 12 additions & 2 deletions ...ve-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ public class FileHiveMetastore
private static final String ADMIN_ROLE_NAME = "admin";
private static final String PRESTO_SCHEMA_FILE_NAME = ".prestoSchema";
private static final String PRESTO_PERMISSIONS_DIRECTORY_NAME = ".prestoPermissions";
private static final String ICEBERG_TABLE_TYPE_NAME = "table_type";
private static final String ICEBERG_TABLE_TYPE_VALUE = "iceberg";
// todo there should be a way to manage the admins list
private static final Set<String> ADMIN_USERS = ImmutableSet.of("admin", "hive", "hdfs");

Expand Down Expand Up @@ -249,7 +251,7 @@ else if (table.getTableType().equals(EXTERNAL_TABLE)) {
if (!externalFileSystem.isDirectory(externalLocation)) {
throw new PrestoException(HIVE_METASTORE_ERROR, "External table location does not exist");
}
if (isChildDirectory(catalogDirectory, externalLocation)) {
if (isChildDirectory(catalogDirectory, externalLocation) && !isIcebergTable(table.getParameters())) {
throw new PrestoException(HIVE_METASTORE_ERROR, "External table location can not be inside the system metadata directory");
}
}
Expand Down Expand Up @@ -459,8 +461,11 @@ public synchronized void renameTable(MetastoreContext metastoreContext, String d
requireNonNull(newDatabaseName, "newDatabaseName is null");
requireNonNull(newTableName, "newTableName is null");

getRequiredTable(metastoreContext, databaseName, tableName);
Table table = getRequiredTable(metastoreContext, databaseName, tableName);
getRequiredDatabase(metastoreContext, newDatabaseName);
if (isIcebergTable(table.getParameters())) {
throw new PrestoException(NOT_SUPPORTED, "Rename not supported for Iceberg tables");
}

// verify new table does not exist
verifyTableNotExists(metastoreContext, newDatabaseName, newTableName);
Expand All @@ -475,6 +480,11 @@ public synchronized void renameTable(MetastoreContext metastoreContext, String d
}
}

private static boolean isIcebergTable(Map<String, String> parameters)
{
return ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(parameters.get(ICEBERG_TABLE_TYPE_NAME));
}

@Override
public synchronized void addColumn(MetastoreContext metastoreContext, String databaseName, String tableName, String columnName, HiveType columnType, String columnComment)
{
Expand Down
4 changes: 4 additions & 0 deletions presto-hive/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,10 @@
<ignoredResourcePattern>parquet.thrift</ignoredResourcePattern>
<ignoredResourcePattern>about.html</ignoredResourcePattern>
</ignoredResourcePatterns>
<ignoredClassPatterns>
<ignoredClassPattern>shaded.parquet.it.unimi.dsi.fastutil.*</ignoredClassPattern>
<ignoredClassPattern>module-info</ignoredClassPattern>
</ignoredClassPatterns>
</configuration>
</plugin>

Expand Down
Loading