Add read support for S3 Tables in Iceberg

.github/workflows/ci.yml

@@ -695,6 +695,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
           S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
+          S3_TABLES_BUCKET: ${{ vars.TRINO_S3_TABLES_BUCKET }}
           GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}
           GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
           ABFS_CONTAINER: ${{ vars.AZURE_ABFS_HIERARCHICAL_CONTAINER }}

plugin/trino-iceberg/README.md

@@ -0,0 +1,30 @@
+# Iceberg Connector Developer Notes
+
+Steps to create TPCH tables on S3 Tables:
+1. Set `AWS_REGION`, `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables.
+2. Replace placeholders in the following command and run it:
+```sh
+./spark-sql \
+--packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1,software.amazon.awssdk:bundle:2.20.10,software.amazon.s3tables:s3-tables-catalog-for-iceberg-runtime:0.1.3,org.apache.kyuubi:kyuubi-spark-connector-tpch_2.12:1.8.0 \
+--conf spark.sql.catalog.s3tablesbucket=org.apache.iceberg.spark.SparkCatalog \
+--conf spark.sql.catalog.s3tablesbucket.catalog-impl=software.amazon.s3tables.iceberg.S3TablesCatalog \
+--conf spark.sql.catalog.s3tablesbucket.warehouse=arn:aws:s3tables:{region}:{account-id}:bucket/{bucket-name} \
+--conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
+--conf spark.sql.catalog.tpch=org.apache.kyuubi.spark.connector.tpch.TPCHCatalog
+```
+
+3. Run the following command to create TPCH tables:
+```sql
+CREATE TABLE s3tablesbucket.tpch.nation AS SELECT 
+  n_nationkey AS nationkey,
+  n_name AS name,
+  n_regionkey AS regionkey,
+  n_comment AS comment
+FROM tpch.tiny.nation;
+
+CREATE TABLE s3tablesbucket.tpch.region AS SELECT 
+  r_regionkey AS regionkey, 
+  r_name AS name, 
+  r_comment AS comment 
+FROM tpch.tiny.region;
+```

plugin/trino-iceberg/pom.xml

@@ -130,6 +130,11 @@
             <artifactId>trino-filesystem-manager</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-filesystem-s3</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>io.trino</groupId>
             <artifactId>trino-hive</artifactId>
@@ -218,6 +223,11 @@
             <artifactId>iceberg-api</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.iceberg</groupId>
+            <artifactId>iceberg-aws</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.iceberg</groupId>
             <artifactId>iceberg-core</artifactId>
@@ -359,12 +369,6 @@
             <scope>runtime</scope>
         </dependency>
 
-        <dependency>
-            <groupId>io.trino</groupId>
-            <artifactId>trino-filesystem-s3</artifactId>
-            <scope>runtime</scope>
-        </dependency>
-
         <dependency>
             <groupId>jakarta.servlet</groupId>
             <artifactId>jakarta.servlet-api</artifactId>
@@ -744,6 +748,7 @@
                                 <exclude>**/TestIcebergGlueTableOperationsInsertFailure.java</exclude>
                                 <exclude>**/TestIcebergGlueCatalogSkipArchive.java</exclude>
                                 <exclude>**/TestIcebergS3AndGlueMetastoreTest.java</exclude>
+                                <exclude>**/TestIcebergS3TablesConnectorSmokeTest.java</exclude>
                                 <exclude>**/TestIcebergGcsConnectorSmokeTest.java</exclude>
                                 <exclude>**/TestIcebergAbfsConnectorSmokeTest.java</exclude>
                                 <exclude>**/Test*FailureRecoveryTest.java</exclude>
@@ -809,6 +814,7 @@
                                 <include>**/TestIcebergGlueTableOperationsInsertFailure.java</include>
                                 <include>**/TestIcebergGlueCatalogSkipArchive.java</include>
                                 <include>**/TestIcebergS3AndGlueMetastoreTest.java</include>
+                                <include>**/TestIcebergS3TablesConnectorSmokeTest.java</include>
                                 <include>**/TestIcebergGcsConnectorSmokeTest.java</include>
                                 <include>**/TestIcebergAbfsConnectorSmokeTest.java</include>
                                 <include>**/TestIcebergSnowflakeCatalogConnectorSmokeTest.java</include>

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/AwsProperties.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.iceberg.catalog.rest;
+
+import java.util.Map;
+
+@FunctionalInterface
+public interface AwsProperties
+{
+    Map<String, String> get();
+}

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogConfig.java

@@ -47,6 +47,8 @@ public enum SessionType
     private Security security = Security.NONE;
     private SessionType sessionType = SessionType.NONE;
     private boolean vendedCredentialsEnabled;
+    private boolean viewEndpointsEnabled = true;
+    private boolean sigV4Enabled;
     private boolean caseInsensitiveNameMatching;
     private Duration caseInsensitiveNameMatchingCacheTtl = new Duration(1, MINUTES);
 
@@ -146,6 +148,32 @@ public IcebergRestCatalogConfig setVendedCredentialsEnabled(boolean vendedCreden
         return this;
     }
 
+    public boolean isViewEndpointsEnabled()
+    {
+        return viewEndpointsEnabled;
+    }
+
+    @Config("iceberg.rest-catalog.view-endpoints-enabled")
+    @ConfigDescription("Enable view endpoints")
+    public IcebergRestCatalogConfig setViewEndpointsEnabled(boolean viewEndpointsEnabled)
+    {
+        this.viewEndpointsEnabled = viewEndpointsEnabled;
+        return this;
+    }
+
+    public boolean isSigV4Enabled()
+    {
+        return sigV4Enabled;
+    }
+
+    @Config("iceberg.rest-catalog.sigv4-enabled")
+    @ConfigDescription("Enable AWS Signature version 4 (SigV4)")
+    public IcebergRestCatalogConfig setSigV4Enabled(boolean sigV4Enabled)
+    {
+        this.sigV4Enabled = sigV4Enabled;
+        return this;
+    }
+
     public boolean isCaseInsensitiveNameMatching()
     {
         return caseInsensitiveNameMatching;

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogModule.java

@@ -13,6 +13,7 @@
  */
 package io.trino.plugin.iceberg.catalog.rest;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.inject.Binder;
 import com.google.inject.Scopes;
 import io.airlift.configuration.AbstractConfigurationAwareModule;
@@ -39,6 +40,14 @@ protected void setup(Binder binder)
                 config -> config.getSecurity() == Security.OAUTH2,
                 new OAuth2SecurityModule(),
                 new NoneSecurityModule()));
+        install(conditionalModule(
+                IcebergRestCatalogConfig.class,
+                IcebergRestCatalogConfig::isSigV4Enabled,
+                internalBinder -> {
+                    configBinder(internalBinder).bindConfig(IcebergRestCatalogSigV4Config.class);
+                    internalBinder.bind(AwsProperties.class).to(SigV4AwsProperties.class).in(Scopes.SINGLETON);
+                },
+                internalBinder -> internalBinder.bind(AwsProperties.class).toInstance(ImmutableMap::of)));
 
         binder.bind(TrinoCatalogFactory.class).to(TrinoIcebergRestCatalogFactory.class).in(Scopes.SINGLETON);
         newOptionalBinder(binder, IcebergFileSystemFactory.class).setBinding().to(IcebergRestCatalogFileSystemFactory.class).in(Scopes.SINGLETON);

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogSigV4Config.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.iceberg.catalog.rest;
+
+import io.airlift.configuration.Config;
+import io.airlift.configuration.ConfigDescription;
+import jakarta.validation.constraints.NotNull;
+import org.apache.iceberg.aws.AwsProperties;
+
+public class IcebergRestCatalogSigV4Config
+{
+    private String signingName = AwsProperties.REST_SIGNING_NAME_DEFAULT;
+
+    @NotNull
+    public String getSigningName()
+    {
+        return signingName;
+    }
+
+    @Config("iceberg.rest-catalog.signing-name")
+    @ConfigDescription("AWS SigV4 signing service name")
+    public IcebergRestCatalogSigV4Config setSigningName(String signingName)
+    {
+        this.signingName = signingName;
+        return this;
+    }
+}

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/SigV4AwsProperties.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.iceberg.catalog.rest;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
+import io.trino.filesystem.s3.S3FileSystemConfig;
+import io.trino.plugin.iceberg.IcebergSecurityConfig;
+
+import java.util.Map;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static io.trino.plugin.iceberg.IcebergSecurityConfig.IcebergSecurity.READ_ONLY;
+import static java.util.Objects.requireNonNull;
+
+public class SigV4AwsProperties
+        implements AwsProperties
+{
+    private final Map<String, String> properties;
+
+    @Inject
+    public SigV4AwsProperties(IcebergSecurityConfig securityConfig, IcebergRestCatalogSigV4Config sigV4Config, S3FileSystemConfig s3Config)
+    {
+        // TODO https://github.com/trinodb/trino/issues/24916 Allow write operations with SigV4
+        checkArgument(securityConfig.getSecuritySystem() == READ_ONLY, "Read-only security system is required");
+        this.properties = ImmutableMap.<String, String>builder()
+                .put("rest.sigv4-enabled", "true")
+                .put("rest.signing-name", sigV4Config.getSigningName())
+                .put("rest.access-key-id", requireNonNull(s3Config.getAwsAccessKey(), "s3.aws-access-key is null"))
+                .put("rest.secret-access-key", requireNonNull(s3Config.getAwsSecretKey(), "s3.aws-secret-key is null"))
+                .put("rest.signing-region", requireNonNull(s3Config.getRegion(), "s3.region is null"))
+                .put("rest-metrics-reporting-enabled", "false")
+                .buildOrThrow();
+    }
+
+    @Override
+    public Map<String, String> get()
+    {
+        return properties;
+    }
+}

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoIcebergRestCatalogFactory.java

@@ -57,7 +57,9 @@ public class TrinoIcebergRestCatalogFactory
     private final boolean nestedNamespaceEnabled;
     private final SessionType sessionType;
     private final boolean vendedCredentialsEnabled;
+    private final boolean viewEndpointsEnabled;
     private final SecurityProperties securityProperties;
+    private final AwsProperties awsProperties;
     private final boolean uniqueTableLocation;
     private final TypeManager typeManager;
     private final boolean caseInsensitiveNameMatching;
@@ -73,6 +75,7 @@ public TrinoIcebergRestCatalogFactory(
             CatalogName catalogName,
             IcebergRestCatalogConfig restConfig,
             SecurityProperties securityProperties,
+            AwsProperties awsProperties,
             IcebergConfig icebergConfig,
             TypeManager typeManager,
             NodeVersion nodeVersion)
@@ -87,7 +90,9 @@ public TrinoIcebergRestCatalogFactory(
         this.nestedNamespaceEnabled = restConfig.isNestedNamespaceEnabled();
         this.sessionType = restConfig.getSessionType();
         this.vendedCredentialsEnabled = restConfig.isVendedCredentialsEnabled();
+        this.viewEndpointsEnabled = restConfig.isViewEndpointsEnabled();
         this.securityProperties = requireNonNull(securityProperties, "securityProperties is null");
+        this.awsProperties = requireNonNull(awsProperties, "awsProperties is null");
         requireNonNull(icebergConfig, "icebergConfig is null");
         this.uniqueTableLocation = icebergConfig.isUniqueTableLocation();
         this.typeManager = requireNonNull(typeManager, "typeManager is null");
@@ -112,9 +117,10 @@ public synchronized TrinoCatalog create(ConnectorIdentity identity)
             properties.put(CatalogProperties.URI, serverUri.toString());
             warehouse.ifPresent(location -> properties.put(CatalogProperties.WAREHOUSE_LOCATION, location));
             prefix.ifPresent(prefix -> properties.put("prefix", prefix));
-            properties.put("view-endpoints-supported", "true");
+            properties.put("view-endpoints-supported", Boolean.toString(viewEndpointsEnabled));
             properties.put("trino-version", trinoVersion);
             properties.putAll(securityProperties.get());
+            properties.putAll(awsProperties.get());
 
             if (vendedCredentialsEnabled) {
                 properties.put("header.X-Iceberg-Access-Delegation", "vended-credentials");

plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogConfig.java

@@ -37,6 +37,8 @@ public void testDefaults()
                 .setSessionType(IcebergRestCatalogConfig.SessionType.NONE)
                 .setSecurity(IcebergRestCatalogConfig.Security.NONE)
                 .setVendedCredentialsEnabled(false)
+                .setViewEndpointsEnabled(true)
+                .setSigV4Enabled(false)
                 .setCaseInsensitiveNameMatching(false)
                 .setCaseInsensitiveNameMatchingCacheTtl(new Duration(1, MINUTES)));
     }
@@ -52,6 +54,8 @@ public void testExplicitPropertyMappings()
                 .put("iceberg.rest-catalog.security", "OAUTH2")
                 .put("iceberg.rest-catalog.session", "USER")
                 .put("iceberg.rest-catalog.vended-credentials-enabled", "true")
+                .put("iceberg.rest-catalog.view-endpoints-enabled", "false")
+                .put("iceberg.rest-catalog.sigv4-enabled", "true")
                 .put("iceberg.rest-catalog.case-insensitive-name-matching", "true")
                 .put("iceberg.rest-catalog.case-insensitive-name-matching.cache-ttl", "3m")
                 .buildOrThrow();
@@ -64,6 +68,8 @@ public void testExplicitPropertyMappings()
                 .setSessionType(IcebergRestCatalogConfig.SessionType.USER)
                 .setSecurity(IcebergRestCatalogConfig.Security.OAUTH2)
                 .setVendedCredentialsEnabled(true)
+                .setViewEndpointsEnabled(false)
+                .setSigV4Enabled(true)
                 .setCaseInsensitiveNameMatching(true)
                 .setCaseInsensitiveNameMatchingCacheTtl(new Duration(3, MINUTES));
 

plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogSigV4Config.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.iceberg.catalog.rest;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
+import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
+import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults;
+
+final class TestIcebergRestCatalogSigV4Config
+{
+    @Test
+    void testDefaults()
+    {
+        assertRecordedDefaults(recordDefaults(IcebergRestCatalogSigV4Config.class)
+                .setSigningName("execute-api"));
+    }
+
+    @Test
+    void testExplicitPropertyMappings()
+    {
+        Map<String, String> properties = ImmutableMap.<String, String>builder()
+                .put("iceberg.rest-catalog.signing-name", "glue")
+                .buildOrThrow();
+
+        IcebergRestCatalogSigV4Config expected = new IcebergRestCatalogSigV4Config()
+                .setSigningName("glue");
+
+        assertFullMapping(properties, expected);
+    }
+}