diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java index 9aad784be865..9710aa92e89c 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java @@ -356,6 +356,16 @@ public class S3FileIOProperties implements Serializable { public static final boolean DUALSTACK_ENABLED_DEFAULT = false; + /** + * Determines if S3 client will allow Cross-Region bucket access, default to false. + * + *

For more details, see + * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/s3-cross-region.html + */ + public static final String CROSS_REGION_ACCESS_ENABLED = "s3.cross-region-access-enabled"; + + public static final boolean CROSS_REGION_ACCESS_ENABLED_DEFAULT = false; + /** * Used by {@link S3FileIO}, prefix used for bucket access point configuration. To set, we can * pass a catalog property. @@ -399,6 +409,7 @@ public class S3FileIOProperties implements Serializable { private final Map bucketToAccessPointMapping; private boolean isPreloadClientEnabled; private boolean isDualStackEnabled; + private boolean isCrossRegionAccessEnabled; private boolean isPathStyleAccess; private boolean isUseArnRegionEnabled; private boolean isAccelerationEnabled; @@ -431,6 +442,7 @@ public S3FileIOProperties() { this.bucketToAccessPointMapping = Collections.emptyMap(); this.isPreloadClientEnabled = PRELOAD_CLIENT_ENABLED_DEFAULT; this.isDualStackEnabled = DUALSTACK_ENABLED_DEFAULT; + this.isCrossRegionAccessEnabled = CROSS_REGION_ACCESS_ENABLED_DEFAULT; this.isPathStyleAccess = PATH_STYLE_ACCESS_DEFAULT; this.isUseArnRegionEnabled = USE_ARN_REGION_ENABLED_DEFAULT; this.isAccelerationEnabled = ACCELERATION_ENABLED_DEFAULT; @@ -472,6 +484,9 @@ public S3FileIOProperties(Map properties) { properties, ACCELERATION_ENABLED, ACCELERATION_ENABLED_DEFAULT); this.isDualStackEnabled = PropertyUtil.propertyAsBoolean(properties, DUALSTACK_ENABLED, DUALSTACK_ENABLED_DEFAULT); + this.isCrossRegionAccessEnabled = + PropertyUtil.propertyAsBoolean( + properties, CROSS_REGION_ACCESS_ENABLED, CROSS_REGION_ACCESS_ENABLED_DEFAULT); try { this.multiPartSize = PropertyUtil.propertyAsInt(properties, MULTIPART_SIZE, MULTIPART_SIZE_DEFAULT); @@ -625,6 +640,10 @@ public boolean isDualStackEnabled() { return this.isDualStackEnabled; } + public boolean isCrossRegionAccessEnabled() { + return this.isCrossRegionAccessEnabled; + } + public boolean isPathStyleAccess() { return this.isPathStyleAccess; } @@ -749,7 +768,7 @@ public void applyCredentialConfigurations( /** * Configure services settings for an S3 client. The settings include: s3DualStack, - * s3UseArnRegion, s3PathStyleAccess, and s3Acceleration + * crossRegionAccessEnabled, s3UseArnRegion, s3PathStyleAccess, and s3Acceleration * *

Sample usage: * @@ -760,6 +779,7 @@ public void applyCredentialConfigurations( public void applyServiceConfigurations(T builder) { builder .dualstackEnabled(isDualStackEnabled) + .crossRegionAccessEnabled(isCrossRegionAccessEnabled) .serviceConfiguration( S3Configuration.builder() .pathStyleAccessEnabled(isPathStyleAccess) diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java index 8dda5e16338e..616e185072fb 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java +++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java @@ -66,6 +66,9 @@ public void testS3FileIOPropertiesDefaultValues() { Assertions.assertThat(S3FileIOProperties.DUALSTACK_ENABLED_DEFAULT) .isEqualTo(s3FileIOProperties.isDualStackEnabled()); + Assertions.assertThat(S3FileIOProperties.CROSS_REGION_ACCESS_ENABLED_DEFAULT) + .isEqualTo(s3FileIOProperties.isCrossRegionAccessEnabled()); + Assertions.assertThat(S3FileIOProperties.PATH_STYLE_ACCESS_DEFAULT) .isEqualTo(s3FileIOProperties.isPathStyleAccess()); @@ -155,6 +158,11 @@ public void testS3FileIOProperties() { S3FileIOProperties.DUALSTACK_ENABLED, String.valueOf(s3FileIOProperties.isDualStackEnabled())); + Assertions.assertThat(map) + .containsEntry( + S3FileIOProperties.CROSS_REGION_ACCESS_ENABLED, + String.valueOf(s3FileIOProperties.isCrossRegionAccessEnabled())); + Assertions.assertThat(map) .containsEntry( S3FileIOProperties.PATH_STYLE_ACCESS, @@ -383,6 +391,7 @@ private Map getTestProperties() { map.put(S3FileIOProperties.USE_ARN_REGION_ENABLED, "true"); map.put(S3FileIOProperties.ACCELERATION_ENABLED, "true"); map.put(S3FileIOProperties.DUALSTACK_ENABLED, "true"); + map.put(S3FileIOProperties.CROSS_REGION_ACCESS_ENABLED, "true"); map.put( S3FileIOProperties.MULTIPART_SIZE, String.valueOf(S3FileIOProperties.MULTIPART_SIZE_DEFAULT)); @@ -428,6 +437,7 @@ public void testApplyCredentialConfigurations() { public void testApplyS3ServiceConfigurations() { Map properties = Maps.newHashMap(); properties.put(S3FileIOProperties.DUALSTACK_ENABLED, "true"); + properties.put(S3FileIOProperties.CROSS_REGION_ACCESS_ENABLED, "true"); properties.put(S3FileIOProperties.PATH_STYLE_ACCESS, "true"); properties.put(S3FileIOProperties.USE_ARN_REGION_ENABLED, "true"); // acceleration enabled has to be set to false if path style is true @@ -439,6 +449,7 @@ public void testApplyS3ServiceConfigurations() { ArgumentCaptor.forClass(S3Configuration.class); Mockito.doReturn(mockA).when(mockA).dualstackEnabled(Mockito.anyBoolean()); + Mockito.doReturn(mockA).when(mockA).crossRegionAccessEnabled(Mockito.anyBoolean()); Mockito.doReturn(mockA).when(mockA).serviceConfiguration(Mockito.any(S3Configuration.class)); s3FileIOProperties.applyServiceConfigurations(mockA); diff --git a/docs/docs/aws.md b/docs/docs/aws.md index eb217e424d44..9ab31e8d45e4 100644 --- a/docs/docs/aws.md +++ b/docs/docs/aws.md @@ -500,6 +500,22 @@ spark-sql --conf spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCata For more details on using S3 Access Grants, please refer to [Managing access with S3 Access Grants](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-grants.html). +### S3 Cross-Region Access + +S3 Cross-Region bucket access can be turned on by setting catalog property `s3.cross-region-access-enabled` to `true`. +This is turned off by default to avoid first S3 API call increased latency. + +For example, to enable S3 Cross-Region bucket access with Spark 3.3, you can start the Spark SQL shell with: +``` +spark-sql --conf spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.catalog.my_catalog.warehouse=s3://my-bucket2/my/key/prefix \ + --conf spark.sql.catalog.my_catalog.type=glue \ + --conf spark.sql.catalog.my_catalog.io-impl=org.apache.iceberg.aws.s3.S3FileIO \ + --conf spark.sql.catalog.my_catalog.s3.cross-region-access-enabled=true +``` + +For more details, please refer to [Cross-Region access for Amazon S3](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/s3-cross-region.html). + ### S3 Acceleration [S3 Acceleration](https://aws.amazon.com/s3/transfer-acceleration/) can be used to speed up transfers to and from Amazon S3 by as much as 50-500% for long-distance transfer of larger objects.