diff --git a/presto-docs/src/main/sphinx/connector/hive.rst b/presto-docs/src/main/sphinx/connector/hive.rst index d28f42679076..341cabb32546 100644 --- a/presto-docs/src/main/sphinx/connector/hive.rst +++ b/presto-docs/src/main/sphinx/connector/hive.rst @@ -327,6 +327,10 @@ Property Name Description set this to the AWS region-specific endpoint (e.g., ``http[s]://.s3-.amazonaws.com``). +``hive.s3.storage-class`` The S3 storage class to use when writing the data. Currently only + ``STANDARD`` and ``INTELLIGENT_TIERING`` storage classes are supported. + Default storage class is ``STANDARD`` + ``hive.s3.signer-type`` Specify a different signer type for S3-compatible storage. Example: ``S3SignerType`` for v2 signer type diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/HiveS3Config.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/HiveS3Config.java index 31cbb828d03c..c3cd31340cad 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/HiveS3Config.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/HiveS3Config.java @@ -35,6 +35,7 @@ public class HiveS3Config private String s3AwsAccessKey; private String s3AwsSecretKey; private String s3Endpoint; + private PrestoS3StorageClass s3StorageClass = PrestoS3StorageClass.STANDARD; private PrestoS3SignerType s3SignerType; private String s3SignerClass; private boolean s3PathStyleAccess; @@ -99,6 +100,20 @@ public HiveS3Config setS3Endpoint(String s3Endpoint) return this; } + @NotNull + public PrestoS3StorageClass getS3StorageClass() + { + return s3StorageClass; + } + + @Config("hive.s3.storage-class") + @ConfigDescription("AWS S3 storage class to use when writing the data") + public HiveS3Config setS3StorageClass(PrestoS3StorageClass s3StorageClass) + { + this.s3StorageClass = s3StorageClass; + return this; + } + public PrestoS3SignerType getS3SignerType() { return s3SignerType; diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3ConfigurationInitializer.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3ConfigurationInitializer.java index ec8982f22e31..9831575e4634 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3ConfigurationInitializer.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3ConfigurationInitializer.java @@ -49,6 +49,7 @@ import static io.prestosql.plugin.hive.s3.PrestoS3FileSystem.S3_SSE_TYPE; import static io.prestosql.plugin.hive.s3.PrestoS3FileSystem.S3_SSL_ENABLED; import static io.prestosql.plugin.hive.s3.PrestoS3FileSystem.S3_STAGING_DIRECTORY; +import static io.prestosql.plugin.hive.s3.PrestoS3FileSystem.S3_STORAGE_CLASS; import static io.prestosql.plugin.hive.s3.PrestoS3FileSystem.S3_USER_AGENT_PREFIX; import static io.prestosql.plugin.hive.s3.PrestoS3FileSystem.S3_USE_INSTANCE_CREDENTIALS; @@ -58,6 +59,7 @@ public class PrestoS3ConfigurationInitializer private final String awsAccessKey; private final String awsSecretKey; private final String endpoint; + private final PrestoS3StorageClass s3StorageClass; private final PrestoS3SignerType signerType; private final boolean pathStyleAccess; private final boolean useInstanceCredentials; @@ -91,6 +93,7 @@ public PrestoS3ConfigurationInitializer(HiveS3Config config) this.awsAccessKey = config.getS3AwsAccessKey(); this.awsSecretKey = config.getS3AwsSecretKey(); this.endpoint = config.getS3Endpoint(); + this.s3StorageClass = config.getS3StorageClass(); this.signerType = config.getS3SignerType(); this.signerClass = config.getS3SignerClass(); this.pathStyleAccess = config.isS3PathStyleAccess(); @@ -136,6 +139,7 @@ public void initializeConfiguration(Configuration config) if (endpoint != null) { config.set(S3_ENDPOINT, endpoint); } + config.set(S3_STORAGE_CLASS, s3StorageClass.name()); if (signerType != null) { config.set(S3_SIGNER_TYPE, signerType.name()); } diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3FileSystem.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3FileSystem.java index 47e45d674b57..954aa8a29499 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3FileSystem.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3FileSystem.java @@ -53,6 +53,7 @@ import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams; +import com.amazonaws.services.s3.model.StorageClass; import com.amazonaws.services.s3.transfer.Transfer; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; @@ -164,6 +165,7 @@ public class PrestoS3FileSystem public static final String S3_ACL_TYPE = "presto.s3.upload-acl-type"; public static final String S3_SKIP_GLACIER_OBJECTS = "presto.s3.skip-glacier-objects"; public static final String S3_REQUESTER_PAYS_ENABLED = "presto.s3.requester-pays.enabled"; + public static final String S3_STORAGE_CLASS = "presto.s3.storage-class"; static final String S3_DIRECTORY_OBJECT_CONTENT_TYPE = "application/x-directory"; @@ -198,6 +200,7 @@ public class PrestoS3FileSystem private PrestoS3AclType s3AclType; private boolean skipGlacierObjects; private boolean requesterPaysEnabled; + private PrestoS3StorageClass s3StorageClass; @Override public void initialize(URI uri, Configuration conf) @@ -243,6 +246,7 @@ public void initialize(URI uri, Configuration conf) String userAgentPrefix = conf.get(S3_USER_AGENT_PREFIX, defaults.getS3UserAgentPrefix()); this.skipGlacierObjects = conf.getBoolean(S3_SKIP_GLACIER_OBJECTS, defaults.isSkipGlacierObjects()); this.requesterPaysEnabled = conf.getBoolean(S3_REQUESTER_PAYS_ENABLED, defaults.isRequesterPaysEnabled()); + this.s3StorageClass = conf.getEnum(S3_STORAGE_CLASS, defaults.getS3StorageClass()); ClientConfiguration configuration = new ClientConfiguration() .withMaxErrorRetry(maxErrorRetries) @@ -404,7 +408,7 @@ public FSDataOutputStream create(Path path, FsPermission permission, boolean ove String key = keyFromPath(qualifiedPath(path)); return new FSDataOutputStream( - new PrestoS3OutputStream(s3, getBucketName(uri), key, tempFile, sseEnabled, sseType, sseKmsKeyId, multiPartUploadMinFileSize, multiPartUploadMinPartSize, s3AclType, requesterPaysEnabled), + new PrestoS3OutputStream(s3, getBucketName(uri), key, tempFile, sseEnabled, sseType, sseKmsKeyId, multiPartUploadMinFileSize, multiPartUploadMinPartSize, s3AclType, requesterPaysEnabled, s3StorageClass), statistics); } @@ -1171,6 +1175,7 @@ private static class PrestoS3OutputStream private final String sseKmsKeyId; private final CannedAccessControlList aclType; private final boolean requesterPaysEnabled; + private final StorageClass s3StorageClass; private boolean closed; @@ -1185,7 +1190,8 @@ public PrestoS3OutputStream( long multiPartUploadMinFileSize, long multiPartUploadMinPartSize, PrestoS3AclType aclType, - boolean requesterPaysEnabled) + boolean requesterPaysEnabled, + PrestoS3StorageClass s3StorageClass) throws IOException { super(new BufferedOutputStream(new FileOutputStream(requireNonNull(tempFile, "tempFile is null")))); @@ -1196,6 +1202,7 @@ public PrestoS3OutputStream( .withMultipartUploadThreshold(multiPartUploadMinFileSize).build(); requireNonNull(aclType, "aclType is null"); + requireNonNull(s3StorageClass, "s3StorageClass is null"); this.aclType = aclType.getCannedACL(); this.host = requireNonNull(host, "host is null"); this.key = requireNonNull(key, "key is null"); @@ -1204,6 +1211,7 @@ public PrestoS3OutputStream( this.sseType = requireNonNull(sseType, "sseType is null"); this.sseKmsKeyId = sseKmsKeyId; this.requesterPaysEnabled = requesterPaysEnabled; + this.s3StorageClass = s3StorageClass.getS3StorageClass(); log.debug("OutputStream for key '%s' using file: %s", key, tempFile); } @@ -1257,6 +1265,7 @@ private void uploadObject() break; } } + request.withStorageClass(s3StorageClass); request.withCannedAcl(aclType); diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3StorageClass.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3StorageClass.java new file mode 100644 index 000000000000..5351c0d0d4f6 --- /dev/null +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3StorageClass.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.plugin.hive.s3; + +import com.amazonaws.services.s3.model.StorageClass; + +import static java.util.Objects.requireNonNull; + +public enum PrestoS3StorageClass { + STANDARD(StorageClass.Standard), + INTELLIGENT_TIERING(StorageClass.IntelligentTiering); + + private StorageClass s3StorageClass; + + PrestoS3StorageClass(StorageClass s3StorageClass) + { + this.s3StorageClass = requireNonNull(s3StorageClass, "s3StorageClass is null"); + } + + public StorageClass getS3StorageClass() + { + return s3StorageClass; + } +} diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/s3/TestHiveS3Config.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/s3/TestHiveS3Config.java index d8c244fe17fd..858966eb0edd 100644 --- a/presto-hive/src/test/java/io/prestosql/plugin/hive/s3/TestHiveS3Config.java +++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/s3/TestHiveS3Config.java @@ -42,6 +42,7 @@ public void testDefaults() .setS3PathStyleAccess(false) .setS3UseInstanceCredentials(true) .setS3IamRole(null) + .setS3StorageClass(PrestoS3StorageClass.STANDARD) .setS3SslEnabled(true) .setS3SseEnabled(false) .setS3SseType(PrestoS3SseType.S3) @@ -77,6 +78,7 @@ public void testExplicitPropertyMappings() .put("hive.s3.path-style-access", "true") .put("hive.s3.use-instance-credentials", "false") .put("hive.s3.iam-role", "roleArn") + .put("hive.s3.storage-class", "INTELLIGENT_TIERING") .put("hive.s3.ssl.enabled", "false") .put("hive.s3.sse.enabled", "true") .put("hive.s3.sse.type", "KMS") @@ -109,6 +111,7 @@ public void testExplicitPropertyMappings() .setS3PathStyleAccess(true) .setS3UseInstanceCredentials(false) .setS3IamRole("roleArn") + .setS3StorageClass(PrestoS3StorageClass.INTELLIGENT_TIERING) .setS3SslEnabled(false) .setS3SseEnabled(true) .setS3SseType(PrestoS3SseType.KMS)