From a93320e527ac8ee87ea78ae162d16c5a90bdfbe8 Mon Sep 17 00:00:00 2001 From: v-jizhang Date: Tue, 20 Jul 2021 15:01:34 -0700 Subject: [PATCH 1/3] Get PrestoS3FileSystem to work with the AWS Default Credentials Provider Cherry-pick of https://github.com/trinodb/trino/pull/741 DefaultAWSCredentialsProviderChain is frequently used by AWS customers and it provides access from a documented list of sources. This especially makes it easier to run Presto on non-EC2 hosts where you don't have the instance profile. (e.g. Macs, during development). This change also makes PrestoS3FileSystem to be consistent with the Glue connector in Presto. Co-authored-by: Anoop Johnson --- .../hive/metastore/glue/GlueHiveMetastore.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java index 7f8a8e3385519..5c7ae2b13863e 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java @@ -18,6 +18,7 @@ import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; import com.amazonaws.metrics.RequestMetricCollector; @@ -204,19 +205,24 @@ else if (config.getPinGlueClientToCurrentRegion()) { } } + asyncGlueClientBuilder.setCredentials(getAwsCredentialsProvider(config)); + + return asyncGlueClientBuilder.build(); + } + + private static AWSCredentialsProvider getAwsCredentialsProvider(GlueHiveMetastoreConfig config) + { if (config.getAwsAccessKey().isPresent() && config.getAwsSecretKey().isPresent()) { - AWSCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider( + return new AWSStaticCredentialsProvider( new BasicAWSCredentials(config.getAwsAccessKey().get(), config.getAwsSecretKey().get())); - asyncGlueClientBuilder.setCredentials(credentialsProvider); } else if (config.getIamRole().isPresent()) { - AWSCredentialsProvider credentialsProvider = new STSAssumeRoleSessionCredentialsProvider + return new STSAssumeRoleSessionCredentialsProvider .Builder(config.getIamRole().get(), "roleSessionName") .build(); - asyncGlueClientBuilder.setCredentials(credentialsProvider); } - return asyncGlueClientBuilder.build(); + return DefaultAWSCredentialsProviderChain.getInstance(); } @Managed From 3aa0e22d4d316b8aa454e35edb72e1c1352cce40 Mon Sep 17 00:00:00 2001 From: v-jizhang Date: Tue, 20 Jul 2021 15:49:58 -0700 Subject: [PATCH 2/3] Enable instance and custom credentials provider for glue Cherry-pick of https://github.com/trinodb/trino/pull/1363 Add a config for glue credential hive.metastore.glue.aws-credentials-provider where value is fully qualified class name. Co-authored-by: Li Yu --- .../metastore/glue/GlueHiveMetastore.java | 20 ++++++++++++++++++- .../glue/GlueHiveMetastoreConfig.java | 13 ++++++++++++ .../glue/TestGlueHiveMetastoreConfig.java | 7 +++++-- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java index 5c7ae2b13863e..bf1be516e1f71 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java @@ -138,6 +138,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.collect.Comparators.lexicographical; +import static java.lang.String.format; import static java.util.Comparator.comparing; import static java.util.Objects.requireNonNull; import static java.util.function.UnaryOperator.identity; @@ -216,15 +217,32 @@ private static AWSCredentialsProvider getAwsCredentialsProvider(GlueHiveMetastor return new AWSStaticCredentialsProvider( new BasicAWSCredentials(config.getAwsAccessKey().get(), config.getAwsSecretKey().get())); } - else if (config.getIamRole().isPresent()) { + if (config.getIamRole().isPresent()) { return new STSAssumeRoleSessionCredentialsProvider .Builder(config.getIamRole().get(), "roleSessionName") .build(); } + if (config.getAwsCredentialsProvider().isPresent()) { + return getCustomAWSCredentialsProvider(config.getAwsCredentialsProvider().get()); + } return DefaultAWSCredentialsProviderChain.getInstance(); } + private static AWSCredentialsProvider getCustomAWSCredentialsProvider(String providerClass) + { + try { + Object instance = Class.forName(providerClass).getConstructor().newInstance(); + if (!(instance instanceof AWSCredentialsProvider)) { + throw new RuntimeException("Invalid credentials provider class: " + instance.getClass().getName()); + } + return (AWSCredentialsProvider) instance; + } + catch (ReflectiveOperationException e) { + throw new RuntimeException(format("Error creating an instance of %s", providerClass), e); + } + } + @Managed @Flatten public GlueMetastoreStats getStats() diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java index d2f17a75492d4..7054934044a12 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java @@ -36,6 +36,7 @@ public class GlueHiveMetastoreConfig private Optional iamRole = Optional.empty(); private Optional awsAccessKey = Optional.empty(); private Optional awsSecretKey = Optional.empty(); + private Optional awsCredentialsProvider = Optional.empty(); public Optional getGlueRegion() { @@ -197,4 +198,16 @@ public GlueHiveMetastoreConfig setAwsSecretKey(String awsSecretKey) this.awsSecretKey = Optional.ofNullable(awsSecretKey); return this; } + + public Optional getAwsCredentialsProvider() + { + return awsCredentialsProvider; + } + + @Config("hive.metastore.glue.aws-credentials-provider") + public GlueHiveMetastoreConfig setAwsCredentialsProvider(String awsCredentialsProvider) + { + this.awsCredentialsProvider = Optional.ofNullable(awsCredentialsProvider); + return this; + } } diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java index b7b072bf14ebd..28a6ea087e50d 100644 --- a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java @@ -39,7 +39,8 @@ public void testDefaults() .setGetPartitionThreads(20) .setIamRole(null) .setAwsAccessKey(null) - .setAwsSecretKey(null)); + .setAwsSecretKey(null) + .setAwsCredentialsProvider(null)); } @Test @@ -58,6 +59,7 @@ public void testExplicitPropertyMapping() .put("hive.metastore.glue.iam-role", "role") .put("hive.metastore.glue.aws-access-key", "ABC") .put("hive.metastore.glue.aws-secret-key", "DEF") + .put("hive.metastore.glue.aws-credentials-provider", "custom") .build(); GlueHiveMetastoreConfig expected = new GlueHiveMetastoreConfig() @@ -72,7 +74,8 @@ public void testExplicitPropertyMapping() .setGetPartitionThreads(42) .setIamRole("role") .setAwsAccessKey("ABC") - .setAwsSecretKey("DEF"); + .setAwsSecretKey("DEF") + .setAwsCredentialsProvider("custom"); assertFullMapping(properties, expected); } From a9f5300d79422bc929b1a8ee9b424c092c147168 Mon Sep 17 00:00:00 2001 From: v-jizhang Date: Tue, 20 Jul 2021 16:28:44 -0700 Subject: [PATCH 3/3] Document missing AWS Glue config in Hive connector Cherry-pick of https://github.com/trinodb/trino/pull/3689 Add configuration property ``hive.metastore.glue.aws-credentials-provider`` to supply a custom credential provider. Co-authored-by: Ashhar Hasan Co-authored-by: Piotr Findeisen --- presto-docs/src/main/sphinx/connector/hive.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/presto-docs/src/main/sphinx/connector/hive.rst b/presto-docs/src/main/sphinx/connector/hive.rst index 7b33d6199f338..d2ac68e51bee6 100644 --- a/presto-docs/src/main/sphinx/connector/hive.rst +++ b/presto-docs/src/main/sphinx/connector/hive.rst @@ -231,6 +231,10 @@ Property Name Description ``hive.metastore.glue.default-warehouse-dir`` Hive Glue metastore default warehouse directory +``hive.metastore.glue.aws-credentials-provider`` Fully qualified name of the Java class to use for obtaining + AWS credentials. Can be used to supply a custom credentials + provider. + ``hive.metastore.glue.aws-access-key`` AWS access key to use to connect to the Glue Catalog. If specified along with ``hive.metastore.glue.aws-secret-key``, this parameter takes precedence over