Skip to content

Add support for static AWS credentials in GlueHiveMetastore #15950

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
highker merged 1 commit into from
May 25, 2021
Merged

Add support for static AWS credentials in GlueHiveMetastore #15950

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions presto-docs/src/main/sphinx/connector/hive.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,16 @@ Property Name Description

``hive.metastore.glue.default-warehouse-dir`` Hive Glue metastore default warehouse directory

``hive.metastore.glue.aws-access-key`` AWS access key to use to connect to the Glue Catalog. If
specified along with ``hive.metastore.glue.aws-secret-key``,
this parameter takes precedence over
``hive.metastore.glue.iam-role``.

``hive.metastore.glue.aws-secret-key`` AWS secret key to use to connect to the Glue Catalog. If
specified along with ``hive.metastore.glue.aws-access-key``,
this parameter takes precedence over
``hive.metastore.glue.iam-role``.

``hive.metastore.glue.catalogid`` The ID of the Glue Catalog in which the metadata database
resides.

Expand Down
9 changes: 8 additions & 1 deletion ...ve-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
import com.amazonaws.metrics.RequestMetricCollector;
Expand Down Expand Up @@ -202,7 +204,12 @@ else if (config.getPinGlueClientToCurrentRegion()) {
}
}

if (config.getIamRole().isPresent()) {
if (config.getAwsAccessKey().isPresent() && config.getAwsSecretKey().isPresent()) {
AWSCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider(
new BasicAWSCredentials(config.getAwsAccessKey().get(), config.getAwsSecretKey().get()));
asyncGlueClientBuilder.setCredentials(credentialsProvider);
}
else if (config.getIamRole().isPresent()) {
AWSCredentialsProvider credentialsProvider = new STSAssumeRoleSessionCredentialsProvider
.Builder(config.getIamRole().get(), "roleSessionName")
.build();
Expand Down
30 changes: 30 additions & 0 deletions ...astore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import com.facebook.airlift.configuration.Config;
import com.facebook.airlift.configuration.ConfigDescription;
import com.facebook.airlift.configuration.ConfigSecuritySensitive;

import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
Expand All @@ -33,6 +34,8 @@ public class GlueHiveMetastoreConfig
private int partitionSegments = 5;
private int getPartitionThreads = 20;
private Optional<String> iamRole = Optional.empty();
private Optional<String> awsAccessKey = Optional.empty();
private Optional<String> awsSecretKey = Optional.empty();

public Optional<String> getGlueRegion()
{
Expand Down Expand Up @@ -167,4 +170,31 @@ public GlueHiveMetastoreConfig setIamRole(String iamRole)
this.iamRole = Optional.ofNullable(iamRole);
return this;
}

public Optional<String> getAwsAccessKey()
{
return awsAccessKey;
}

@Config("hive.metastore.glue.aws-access-key")
@ConfigDescription("Hive Glue metastore AWS access key")
public GlueHiveMetastoreConfig setAwsAccessKey(String awsAccessKey)
{
this.awsAccessKey = Optional.ofNullable(awsAccessKey);
return this;
}

public Optional<String> getAwsSecretKey()
{
return awsSecretKey;
}

@Config("hive.metastore.glue.aws-secret-key")
@ConfigDescription("Hive Glue metastore AWS secret key")
@ConfigSecuritySensitive
public GlueHiveMetastoreConfig setAwsSecretKey(String awsSecretKey)
{
this.awsSecretKey = Optional.ofNullable(awsSecretKey);
return this;
}
}
10 changes: 8 additions & 2 deletions ...re/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@ public void testDefaults()
.setCatalogId(null)
.setPartitionSegments(5)
.setGetPartitionThreads(20)
.setIamRole(null));
.setIamRole(null)
.setAwsAccessKey(null)
.setAwsSecretKey(null));
}

@Test
Expand All @@ -54,6 +56,8 @@ public void testExplicitPropertyMapping()
.put("hive.metastore.glue.partitions-segments", "10")
.put("hive.metastore.glue.get-partition-threads", "42")
.put("hive.metastore.glue.iam-role", "role")
.put("hive.metastore.glue.aws-access-key", "ABC")
.put("hive.metastore.glue.aws-secret-key", "DEF")
.build();

GlueHiveMetastoreConfig expected = new GlueHiveMetastoreConfig()
Expand All @@ -66,7 +70,9 @@ public void testExplicitPropertyMapping()
.setCatalogId("0123456789")
.setPartitionSegments(10)
.setGetPartitionThreads(42)
.setIamRole("role");
.setIamRole("role")
.setAwsAccessKey("ABC")
.setAwsSecretKey("DEF");

assertFullMapping(properties, expected);
}
Expand Down