Skip to content

AWS: create HttpClientProperties, move s3 related methods into S3FileIOProperties #7562

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
amogh-jahagirdar merged 8 commits into from
May 10, 2023
Merged

AWS: create HttpClientProperties, move s3 related methods into S3FileIOProperties #7562

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
1b74bca
AWS: create separate class for HttpClientProperties, move s3 helper m…
akshayakp97 May 8, 2023
5f50536
fix javadoc in S3FileIOProperties and HttpClientProperties
akshayakp97 May 8, 2023
65891ad
add AwsClientProperties class for aws creds provider
akshayakp97 May 9, 2023
7769028
fix javadoc in AwsProperties
akshayakp97 May 9, 2023
5601d12
add client region to AwsClientProperties, refactor credentialsProvide…
akshayakp97 May 9, 2023
74876a0
check secret key for null, update javadoc
akshayakp97 May 9, 2023
4c7494b
both access key and secret key should be non-null
akshayakp97 May 9, 2023
c59b956
remove javadoc from private method
akshayakp97 May 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 85 additions & 4 deletions aws/src/main/java/org/apache/iceberg/aws/AwsProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -493,24 +493,37 @@ public class AwsProperties implements Serializable {
* AwsClientFactory} If set, all AWS clients will use this specified HTTP client. If not set,
* {@link #HTTP_CLIENT_TYPE_DEFAULT} will be used. For specific types supported, see
* HTTP_CLIENT_TYPE_* defined below.
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
public static final String HTTP_CLIENT_TYPE = "http-client.type";
@Deprecated public static final String HTTP_CLIENT_TYPE = "http-client.type";

/**
* If this is set under {@link #HTTP_CLIENT_TYPE}, {@link
* software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient} will be used as the HTTP
* Client in {@link AwsClientFactory}
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
public static final String HTTP_CLIENT_TYPE_URLCONNECTION = "urlconnection";
@Deprecated public static final String HTTP_CLIENT_TYPE_URLCONNECTION = "urlconnection";

/**
* If this is set under {@link #HTTP_CLIENT_TYPE}, {@link
* software.amazon.awssdk.http.apache.ApacheHttpClient} will be used as the HTTP Client in {@link
* AwsClientFactory}
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
public static final String HTTP_CLIENT_TYPE_APACHE = "apache";
@Deprecated public static final String HTTP_CLIENT_TYPE_APACHE = "apache";

public static final String HTTP_CLIENT_TYPE_DEFAULT = HTTP_CLIENT_TYPE_APACHE;
/**
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead

@amogh-jahagirdar amogh-jahagirdar May 9, 2023

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The indentation for instead on the newline (and in all the other places) looks off to me, is this how spotless is formatting it?

@akshayakp97 akshayakp97 May 9, 2023

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yup

*/
@Deprecated public static final String HTTP_CLIENT_TYPE_DEFAULT = HTTP_CLIENT_TYPE_APACHE;

/**
* Used to configure the connection timeout in milliseconds for {@link
Expand All @@ -519,7 +532,11 @@ public class AwsProperties implements Serializable {
*
* <p>For more details, see
* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/urlconnection/UrlConnectionHttpClient.Builder.html
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
@Deprecated
public static final String HTTP_CLIENT_URLCONNECTION_CONNECTION_TIMEOUT_MS =
"http-client.urlconnection.connection-timeout-ms";

Expand All @@ -530,7 +547,11 @@ public class AwsProperties implements Serializable {
*
* <p>For more details, see
* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/urlconnection/UrlConnectionHttpClient.Builder.html
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
@Deprecated
public static final String HTTP_CLIENT_URLCONNECTION_SOCKET_TIMEOUT_MS =
"http-client.urlconnection.socket-timeout-ms";

Expand All @@ -541,7 +562,11 @@ public class AwsProperties implements Serializable {
*
* <p>For more details, see
* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
@Deprecated
public static final String HTTP_CLIENT_APACHE_CONNECTION_TIMEOUT_MS =
"http-client.apache.connection-timeout-ms";

Expand All @@ -552,7 +577,11 @@ public class AwsProperties implements Serializable {
*
* <p>For more details, see
* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
@Deprecated
public static final String HTTP_CLIENT_APACHE_SOCKET_TIMEOUT_MS =
"http-client.apache.socket-timeout-ms";

Expand All @@ -563,7 +592,11 @@ public class AwsProperties implements Serializable {
*
* <p>For more details, see
* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
@Deprecated
public static final String HTTP_CLIENT_APACHE_CONNECTION_ACQUISITION_TIMEOUT_MS =
"http-client.apache.connection-acquisition-timeout-ms";

Expand All @@ -574,7 +607,11 @@ public class AwsProperties implements Serializable {
*
* <p>For more details, see
* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
@Deprecated
public static final String HTTP_CLIENT_APACHE_CONNECTION_MAX_IDLE_TIME_MS =
"http-client.apache.connection-max-idle-time-ms";

Expand All @@ -585,7 +622,11 @@ public class AwsProperties implements Serializable {
*
* <p>For more details, see
* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
@Deprecated
public static final String HTTP_CLIENT_APACHE_CONNECTION_TIME_TO_LIVE_MS =
"http-client.apache.connection-time-to-live-ms";

Expand All @@ -598,7 +639,11 @@ public class AwsProperties implements Serializable {
*
* <p>For more details, see
* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
@Deprecated
public static final String HTTP_CLIENT_APACHE_EXPECT_CONTINUE_ENABLED =
"http-client.apache.expect-continue-enabled";

Expand All @@ -609,7 +654,11 @@ public class AwsProperties implements Serializable {
*
* <p>For more details, see
* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
@Deprecated
public static final String HTTP_CLIENT_APACHE_MAX_CONNECTIONS =
"http-client.apache.max-connections";

Expand All @@ -622,7 +671,11 @@ public class AwsProperties implements Serializable {
*
* <p>For more details, see
* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
@Deprecated
public static final String HTTP_CLIENT_APACHE_TCP_KEEP_ALIVE_ENABLED =
"http-client.apache.tcp-keep-alive-enabled";

Expand All @@ -635,7 +688,11 @@ public class AwsProperties implements Serializable {
*
* <p>For more details, see
* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
@Deprecated
public static final String HTTP_CLIENT_APACHE_USE_IDLE_CONNECTION_REAPER_ENABLED =
"http-client.apache.use-idle-connection-reaper-enabled";
/**
Expand Down Expand Up @@ -1505,7 +1562,11 @@ public void setClientRegion(String clientRegion) {
* <pre>
* S3Client.builder().applyMutation(awsProperties::applyS3CredentialConfigurations)
* </pre>
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties}
* instead
*/
@Deprecated
public <T extends S3ClientBuilder> void applyS3CredentialConfigurations(T builder) {
builder.credentialsProvider(
s3RemoteSigningEnabled
Expand Down Expand Up @@ -1552,7 +1613,11 @@ public <T extends AwsClientBuilder> void applyClientCredentialConfigurations(T b
* <pre>
* S3Client.builder().applyMutation(awsProperties::applyS3ServiceConfigurations)
* </pre>
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties}
* instead
*/
@Deprecated
public <T extends S3ClientBuilder> void applyS3ServiceConfigurations(T builder) {
builder
.dualstackEnabled(s3DualStackEnabled)
Expand All @@ -1572,7 +1637,11 @@ public <T extends S3ClientBuilder> void applyS3ServiceConfigurations(T builder)
* <pre>
* S3Client.builder().applyMutation(awsProperties::applyS3SignerConfiguration)
* </pre>
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties}
* instead
*/
@Deprecated
public <T extends S3ClientBuilder> void applyS3SignerConfiguration(T builder) {
if (s3RemoteSigningEnabled) {
builder.overrideConfiguration(
Expand All @@ -1591,7 +1660,11 @@ public <T extends S3ClientBuilder> void applyS3SignerConfiguration(T builder) {
* <pre>
* S3Client.builder().applyMutation(awsProperties::applyHttpClientConfigurations)
* </pre>
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
@Deprecated
public <T extends AwsSyncClientBuilder> void applyHttpClientConfigurations(T builder) {
if (Strings.isNullOrEmpty(httpClientType)) {
httpClientType = HTTP_CLIENT_TYPE_DEFAULT;
Expand Down Expand Up @@ -1620,7 +1693,11 @@ public <T extends AwsSyncClientBuilder> void applyHttpClientConfigurations(T bui
* <pre>
* S3Client.builder().applyMutation(awsProperties::applyS3EndpointConfigurations)
* </pre>
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.s3.S3FileIOProperties}
* instead
*/
@Deprecated
public <T extends S3ClientBuilder> void applyS3EndpointConfigurations(T builder) {
configureEndpoint(builder, s3Endpoint);
}
Expand Down Expand Up @@ -1760,7 +1837,11 @@ private <T extends SdkClientBuilder> void configureEndpoint(T builder, String en
* software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient} and {@link
* software.amazon.awssdk.http.apache.ApacheHttpClient}, since including both will cause error
* described in <a href="https://github.com/apache/iceberg/issues/6715">issue#6715</a>
*
* @deprecated will be removed in 1.4.0, use {@link org.apache.iceberg.aws.HttpClientProperties}
* instead
*/
@Deprecated
private <T> T loadHttpClientConfigurations(String impl) {
Object httpClientConfigurations;
try {
Expand Down
Loading