[HADOOP-18562]: S3A: support custom S3 and STS headers. #6550
+132
−0
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -22,6 +22,9 @@ | |
| import java.net.URI; | ||
| import java.net.URISyntaxException; | ||
| import java.time.Duration; | ||
| import java.util.Arrays; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| import org.slf4j.Logger; | ||
|
|
@@ -72,6 +75,8 @@ | |
| import static org.apache.hadoop.fs.s3a.Constants.SIGNING_ALGORITHM_STS; | ||
| import static org.apache.hadoop.fs.s3a.Constants.SOCKET_TIMEOUT; | ||
| import static org.apache.hadoop.fs.s3a.Constants.USER_AGENT_PREFIX; | ||
| import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_HEADERS_S3; | ||
| import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_HEADERS_STS; | ||
| import static org.apache.hadoop.fs.s3a.impl.ConfigurationHelper.enforceMinimumDuration; | ||
| import static org.apache.hadoop.fs.s3a.impl.ConfigurationHelper.getDuration; | ||
| import static org.apache.hadoop.util.Preconditions.checkArgument; | ||
|
|
@@ -116,6 +121,8 @@ public static ClientOverrideConfiguration.Builder createClientConfigBuilder(Conf | |
|
|
||
| initUserAgent(conf, overrideConfigBuilder); | ||
|
|
||
| initRequestHeaders(conf, overrideConfigBuilder, awsServiceIdentifier); | ||
|
|
||
| String signer = conf.getTrimmed(SIGNING_ALGORITHM, ""); | ||
| if (!signer.isEmpty()) { | ||
| LOG.debug("Signer override = {}", signer); | ||
|
|
@@ -407,6 +414,36 @@ private static void initSigner(Configuration conf, | |
| } | ||
| } | ||
|
|
||
| /** | ||
| * Initialize custom request headers for AWS clients. | ||
| * @param conf hadoop configuration | ||
| * @param clientConfig client configuration to update | ||
| * @param awsServiceIdentifier service name | ||
| */ | ||
| private static void initRequestHeaders(Configuration conf, | ||
| ClientOverrideConfiguration.Builder clientConfig, String awsServiceIdentifier) { | ||
| String configKey = null; | ||
| switch (awsServiceIdentifier) { | ||
| case AWS_SERVICE_IDENTIFIER_S3: | ||
| configKey = CUSTOM_HEADERS_S3; | ||
| break; | ||
| case AWS_SERVICE_IDENTIFIER_STS: | ||
| configKey = CUSTOM_HEADERS_STS; | ||
| break; | ||
| default: | ||
| // No known service. | ||
| } | ||
| if (configKey != null) { | ||
| Map<String, String> awsClientCustomHeadersMap = | ||
| S3AUtils.getTrimmedStringCollectionSplitByEquals(conf, configKey); | ||
| awsClientCustomHeadersMap.forEach((header, valueString) -> { | ||
| List<String> headerValues = Arrays.asList(valueString.split(";")); | ||
|
There was a problem hiding this comment. This is the largest change in this revision. Having ":" as value separator is not a good idea. Because, 1/ Some header names in aws start with Hence changed delimiter to ";" which is not so common - https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html#object-key-guidelines let me know if you have better ideas. |
||
| clientConfig.putHeader(header, headerValues); | ||
| }); | ||
| LOG.debug("headers for {} client = {}", awsServiceIdentifier, clientConfig.headers()); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Configures request timeout in the client configuration. | ||
| * This is independent of the timeouts set in the sync and async HTTP clients; | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
value: {@value}wouldn't render with toLowerCase(), added for other statements.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
ok. I'm going to propose here doing all the next lines
in a single <pre> as there's enough formatting it wouldn't render right. you can then cut the {@literal} bits.
The goal here is generally "looks good on an IDE rendered view"