From 9fe072e053fa52527877d639b65063eda4769881 Mon Sep 17 00:00:00 2001 From: Ashutosh Date: Sat, 16 Jul 2022 16:47:56 -0400 Subject: [PATCH 1/4] added new parameter object in s3ClientCreationParameters that returns full s3a path --- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 1 + .../apache/hadoop/fs/s3a/S3ClientFactory.java | 29 ++++++++++++++++++- .../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 1 + .../ITestSessionDelegationInFileystem.java | 1 + 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 40671e0d334cc..032b482d6f6b1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -888,6 +888,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { S3ClientFactory.S3ClientCreationParameters parameters = null; parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(credentials) + .withPath(name) .withEndpoint(endpoint) .withMetrics(statisticsContext.newStatisticsFromAwsSdk()) .withPathStyleAccess(conf.getBoolean(PATH_STYLE_ACCESS, false)) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 34674c788901f..78617906b2d64 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -115,6 +115,12 @@ final class S3ClientCreationParameters { */ private String userAgentSuffix = ""; + /** + * S3A path. + * added in HADOOP-18330 + */ + private URI pathUri; + /** * List of request handlers to include in the chain * of request execution in the SDK. @@ -168,7 +174,7 @@ public S3ClientCreationParameters withMetrics( } /** - * Set requester pays option. + * Requester pays option. Not yet wired up. * @param value new value * @return the builder */ @@ -264,5 +270,26 @@ public S3ClientCreationParameters withHeader( public Map getHeaders() { return headers; } + + /** + * Get the full s3 path. + * added in HADOOP-18330 + * @return path URI + */ + public URI getPath() { + return pathUri; + } + + /** + * Set full s3a path. + * added in HADOOP-18330 + * @param value new value + * @return the builder + */ + public S3ClientCreationParameters withPath( + final URI value) { + pathUri = value; + return this; + } } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java index 761dd558063ba..db19a6bb85806 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java @@ -176,6 +176,7 @@ private AmazonS3 createS3Client(Configuration conf, S3ClientFactory.S3ClientCreationParameters parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(new AnonymousAWSCredentialsProvider()) + .withPath(new URI("s3a://localhost/")) .withEndpoint(endpoint) .withMetrics(new EmptyS3AStatisticsContext() .newStatisticsFromAwsSdk()); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java index bc223bad457c1..dfa1d75d92e12 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java @@ -589,6 +589,7 @@ protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS) S3ClientFactory.S3ClientCreationParameters parameters = null; parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(testingCreds) + .withPath(new URI("s3a://localhost/")) .withEndpoint(DEFAULT_ENDPOINT) .withMetrics(new EmptyS3AStatisticsContext() .newStatisticsFromAwsSdk()) From 0da1d072df2d7f52d259eeb60d4500b7b576b4d2 Mon Sep 17 00:00:00 2001 From: Ashutosh Date: Sat, 16 Jul 2022 16:52:19 -0400 Subject: [PATCH 2/4] revert overwritten javadoc --- .../main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 78617906b2d64..7786a551cb183 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -174,7 +174,7 @@ public S3ClientCreationParameters withMetrics( } /** - * Requester pays option. Not yet wired up. + * Set requester pays option. * @param value new value * @return the builder */ @@ -288,7 +288,7 @@ public URI getPath() { */ public S3ClientCreationParameters withPath( final URI value) { - pathUri = value; + pathUri = value;C return this; } } From 19ee4f51782c0cd33de1ec1715613ecd5744ea25 Mon Sep 17 00:00:00 2001 From: Ashutosh Date: Tue, 19 Jul 2022 09:42:56 -0400 Subject: [PATCH 3/4] delete typo --- .../src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 7786a551cb183..9b09eba220405 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -288,7 +288,7 @@ public URI getPath() { */ public S3ClientCreationParameters withPath( final URI value) { - pathUri = value;C + pathUri = value; return this; } } From 1de79b6766c59f26890a2968c83a7e0ad85cf9e4 Mon Sep 17 00:00:00 2001 From: Ashutosh Date: Wed, 20 Jul 2022 12:50:45 -0400 Subject: [PATCH 4/4] update getter and setter for consistency --- .../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +- .../main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java | 4 ++-- .../java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java | 2 +- .../auth/delegation/ITestSessionDelegationInFileystem.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 032b482d6f6b1..406f40bff954b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -888,7 +888,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { S3ClientFactory.S3ClientCreationParameters parameters = null; parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(credentials) - .withPath(name) + .withPathUri(name) .withEndpoint(endpoint) .withMetrics(statisticsContext.newStatisticsFromAwsSdk()) .withPathStyleAccess(conf.getBoolean(PATH_STYLE_ACCESS, false)) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 9b09eba220405..2216599679b2a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -276,7 +276,7 @@ public Map getHeaders() { * added in HADOOP-18330 * @return path URI */ - public URI getPath() { + public URI getPathUri() { return pathUri; } @@ -286,7 +286,7 @@ public URI getPath() { * @param value new value * @return the builder */ - public S3ClientCreationParameters withPath( + public S3ClientCreationParameters withPathUri( final URI value) { pathUri = value; return this; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java index db19a6bb85806..40217484f2713 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java @@ -176,7 +176,7 @@ private AmazonS3 createS3Client(Configuration conf, S3ClientFactory.S3ClientCreationParameters parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(new AnonymousAWSCredentialsProvider()) - .withPath(new URI("s3a://localhost/")) + .withPathUri(new URI("s3a://localhost/")) .withEndpoint(endpoint) .withMetrics(new EmptyS3AStatisticsContext() .newStatisticsFromAwsSdk()); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java index dfa1d75d92e12..03b7a0b07aedb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java @@ -589,7 +589,7 @@ protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS) S3ClientFactory.S3ClientCreationParameters parameters = null; parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(testingCreds) - .withPath(new URI("s3a://localhost/")) + .withPathUri(new URI("s3a://localhost/")) .withEndpoint(DEFAULT_ENDPOINT) .withMetrics(new EmptyS3AStatisticsContext() .newStatisticsFromAwsSdk())