From 74d5e3f853108277018667c79be60fa647e1156d Mon Sep 17 00:00:00 2001 From: ashutoshpant <51799223+ashutoshpant@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:42:50 -0400 Subject: [PATCH 1/3] Created a new parameter object that holds the s3a path --- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 1 + .../apache/hadoop/fs/s3a/S3ClientFactory.java | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 5219b04f1797d..86b9319ca1ad1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -888,6 +888,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { S3ClientFactory.S3ClientCreationParameters parameters = null; parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(credentials) + .withPath(name) .withEndpoint(endpoint) .withMetrics(statisticsContext.newStatisticsFromAwsSdk()) .withPathStyleAccess(conf.getBoolean(PATH_STYLE_ACCESS, false)) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 5ef99ed6f5c3c..cdaabf226f96c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -115,6 +115,11 @@ final class S3ClientCreationParameters { */ private String userAgentSuffix = ""; + /** + * S3A path. + */ + private URI pathUri; + /** * List of request handlers to include in the chain * of request execution in the SDK. @@ -264,5 +269,20 @@ public S3ClientCreationParameters withHeader( public Map getHeaders() { return headers; } + + public URI getPath() { + return pathUri; + } + + /** + * Set full s3a path. + * @param value new value + * @return the builder + */ + public S3ClientCreationParameters withPath( + final URI value) { + pathUri = value; + return this; + } } } From 936afc6d00bd2c0d947cf8cbf4dea717c9714421 Mon Sep 17 00:00:00 2001 From: ashutoshpant <51799223+ashutoshpant@users.noreply.github.com> Date: Wed, 13 Jul 2022 12:37:09 -0400 Subject: [PATCH 2/3] Added javadoc and updated test with the new parameter --- .../main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java | 5 +++++ .../org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java | 1 + .../auth/delegation/ITestSessionDelegationInFileystem.java | 1 + 3 files changed, 7 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index cdaabf226f96c..0b01ec3300742 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -270,6 +270,11 @@ public Map getHeaders() { return headers; } + /** + * Get the full s3 path. + * added in HADOOP-18330 + * @return path URI + */ public URI getPath() { return pathUri; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java index 761dd558063ba..db19a6bb85806 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java @@ -176,6 +176,7 @@ private AmazonS3 createS3Client(Configuration conf, S3ClientFactory.S3ClientCreationParameters parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(new AnonymousAWSCredentialsProvider()) + .withPath(new URI("s3a://localhost/")) .withEndpoint(endpoint) .withMetrics(new EmptyS3AStatisticsContext() .newStatisticsFromAwsSdk()); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java index 47fad292156b0..767848f65c7fb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java @@ -590,6 +590,7 @@ protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS) S3ClientFactory.S3ClientCreationParameters parameters = null; parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(testingCreds) + .withPath(new URI("s3a://localhost/")) .withEndpoint(DEFAULT_ENDPOINT) .withMetrics(new EmptyS3AStatisticsContext() .newStatisticsFromAwsSdk()) From cc79f799b1297c688fab959a6bc68de0f8350499 Mon Sep 17 00:00:00 2001 From: ashutoshpant <51799223+ashutoshpant@users.noreply.github.com> Date: Wed, 13 Jul 2022 12:39:18 -0400 Subject: [PATCH 3/3] update javadoc to reflect that "path" is a new addition --- .../src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 0b01ec3300742..6ff239c6d826f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -117,6 +117,7 @@ final class S3ClientCreationParameters { /** * S3A path. + * added in HADOOP-18330 */ private URI pathUri; @@ -281,6 +282,7 @@ public URI getPath() { /** * Set full s3a path. + * added in HADOOP-18330 * @param value new value * @return the builder */