, ClientT> void configureEndpointAndRegion(
+ BuilderT builder, S3ClientCreationParameters parameters, Configuration conf) {
+ URI endpoint = getS3Endpoint(parameters.getEndpoint(), conf);
+
+ String configuredRegion = parameters.getRegion();
+ Region region = null;
+ String origin = "";
+
+ // If the region was configured, set it.
+ if (configuredRegion != null && !configuredRegion.isEmpty()) {
+ origin = AWS_REGION;
+ region = Region.of(configuredRegion);
}
- }
- /**
- * Configure classic S3 client.
- *
- * This includes: endpoint, Path Access and possibly other
- * options.
- *
- * @param s3 S3 Client.
- * @param endPoint s3 endpoint, may be empty
- * @param pathStyleAccess enable path style access?
- * @return S3 client
- * @throws IllegalArgumentException if misconfigured
- */
- protected static AmazonS3 configureAmazonS3Client(AmazonS3 s3,
- final String endPoint,
- final boolean pathStyleAccess)
- throws IllegalArgumentException {
- if (!endPoint.isEmpty()) {
- try {
- s3.setEndpoint(endPoint);
- } catch (IllegalArgumentException e) {
- String msg = "Incorrect endpoint: " + e.getMessage();
- LOG.error(msg);
- throw new IllegalArgumentException(msg, e);
+ if (endpoint != null) {
+ builder.endpointOverride(endpoint);
+ // No region was configured, try to determine it from the endpoint.
+ if (region == null) {
+ region = getS3RegionFromEndpoint(parameters.getEndpoint());
+ if (region != null) {
+ origin = "endpoint";
+ }
}
+ LOG.debug("Setting endpoint to {}", endpoint);
}
- if (pathStyleAccess) {
- LOG.debug("Enabling path style access!");
- s3.setS3ClientOptions(S3ClientOptions.builder()
- .setPathStyleAccess(true)
- .build());
+
+ if (region != null) {
+ builder.region(region);
+ } else if (configuredRegion == null) {
+ // no region is configured, and none could be determined from the endpoint.
+ // Use US_EAST_2 as default.
+ region = Region.of(AWS_S3_DEFAULT_REGION);
+ builder.crossRegionAccessEnabled(true);
+ builder.region(region);
+ origin = "cross region access fallback";
+ } else if (configuredRegion.isEmpty()) {
+ // region configuration was set to empty string.
+ // allow this if people really want it; it is OK to rely on this
+ // when deployed in EC2.
+ WARN_OF_DEFAULT_REGION_CHAIN.warn(SDK_REGION_CHAIN_IN_USE);
+ LOG.debug(SDK_REGION_CHAIN_IN_USE);
+ origin = "SDK region chain";
}
- return s3;
+
+ LOG.debug("Setting region to {} from {}", region, origin);
}
/**
- * Given an endpoint string, return an endpoint config, or null, if none
- * is needed.
- *
- * This is a pretty painful piece of code. It is trying to replicate
- * what AwsClient.setEndpoint() does, because you can't
- * call that setter on an AwsClient constructed via
- * the builder, and you can't pass a metrics collector
- * down except through the builder.
- *
- * Note also that AWS signing is a mystery which nobody fully
- * understands, especially given all problems surface in a
- * "400 bad request" response, which, like all security systems,
- * provides minimal diagnostics out of fear of leaking
- * secrets.
+ * Given a endpoint string, create the endpoint URI.
*
* @param endpoint possibly null endpoint.
- * @param awsConf config to build the URI from.
- * @param awsRegion AWS S3 Region if the corresponding config is set.
- * @return a configuration for the S3 client builder.
+ * @param conf config to build the URI from.
+ * @return an endpoint uri
*/
- @VisibleForTesting
- public static AwsClientBuilder.EndpointConfiguration
- createEndpointConfiguration(
- final String endpoint, final ClientConfiguration awsConf,
- String awsRegion) {
- LOG.debug("Creating endpoint configuration for \"{}\"", endpoint);
+ private static URI getS3Endpoint(String endpoint, final Configuration conf) {
+
+ boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS);
+
+ String protocol = secureConnections ? "https" : "http";
+
if (endpoint == null || endpoint.isEmpty()) {
- // the default endpoint...we should be using null at this point.
- LOG.debug("Using default endpoint -no need to generate a configuration");
+ // don't set an endpoint if none is configured, instead let the SDK figure it out.
return null;
}
- final URI epr = RuntimeHttpUtils.toUri(endpoint, awsConf);
- LOG.debug("Endpoint URI = {}", epr);
- String region = awsRegion;
- if (StringUtils.isBlank(region)) {
- if (!ServiceUtils.isS3USStandardEndpoint(endpoint)) {
- LOG.debug("Endpoint {} is not the default; parsing", epr);
- region = AwsHostNameUtils.parseRegion(
- epr.getHost(),
- S3_SERVICE_NAME);
- } else {
- // US-east, set region == null.
- LOG.debug("Endpoint {} is the standard one; declare region as null",
- epr);
- region = null;
- }
+ if (!endpoint.contains("://")) {
+ endpoint = String.format("%s://%s", protocol, endpoint);
+ }
+
+ try {
+ return new URI(endpoint);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(e);
}
- LOG.debug("Region for endpoint {}, URI {} is determined as {}",
- endpoint, epr, region);
- return new AwsClientBuilder.EndpointConfiguration(endpoint, region);
}
+
+ /**
+ * Parses the endpoint to get the region.
+ * If endpoint is the central one, use US_EAST_1.
+ *
+ * @param endpoint the configure endpoint.
+ * @return the S3 region, null if unable to resolve from endpoint.
+ */
+ private static Region getS3RegionFromEndpoint(String endpoint) {
+
+ if(!endpoint.endsWith(CENTRAL_ENDPOINT)) {
+ LOG.debug("Endpoint {} is not the default; parsing", endpoint);
+ return AwsHostNameUtils.parseSigningRegion(endpoint, S3_SERVICE_NAME).orElse(null);
+ }
+
+ // endpoint is for US_EAST_1;
+ return Region.US_EAST_1;
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java
index f3e00cc61d762..f5c9c43a49cb1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java
@@ -36,7 +36,7 @@ public class FailureInjectionPolicy {
public static final String DEFAULT_DELAY_KEY_SUBSTRING = "DELAY_LISTING_ME";
private static final Logger LOG =
- LoggerFactory.getLogger(InconsistentAmazonS3Client.class);
+ LoggerFactory.getLogger(FailureInjectionPolicy.class);
/**
* Probability of throttling a request.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
deleted file mode 100644
index 317f2c7857630..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.SdkClientException;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsResult;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.ListObjectsV2Result;
-import com.amazonaws.services.s3.model.MultipartUploadListing;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * A wrapper around {@link com.amazonaws.services.s3.AmazonS3} that injects
- * failures.
- * It used to also inject inconsistency, but this was removed with S3Guard;
- * what is retained is the ability to throttle AWS operations and for the
- * input stream to be inconsistent.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class InconsistentAmazonS3Client extends AmazonS3Client {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(InconsistentAmazonS3Client.class);
-
- private FailureInjectionPolicy policy;
-
- /**
- * Counter of failures since last reset.
- */
- private final AtomicLong failureCounter = new AtomicLong(0);
-
-
- /**
- * Instantiate.
- * This subclasses a deprecated constructor of the parent
- * {@code AmazonS3Client} class; we can't use the builder API because,
- * that only creates the consistent client.
- * @param credentials credentials to auth.
- * @param clientConfiguration connection settings
- * @param conf hadoop configuration.
- */
- @SuppressWarnings("deprecation")
- public InconsistentAmazonS3Client(AWSCredentialsProvider credentials,
- ClientConfiguration clientConfiguration, Configuration conf) {
- super(credentials, clientConfiguration);
- policy = new FailureInjectionPolicy(conf);
- }
-
- /**
- * A way for tests to patch in a different fault injection policy at runtime.
- * @param fs filesystem under test
- * @param policy failure injection settings to set
- * @throws Exception on failure
- */
- public static void setFailureInjectionPolicy(S3AFileSystem fs,
- FailureInjectionPolicy policy) throws Exception {
- AmazonS3 s3 = fs.getAmazonS3ClientForTesting("s3guard");
- InconsistentAmazonS3Client ic = InconsistentAmazonS3Client.castFrom(s3);
- ic.replacePolicy(policy);
- }
-
- private void replacePolicy(FailureInjectionPolicy pol) {
- this.policy = pol;
- }
-
- @Override
- public String toString() {
- return String.format("Inconsistent S3 Client: %s; failure count %d",
- policy, failureCounter.get());
- }
-
- /**
- * Convenience function for test code to cast from supertype.
- * @param c supertype to cast from
- * @return subtype, not null
- * @throws Exception on error
- */
- public static InconsistentAmazonS3Client castFrom(AmazonS3 c) throws
- Exception {
- InconsistentAmazonS3Client ic = null;
- if (c instanceof InconsistentAmazonS3Client) {
- ic = (InconsistentAmazonS3Client) c;
- }
- Preconditions.checkNotNull(ic, "Not an instance of " +
- "InconsistentAmazonS3Client");
- return ic;
- }
-
- @Override
- public DeleteObjectsResult deleteObjects(DeleteObjectsRequest
- deleteObjectsRequest)
- throws AmazonClientException, AmazonServiceException {
- maybeFail();
- return super.deleteObjects(deleteObjectsRequest);
- }
-
- @Override
- public void deleteObject(DeleteObjectRequest deleteObjectRequest)
- throws AmazonClientException, AmazonServiceException {
- String key = deleteObjectRequest.getKey();
- LOG.debug("key {}", key);
- maybeFail();
- super.deleteObject(deleteObjectRequest);
- }
-
- /* We should only need to override this version of putObject() */
- @Override
- public PutObjectResult putObject(PutObjectRequest putObjectRequest)
- throws AmazonClientException, AmazonServiceException {
- LOG.debug("key {}", putObjectRequest.getKey());
- maybeFail();
- return super.putObject(putObjectRequest);
- }
-
- /* We should only need to override these versions of listObjects() */
- @Override
- public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
- throws AmazonClientException, AmazonServiceException {
- maybeFail();
- return super.listObjects(listObjectsRequest);
- }
-
- /* consistent listing with possibility of failing. */
- @Override
- public ListObjectsV2Result listObjectsV2(ListObjectsV2Request request)
- throws AmazonClientException, AmazonServiceException {
- maybeFail();
- return super.listObjectsV2(request);
- }
-
-
- @Override
- public CompleteMultipartUploadResult completeMultipartUpload(
- CompleteMultipartUploadRequest completeMultipartUploadRequest)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.completeMultipartUpload(completeMultipartUploadRequest);
- }
-
- @Override
- public UploadPartResult uploadPart(UploadPartRequest uploadPartRequest)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.uploadPart(uploadPartRequest);
- }
-
- @Override
- public InitiateMultipartUploadResult initiateMultipartUpload(
- InitiateMultipartUploadRequest initiateMultipartUploadRequest)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.initiateMultipartUpload(initiateMultipartUploadRequest);
- }
-
- @Override
- public MultipartUploadListing listMultipartUploads(
- ListMultipartUploadsRequest listMultipartUploadsRequest)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.listMultipartUploads(listMultipartUploadsRequest);
- }
-
- /**
- * Set the probability of throttling a request.
- * @param throttleProbability the probability of a request being throttled.
- */
- public void setThrottleProbability(float throttleProbability) {
- policy.setThrottleProbability(throttleProbability);
- }
-
- /**
- * Conditionally fail the operation.
- * @param errorMsg description of failure
- * @param statusCode http status code for error
- * @throws AmazonClientException if the client chooses to fail
- * the request.
- */
- private void maybeFail(String errorMsg, int statusCode)
- throws AmazonClientException {
- // code structure here is to line up for more failures later
- AmazonServiceException ex = null;
- if (FailureInjectionPolicy.trueWithProbability(policy.getThrottleProbability())) {
- // throttle the request
- ex = new AmazonServiceException(errorMsg
- + " count = " + (failureCounter.get() + 1), null);
- ex.setStatusCode(statusCode);
- }
-
- int failureLimit = policy.getFailureLimit();
- if (ex != null) {
- long count = failureCounter.incrementAndGet();
- if (failureLimit == 0
- || (failureLimit > 0 && count < failureLimit)) {
- throw ex;
- }
- }
- }
-
- private void maybeFail() {
- maybeFail("throttled", 503);
- }
-
- /**
- * Set the limit on failures before all operations pass through.
- * This resets the failure count.
- * @param limit limit; "0" means "no limit"
- */
- public void setFailureLimit(int limit) {
- policy.setFailureLimit(limit);
- failureCounter.set(0);
- }
-
- @Override
- public S3Object getObject(GetObjectRequest var1) throws SdkClientException,
- AmazonServiceException {
- maybeFail();
- return super.getObject(var1);
- }
-
- @Override
- public S3Object getObject(String bucketName, String key)
- throws SdkClientException, AmazonServiceException {
- maybeFail();
- return super.getObject(bucketName, key);
-
- }
-
- /** Since ObjectListing is immutable, we just override it with wrapper. */
- @SuppressWarnings("serial")
- private static class CustomObjectListing extends ObjectListing {
-
- private final List customListing;
- private final List customPrefixes;
-
- CustomObjectListing(ObjectListing rawListing,
- List customListing,
- List customPrefixes) {
- super();
- this.customListing = customListing;
- this.customPrefixes = customPrefixes;
-
- this.setBucketName(rawListing.getBucketName());
- this.setCommonPrefixes(rawListing.getCommonPrefixes());
- this.setDelimiter(rawListing.getDelimiter());
- this.setEncodingType(rawListing.getEncodingType());
- this.setMarker(rawListing.getMarker());
- this.setMaxKeys(rawListing.getMaxKeys());
- this.setNextMarker(rawListing.getNextMarker());
- this.setPrefix(rawListing.getPrefix());
- this.setTruncated(rawListing.isTruncated());
- }
-
- @Override
- public List getObjectSummaries() {
- return customListing;
- }
-
- @Override
- public List getCommonPrefixes() {
- return customPrefixes;
- }
- }
-
- @SuppressWarnings("serial")
- private static class CustomListObjectsV2Result extends ListObjectsV2Result {
-
- private final List customListing;
- private final List customPrefixes;
-
- CustomListObjectsV2Result(ListObjectsV2Result raw,
- List customListing, List customPrefixes) {
- super();
- this.customListing = customListing;
- this.customPrefixes = customPrefixes;
-
- this.setBucketName(raw.getBucketName());
- this.setCommonPrefixes(raw.getCommonPrefixes());
- this.setDelimiter(raw.getDelimiter());
- this.setEncodingType(raw.getEncodingType());
- this.setStartAfter(raw.getStartAfter());
- this.setMaxKeys(raw.getMaxKeys());
- this.setContinuationToken(raw.getContinuationToken());
- this.setPrefix(raw.getPrefix());
- this.setTruncated(raw.isTruncated());
- }
-
- @Override
- public List getObjectSummaries() {
- return customListing;
- }
-
- @Override
- public List getCommonPrefixes() {
- return customPrefixes;
- }
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java
deleted file mode 100644
index 4bfcc8aba3af3..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.services.s3.AmazonS3;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * S3 Client factory used for testing with eventual consistency fault injection.
- * This client is for testing only; it is in the production
- * {@code hadoop-aws} module to enable integration tests to use this
- * just by editing the Hadoop configuration used to bring up the client.
- *
- * The factory uses the older constructor-based instantiation/configuration
- * of the client, so does not wire up metrics, handlers etc.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class InconsistentS3ClientFactory extends DefaultS3ClientFactory {
-
- @Override
- protected AmazonS3 buildAmazonS3Client(
- final ClientConfiguration awsConf,
- final S3ClientCreationParameters parameters) {
- LOG.warn("** FAILURE INJECTION ENABLED. Do not run in production! **");
- LOG.warn("List inconsistency is no longer emulated; only throttling and read errors");
- InconsistentAmazonS3Client s3
- = new InconsistentAmazonS3Client(
- parameters.getCredentialSet(), awsConf, getConf());
- configureAmazonS3Client(s3,
- parameters.getEndpoint(),
- parameters.isPathStyleAccess());
- return s3;
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
index 67c8e7d809cc8..9b2c95a90c76f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
@@ -24,8 +24,7 @@
import java.util.concurrent.Future;
import javax.annotation.Nullable;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.SdkBaseException;
+import software.amazon.awssdk.core.exception.SdkException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,6 +38,7 @@
import org.apache.hadoop.util.functional.InvocationRaisingIOE;
import org.apache.hadoop.util.Preconditions;
+
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration;
/**
@@ -120,7 +120,7 @@ public static T once(String action, String path,
throws IOException {
try (DurationInfo ignored = new DurationInfo(LOG, false, "%s", action)) {
return operation.apply();
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw S3AUtils.translateException(action, path, e);
}
}
@@ -145,7 +145,7 @@ public static T onceTrackingDuration(
throws IOException {
try {
return invokeTrackingDuration(tracker, operation);
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw S3AUtils.translateException(action, path, e);
}
}
@@ -170,7 +170,7 @@ public static void once(String action, String path,
/**
*
- * Wait for a future, translating AmazonClientException into an IOException.
+ * Wait for a future, translating SdkException into an IOException.
* @param action action to execute (used in error messages)
* @param path path of work (used in error messages)
* @param future future to await for
@@ -186,7 +186,7 @@ public static T onceInTheFuture(String action,
throws IOException {
try (DurationInfo ignored = new DurationInfo(LOG, false, "%s", action)) {
return FutureIO.awaitFuture(future);
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw S3AUtils.translateException(action, path, e);
}
}
@@ -444,7 +444,7 @@ public T retryUntranslated(
* @param operation operation to execute
* @return the result of the call
* @throws IOException any IOE raised
- * @throws SdkBaseException any AWS exception raised
+ * @throws SdkException any AWS exception raised
* @throws RuntimeException : these are never caught and retries.
*/
@Retries.RetryRaw
@@ -466,7 +466,7 @@ public T retryUntranslated(
}
// execute the operation, returning if successful
return operation.apply();
- } catch (IOException | SdkBaseException e) {
+ } catch (IOException | SdkException e) {
caught = e;
}
// you only get here if the operation didn't complete
@@ -479,7 +479,7 @@ public T retryUntranslated(
translated = (IOException) caught;
} else {
translated = S3AUtils.translateException(text, "",
- (SdkBaseException)caught);
+ (SdkException) caught);
}
try {
@@ -518,11 +518,10 @@ public T retryUntranslated(
if (caught instanceof IOException) {
throw (IOException) caught;
} else {
- throw (SdkBaseException) caught;
+ throw (SdkException) caught;
}
}
-
/**
* Execute an operation; any exception raised is simply caught and
* logged at debug.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
index 6c39cc4b64240..490deaaab04d9 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
+import software.amazon.awssdk.services.s3.model.CommonPrefix;
+import software.amazon.awssdk.services.s3.model.S3Object;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.VisibleForTesting;
@@ -277,19 +278,19 @@ public S3ListRequest createListObjectsRequest(String key,
}
/**
- * Interface to implement by the logic deciding whether to accept a summary
+ * Interface to implement the logic deciding whether to accept a s3Object
* entry or path as a valid file or directory.
*/
interface FileStatusAcceptor {
/**
- * Predicate to decide whether or not to accept a summary entry.
+ * Predicate to decide whether or not to accept a s3Object entry.
* @param keyPath qualified path to the entry
- * @param summary summary entry
+ * @param s3Object s3Object entry
* @return true if the entry is accepted (i.e. that a status entry
* should be generated.
*/
- boolean accept(Path keyPath, S3ObjectSummary summary);
+ boolean accept(Path keyPath, S3Object s3Object);
/**
* Predicate to decide whether or not to accept a prefix.
@@ -451,21 +452,21 @@ private boolean buildNextStatusBatch(S3ListResult objects) {
int added = 0, ignored = 0;
// list to fill in with results. Initial size will be list maximum.
List stats = new ArrayList<>(
- objects.getObjectSummaries().size() +
+ objects.getS3Objects().size() +
objects.getCommonPrefixes().size());
// objects
- for (S3ObjectSummary summary : objects.getObjectSummaries()) {
- String key = summary.getKey();
+ for (S3Object s3Object : objects.getS3Objects()) {
+ String key = s3Object.key();
Path keyPath = getStoreContext().getContextAccessors().keyToPath(key);
if (LOG.isDebugEnabled()) {
- LOG.debug("{}: {}", keyPath, stringify(summary));
+ LOG.debug("{}: {}", keyPath, stringify(s3Object));
}
// Skip over keys that are ourselves and old S3N _$folder$ files
- if (acceptor.accept(keyPath, summary) && filter.accept(keyPath)) {
- S3AFileStatus status = createFileStatus(keyPath, summary,
+ if (acceptor.accept(keyPath, s3Object) && filter.accept(keyPath)) {
+ S3AFileStatus status = createFileStatus(keyPath, s3Object,
listingOperationCallbacks.getDefaultBlockSize(keyPath),
getStoreContext().getUsername(),
- summary.getETag(), null, isCSEEnabled);
+ s3Object.eTag(), null, isCSEEnabled);
LOG.debug("Adding: {}", status);
stats.add(status);
added++;
@@ -476,11 +477,11 @@ private boolean buildNextStatusBatch(S3ListResult objects) {
}
// prefixes: always directories
- for (String prefix : objects.getCommonPrefixes()) {
+ for (CommonPrefix prefix : objects.getCommonPrefixes()) {
Path keyPath = getStoreContext()
.getContextAccessors()
- .keyToPath(prefix);
- if (acceptor.accept(keyPath, prefix) && filter.accept(keyPath)) {
+ .keyToPath(prefix.prefix());
+ if (acceptor.accept(keyPath, prefix.prefix()) && filter.accept(keyPath)) {
S3AFileStatus status = new S3AFileStatus(Tristate.FALSE, keyPath,
getStoreContext().getUsername());
LOG.debug("Adding directory: {}", status);
@@ -731,18 +732,18 @@ public AcceptFilesOnly(Path qualifiedPath) {
}
/**
- * Reject a summary entry if the key path is the qualified Path, or
+ * Reject a s3Object entry if the key path is the qualified Path, or
* it ends with {@code "_$folder$"}.
* @param keyPath key path of the entry
- * @param summary summary entry
+ * @param s3Object s3Object entry
* @return true if the entry is accepted (i.e. that a status entry
* should be generated.
*/
@Override
- public boolean accept(Path keyPath, S3ObjectSummary summary) {
+ public boolean accept(Path keyPath, S3Object s3Object) {
return !keyPath.equals(qualifiedPath)
- && !summary.getKey().endsWith(S3N_FOLDER_SUFFIX)
- && !objectRepresentsDirectory(summary.getKey());
+ && !s3Object.key().endsWith(S3N_FOLDER_SUFFIX)
+ && !objectRepresentsDirectory(s3Object.key());
}
/**
@@ -767,8 +768,8 @@ public boolean accept(FileStatus status) {
*/
static class AcceptAllButS3nDirs implements FileStatusAcceptor {
- public boolean accept(Path keyPath, S3ObjectSummary summary) {
- return !summary.getKey().endsWith(S3N_FOLDER_SUFFIX);
+ public boolean accept(Path keyPath, S3Object s3Object) {
+ return !s3Object.key().endsWith(S3N_FOLDER_SUFFIX);
}
public boolean accept(Path keyPath, String prefix) {
@@ -799,17 +800,17 @@ public AcceptAllButSelfAndS3nDirs(Path qualifiedPath) {
}
/**
- * Reject a summary entry if the key path is the qualified Path, or
+ * Reject a s3Object entry if the key path is the qualified Path, or
* it ends with {@code "_$folder$"}.
* @param keyPath key path of the entry
- * @param summary summary entry
+ * @param s3Object s3Object entry
* @return true if the entry is accepted (i.e. that a status entry
* should be generated.)
*/
@Override
- public boolean accept(Path keyPath, S3ObjectSummary summary) {
+ public boolean accept(Path keyPath, S3Object s3Object) {
return !keyPath.equals(qualifiedPath) &&
- !summary.getKey().endsWith(S3N_FOLDER_SUFFIX);
+ !s3Object.key().endsWith(S3N_FOLDER_SUFFIX);
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
index d8c820cd8a121..b2057c211da7b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java
@@ -23,19 +23,20 @@
import java.util.NoSuchElementException;
import javax.annotation.Nullable;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.MultipartUploadListing;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsResponse;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.s3a.api.RequestFactory;
import org.apache.hadoop.fs.s3a.impl.StoreContext;
import org.apache.hadoop.fs.store.audit.AuditSpan;
-import static org.apache.hadoop.fs.s3a.Statistic.MULTIPART_UPLOAD_LIST;
+import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_MULTIPART_UPLOAD_LIST;
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfOperation;
@@ -43,7 +44,7 @@
* MultipartUtils upload-specific functions for use by S3AFileSystem and Hadoop
* CLI.
* The Audit span active when
- * {@link #listMultipartUploads(StoreContext, AmazonS3, String, int)}
+ * {@link #listMultipartUploads(StoreContext, S3Client, String, int)}
* was invoked is retained for all subsequent operations.
*/
public final class MultipartUtils {
@@ -65,9 +66,9 @@ private MultipartUtils() { }
* @param maxKeys maximum batch size to request at a time from S3.
* @return an iterator of matching uploads
*/
- static MultipartUtils.UploadIterator listMultipartUploads(
+ static RemoteIterator listMultipartUploads(
final StoreContext storeContext,
- AmazonS3 s3,
+ S3Client s3,
@Nullable String prefix,
int maxKeys)
throws IOException {
@@ -84,14 +85,14 @@ static MultipartUtils.UploadIterator listMultipartUploads(
* at the time the iterator was constructed.
*/
static class ListingIterator implements
- RemoteIterator {
+ RemoteIterator {
private final String prefix;
private final RequestFactory requestFactory;
private final int maxKeys;
- private final AmazonS3 s3;
+ private final S3Client s3;
private final Invoker invoker;
private final AuditSpan auditSpan;
@@ -101,7 +102,7 @@ static class ListingIterator implements
/**
* Most recent listing results.
*/
- private MultipartUploadListing listing;
+ private ListMultipartUploadsResponse listing;
/**
* Indicator that this is the first listing.
@@ -114,7 +115,7 @@ static class ListingIterator implements
private int listCount = 0;
ListingIterator(final StoreContext storeContext,
- AmazonS3 s3,
+ S3Client s3,
@Nullable String prefix,
int maxKeys) throws IOException {
this.storeContext = storeContext;
@@ -153,7 +154,7 @@ public boolean hasNext() throws IOException {
*/
@Override
@Retries.RetryTranslated
- public MultipartUploadListing next() throws IOException {
+ public ListMultipartUploadsResponse next() throws IOException {
if (firstListing) {
firstListing = false;
} else {
@@ -171,32 +172,34 @@ public MultipartUploadListing next() throws IOException {
public String toString() {
return "Upload iterator: prefix " + prefix
+ "; list count " + listCount
- + "; upload count " + listing.getMultipartUploads().size()
+ + "; upload count " + listing.uploads().size()
+ "; isTruncated=" + listing.isTruncated();
}
@Retries.RetryTranslated
private void requestNextBatch() throws IOException {
try (AuditSpan span = auditSpan.activate()) {
- ListMultipartUploadsRequest req = requestFactory
- .newListMultipartUploadsRequest(prefix);
+ ListMultipartUploadsRequest.Builder requestBuilder = requestFactory
+ .newListMultipartUploadsRequestBuilder(prefix);
if (!firstListing) {
- req.setKeyMarker(listing.getNextKeyMarker());
- req.setUploadIdMarker(listing.getNextUploadIdMarker());
+ requestBuilder.keyMarker(listing.nextKeyMarker());
+ requestBuilder.uploadIdMarker(listing.nextUploadIdMarker());
}
- req.setMaxUploads(maxKeys);
+ requestBuilder.maxUploads(maxKeys);
+
+ ListMultipartUploadsRequest request = requestBuilder.build();
LOG.debug("[{}], Requesting next {} uploads prefix {}, " +
"next key {}, next upload id {}", listCount, maxKeys, prefix,
- req.getKeyMarker(), req.getUploadIdMarker());
+ request.keyMarker(), request.uploadIdMarker());
listCount++;
listing = invoker.retry("listMultipartUploads", prefix, true,
trackDurationOfOperation(storeContext.getInstrumentation(),
- MULTIPART_UPLOAD_LIST.getSymbol(),
- () -> s3.listMultipartUploads(req)));
+ OBJECT_MULTIPART_UPLOAD_LIST.getSymbol(),
+ () -> s3.listMultipartUploads(requestBuilder.build())));
LOG.debug("Listing found {} upload(s)",
- listing.getMultipartUploads().size());
+ listing.uploads().size());
LOG.debug("New listing state: {}", this);
}
}
@@ -216,14 +219,22 @@ public static class UploadIterator
*/
private ListingIterator lister;
/** Current listing: the last upload listing we fetched. */
- private MultipartUploadListing listing;
+ private ListMultipartUploadsResponse listing;
/** Iterator over the current listing. */
private ListIterator batchIterator;
+ /**
+ * Construct an iterator to list uploads under a path.
+ * @param storeContext store context
+ * @param s3 s3 client
+ * @param maxKeys max # of keys to list per batch
+ * @param prefix prefix
+ * @throws IOException listing failure.
+ */
@Retries.RetryTranslated
public UploadIterator(
final StoreContext storeContext,
- AmazonS3 s3,
+ S3Client s3,
int maxKeys,
@Nullable String prefix)
throws IOException {
@@ -249,7 +260,7 @@ public MultipartUpload next() throws IOException {
private boolean requestNextBatch() throws IOException {
if (lister.hasNext()) {
listing = lister.next();
- batchIterator = listing.getMultipartUploads().listIterator();
+ batchIterator = listing.uploads().listIterator();
return batchIterator.hasNext();
}
return false;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java
index 0ce022aa88588..7ee6c55c191b7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java
@@ -18,59 +18,55 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.event.ProgressEvent;
-import com.amazonaws.event.ProgressEventType;
-import com.amazonaws.event.ProgressListener;
-import com.amazonaws.services.s3.transfer.Upload;
+import software.amazon.awssdk.transfer.s3.model.ObjectTransfer;
+import software.amazon.awssdk.transfer.s3.progress.TransferListener;
import org.apache.hadoop.util.Progressable;
import org.slf4j.Logger;
-import static com.amazonaws.event.ProgressEventType.TRANSFER_COMPLETED_EVENT;
-import static com.amazonaws.event.ProgressEventType.TRANSFER_PART_STARTED_EVENT;
/**
* Listener to progress from AWS regarding transfers.
*/
-public class ProgressableProgressListener implements ProgressListener {
+public class ProgressableProgressListener implements TransferListener {
private static final Logger LOG = S3AFileSystem.LOG;
private final S3AFileSystem fs;
private final String key;
private final Progressable progress;
private long lastBytesTransferred;
- private final Upload upload;
/**
* Instantiate.
* @param fs filesystem: will be invoked with statistics updates
* @param key key for the upload
- * @param upload source of events
* @param progress optional callback for progress.
*/
public ProgressableProgressListener(S3AFileSystem fs,
String key,
- Upload upload,
Progressable progress) {
this.fs = fs;
this.key = key;
- this.upload = upload;
this.progress = progress;
this.lastBytesTransferred = 0;
}
@Override
- public void progressChanged(ProgressEvent progressEvent) {
- if (progress != null) {
- progress.progress();
- }
+ public void transferInitiated(TransferListener.Context.TransferInitiated context) {
+ fs.incrementWriteOperations();
+ }
+
+ @Override
+ public void transferComplete(TransferListener.Context.TransferComplete context) {
+ fs.incrementWriteOperations();
+ }
+
+ @Override
+ public void bytesTransferred(TransferListener.Context.BytesTransferred context) {
- // There are 3 http ops here, but this should be close enough for now
- ProgressEventType pet = progressEvent.getEventType();
- if (pet == TRANSFER_PART_STARTED_EVENT ||
- pet == TRANSFER_COMPLETED_EVENT) {
- fs.incrementWriteOperations();
+ if(progress != null) {
+ progress.progress();
}
- long transferred = upload.getProgress().getBytesTransferred();
+ long transferred = context.progressSnapshot().transferredBytes();
long delta = transferred - lastBytesTransferred;
fs.incrementPutProgressStatistics(key, delta);
lastBytesTransferred = transferred;
@@ -79,11 +75,13 @@ public void progressChanged(ProgressEvent progressEvent) {
/**
* Method to invoke after upload has completed.
* This can handle race conditions in setup/teardown.
+ * @param upload upload which has just completed.
* @return the number of bytes which were transferred after the notification
*/
- public long uploadCompleted() {
- long delta = upload.getProgress().getBytesTransferred() -
- lastBytesTransferred;
+ public long uploadCompleted(ObjectTransfer upload) {
+
+ long delta =
+ upload.progress().snapshot().transferredBytes() - lastBytesTransferred;
if (delta > 0) {
LOG.debug("S3A write delta changed after finished: {} bytes", delta);
fs.incrementPutProgressStatistics(key, delta);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
index 43a2b7e0dbd5b..de0f59154e995 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
@@ -31,15 +31,16 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.event.ProgressEvent;
-import com.amazonaws.event.ProgressEventType;
-import com.amazonaws.event.ProgressListener;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
+
+import org.apache.hadoop.fs.s3a.impl.ProgressListener;
+import org.apache.hadoop.fs.s3a.impl.ProgressListenerEvent;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
import org.apache.hadoop.fs.statistics.IOStatisticsAggregator;
import org.apache.hadoop.util.Preconditions;
@@ -69,6 +70,7 @@
import static java.util.Objects.requireNonNull;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.Statistic.*;
+import static org.apache.hadoop.fs.s3a.impl.ProgressListenerEvent.*;
import static org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext.EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS;
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration;
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation;
@@ -192,7 +194,7 @@ class S3ABlockOutputStream extends OutputStream implements
this.executorService = MoreExecutors.listeningDecorator(
builder.executorService);
this.multiPartUpload = null;
- final Progressable progress = builder.progress;
+ Progressable progress = builder.progress;
this.progressListener = (progress instanceof ProgressListener) ?
(ProgressListener) progress
: new ProgressableListener(progress);
@@ -439,7 +441,7 @@ public void close() throws IOException {
uploadCurrentBlock(true);
}
// wait for the partial uploads to finish
- final List partETags =
+ final List partETags =
multiPartUpload.waitForAllPartUploads();
bytes = bytesSubmitted;
@@ -597,27 +599,28 @@ private long putObject() throws IOException {
final PutObjectRequest putObjectRequest = uploadData.hasFile() ?
writeOperationHelper.createPutObjectRequest(
key,
- uploadData.getFile(),
- builder.putOptions)
+ uploadData.getFile().length(),
+ builder.putOptions,
+ true)
: writeOperationHelper.createPutObjectRequest(
key,
- uploadData.getUploadStream(),
size,
- builder.putOptions);
- BlockUploadProgress callback =
- new BlockUploadProgress(
- block, progressListener, now());
- putObjectRequest.setGeneralProgressListener(callback);
+ builder.putOptions,
+ false);
+
+ BlockUploadProgress progressCallback =
+ new BlockUploadProgress(block, progressListener, now());
statistics.blockUploadQueued(size);
- ListenableFuture putObjectResult =
+ ListenableFuture putObjectResult =
executorService.submit(() -> {
try {
// the putObject call automatically closes the input
// stream afterwards.
- return writeOperationHelper.putObject(
- putObjectRequest,
- builder.putOptions,
- statistics);
+ PutObjectResponse response =
+ writeOperationHelper.putObject(putObjectRequest, builder.putOptions, uploadData,
+ uploadData.hasFile(), statistics);
+ progressCallback.progressChanged(REQUEST_BYTE_TRANSFER_EVENT);
+ return response;
} finally {
cleanupWithLogger(LOG, uploadData, block);
}
@@ -761,7 +764,7 @@ protected IOStatisticsAggregator getThreadIOStatistics() {
*/
private class MultiPartUpload {
private final String uploadId;
- private final List> partETagsFutures;
+ private final List> partETagsFutures;
private int partsSubmitted;
private int partsUploaded;
private long bytesSubmitted;
@@ -866,18 +869,19 @@ private void uploadBlockAsync(final S3ADataBlocks.DataBlock block,
final int currentPartNumber = partETagsFutures.size() + 1;
final UploadPartRequest request;
final S3ADataBlocks.BlockUploadData uploadData;
+ final RequestBody requestBody;
try {
uploadData = block.startUpload();
- request = writeOperationHelper.newUploadPartRequest(
+ requestBody = uploadData.hasFile()
+ ? RequestBody.fromFile(uploadData.getFile())
+ : RequestBody.fromInputStream(uploadData.getUploadStream(), size);
+
+ request = writeOperationHelper.newUploadPartRequestBuilder(
key,
uploadId,
currentPartNumber,
- size,
- uploadData.getUploadStream(),
- uploadData.getFile(),
- 0L);
- request.setLastPart(isLast);
- } catch (SdkBaseException aws) {
+ size).build();
+ } catch (SdkException aws) {
// catch and translate
IOException e = translateException("upload", key, aws);
// failure to start the upload.
@@ -888,28 +892,38 @@ private void uploadBlockAsync(final S3ADataBlocks.DataBlock block,
noteUploadFailure(e);
throw e;
}
- BlockUploadProgress callback =
- new BlockUploadProgress(
- block, progressListener, now());
- request.setGeneralProgressListener(callback);
+
+ BlockUploadProgress progressCallback =
+ new BlockUploadProgress(block, progressListener, now());
+
statistics.blockUploadQueued(block.dataSize());
- ListenableFuture partETagFuture =
+ ListenableFuture partETagFuture =
executorService.submit(() -> {
// this is the queued upload operation
// do the upload
try {
LOG.debug("Uploading part {} for id '{}'",
currentPartNumber, uploadId);
- PartETag partETag = writeOperationHelper.uploadPart(request, statistics)
- .getPartETag();
+
+ progressCallback.progressChanged(TRANSFER_PART_STARTED_EVENT);
+
+ UploadPartResponse response = writeOperationHelper
+ .uploadPart(request, requestBody, statistics);
LOG.debug("Completed upload of {} to part {}",
- block, partETag.getETag());
+ block, response.eTag());
LOG.debug("Stream statistics of {}", statistics);
partsUploaded++;
- return partETag;
+
+ progressCallback.progressChanged(TRANSFER_PART_COMPLETED_EVENT);
+
+ return CompletedPart.builder()
+ .eTag(response.eTag())
+ .partNumber(currentPartNumber)
+ .build();
} catch (IOException e) {
// save immediately.
noteUploadFailure(e);
+ progressCallback.progressChanged(TRANSFER_PART_FAILED_EVENT);
throw e;
} finally {
// close the stream and block
@@ -924,7 +938,7 @@ private void uploadBlockAsync(final S3ADataBlocks.DataBlock block,
* @return list of results
* @throws IOException IO Problems
*/
- private List waitForAllPartUploads() throws IOException {
+ private List waitForAllPartUploads() throws IOException {
LOG.debug("Waiting for {} uploads to complete", partETagsFutures.size());
try {
return Futures.allAsList(partETagsFutures).get();
@@ -948,7 +962,7 @@ private List waitForAllPartUploads() throws IOException {
*/
private void cancelAllActiveFutures() {
LOG.debug("Cancelling futures");
- for (ListenableFuture future : partETagsFutures) {
+ for (ListenableFuture future : partETagsFutures) {
future.cancel(true);
}
}
@@ -960,7 +974,7 @@ private void cancelAllActiveFutures() {
* @param partETags list of partial uploads
* @throws IOException on any problem
*/
- private void complete(List partETags)
+ private void complete(List partETags)
throws IOException {
maybeRethrowUploadFailure();
AtomicInteger errorCount = new AtomicInteger(0);
@@ -1005,22 +1019,24 @@ private IOException abort() {
}
}
+
/**
* The upload progress listener registered for events returned
* during the upload of a single block.
* It updates statistics and handles the end of the upload.
* Transfer failures are logged at WARN.
*/
- private final class BlockUploadProgress implements ProgressListener {
+ private final class BlockUploadProgress {
+
private final S3ADataBlocks.DataBlock block;
private final ProgressListener nextListener;
private final Instant transferQueueTime;
private Instant transferStartTime;
+ private long size;
/**
* Track the progress of a single block upload.
* @param block block to monitor
- * @param nextListener optional next progress listener
* @param transferQueueTime time the block was transferred
* into the queue
*/
@@ -1029,20 +1045,17 @@ private BlockUploadProgress(S3ADataBlocks.DataBlock block,
Instant transferQueueTime) {
this.block = block;
this.transferQueueTime = transferQueueTime;
+ this.size = block.dataSize();
this.nextListener = nextListener;
}
- @Override
- public void progressChanged(ProgressEvent progressEvent) {
- ProgressEventType eventType = progressEvent.getEventType();
- long bytesTransferred = progressEvent.getBytesTransferred();
+ public void progressChanged(ProgressListenerEvent eventType) {
- long size = block.dataSize();
switch (eventType) {
case REQUEST_BYTE_TRANSFER_EVENT:
// bytes uploaded
- statistics.bytesTransferred(bytesTransferred);
+ statistics.bytesTransferred(size);
break;
case TRANSFER_PART_STARTED_EVENT:
@@ -1057,6 +1070,7 @@ public void progressChanged(ProgressEvent progressEvent) {
statistics.blockUploadCompleted(
Duration.between(transferStartTime, now()),
size);
+ statistics.bytesTransferred(size);
break;
case TRANSFER_PART_FAILED_EVENT:
@@ -1071,13 +1085,13 @@ public void progressChanged(ProgressEvent progressEvent) {
}
if (nextListener != null) {
- nextListener.progressChanged(progressEvent);
+ nextListener.progressChanged(eventType, size);
}
}
}
/**
- * Bridge from AWS {@code ProgressListener} to Hadoop {@link Progressable}.
+ * Bridge from {@link ProgressListener} to Hadoop {@link Progressable}.
*/
private static class ProgressableListener implements ProgressListener {
private final Progressable progress;
@@ -1086,7 +1100,7 @@ private static class ProgressableListener implements ProgressListener {
this.progress = progress;
}
- public void progressChanged(ProgressEvent progressEvent) {
+ public void progressChanged(ProgressListenerEvent eventType, int bytesTransferred) {
if (progress != null) {
progress.progress();
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
index ddaf059b055de..5aa43245c2f01 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
@@ -47,7 +47,7 @@
* Set of classes to support output streaming into blocks which are then
* uploaded as to S3 as a single PUT, or as part of a multipart request.
*/
-final class S3ADataBlocks {
+public final class S3ADataBlocks {
private static final Logger LOG =
LoggerFactory.getLogger(S3ADataBlocks.class);
@@ -101,7 +101,7 @@ static BlockFactory createFactory(S3AFileSystem owner,
* It can be one of a file or an input stream.
* When closed, any stream is closed. Any source file is untouched.
*/
- static final class BlockUploadData implements Closeable {
+ public static final class BlockUploadData implements Closeable {
private final File file;
private final InputStream uploadStream;
@@ -109,7 +109,7 @@ static final class BlockUploadData implements Closeable {
* File constructor; input stream will be null.
* @param file file to upload
*/
- BlockUploadData(File file) {
+ public BlockUploadData(File file) {
Preconditions.checkArgument(file.exists(), "No file: " + file);
this.file = file;
this.uploadStream = null;
@@ -119,7 +119,7 @@ static final class BlockUploadData implements Closeable {
* Stream constructor, file field will be null.
* @param uploadStream stream to upload
*/
- BlockUploadData(InputStream uploadStream) {
+ public BlockUploadData(InputStream uploadStream) {
Preconditions.checkNotNull(uploadStream, "rawUploadStream");
this.uploadStream = uploadStream;
this.file = null;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java
index b599790a1cf76..6cacdff6c42d8 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java
@@ -33,13 +33,14 @@ public enum S3AEncryptionMethods {
SSE_KMS("SSE-KMS", true, false),
SSE_C("SSE-C", true, true),
CSE_KMS("CSE-KMS", false, true),
- CSE_CUSTOM("CSE-CUSTOM", false, true);
+ CSE_CUSTOM("CSE-CUSTOM", false, true),
+ DSSE_KMS("DSSE-KMS", true, false);
/**
* Error string when {@link #getMethod(String)} fails.
* Used in tests.
*/
- static final String UNKNOWN_ALGORITHM
+ public static final String UNKNOWN_ALGORITHM
= "Unknown encryption algorithm ";
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 6d9fd76d1346e..db2cedd37fc90 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -21,12 +21,14 @@
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.file.AccessDeniedException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -41,6 +43,7 @@
import java.util.Objects;
import java.util.TreeSet;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
@@ -48,42 +51,49 @@
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.Headers;
-import com.amazonaws.services.s3.model.CannedAccessControlList;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsResult;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.MultiObjectDeleteException;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.SelectObjectContentResult;
-import com.amazonaws.services.s3.model.StorageClass;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
-import com.amazonaws.services.s3.transfer.Copy;
-import com.amazonaws.services.s3.transfer.TransferManager;
-import com.amazonaws.services.s3.transfer.TransferManagerConfiguration;
-import com.amazonaws.services.s3.transfer.Upload;
-import com.amazonaws.services.s3.transfer.model.CopyResult;
-import com.amazonaws.services.s3.transfer.model.UploadResult;
-import com.amazonaws.event.ProgressListener;
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
+import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.S3Error;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler;
+import software.amazon.awssdk.services.s3.model.StorageClass;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
+import software.amazon.awssdk.transfer.s3.model.CompletedCopy;
+import software.amazon.awssdk.transfer.s3.model.CompletedFileUpload;
+import software.amazon.awssdk.transfer.s3.model.Copy;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
+import software.amazon.awssdk.transfer.s3.model.CopyRequest;
+import software.amazon.awssdk.transfer.s3.model.FileUpload;
+import software.amazon.awssdk.transfer.s3.model.UploadFileRequest;
import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool;
import org.slf4j.Logger;
@@ -108,11 +118,12 @@
import org.apache.hadoop.fs.s3a.auth.SignerManager;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationOperations;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
+import org.apache.hadoop.fs.s3a.impl.AWSCannedACL;
+import org.apache.hadoop.fs.s3a.impl.AWSHeaders;
import org.apache.hadoop.fs.s3a.impl.BulkDeleteRetryHandler;
import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
import org.apache.hadoop.fs.s3a.impl.ContextAccessors;
import org.apache.hadoop.fs.s3a.impl.CopyFromLocalOperation;
-import org.apache.hadoop.fs.s3a.impl.CopyOutcome;
import org.apache.hadoop.fs.s3a.impl.CreateFileBuilder;
import org.apache.hadoop.fs.s3a.impl.DeleteOperation;
import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy;
@@ -122,6 +133,7 @@
import org.apache.hadoop.fs.s3a.impl.InternalConstants;
import org.apache.hadoop.fs.s3a.impl.ListingOperationCallbacks;
import org.apache.hadoop.fs.s3a.impl.MkdirOperation;
+import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException;
import org.apache.hadoop.fs.s3a.impl.OpenFileSupport;
import org.apache.hadoop.fs.s3a.impl.OperationCallbacks;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
@@ -131,7 +143,6 @@
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
import org.apache.hadoop.fs.s3a.impl.StoreContext;
import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder;
-import org.apache.hadoop.fs.s3a.impl.V2Migration;
import org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream;
import org.apache.hadoop.fs.s3a.tools.MarkerToolOperations;
import org.apache.hadoop.fs.s3a.tools.MarkerToolOperationsImpl;
@@ -141,6 +152,7 @@
import org.apache.hadoop.fs.statistics.IOStatisticsSource;
import org.apache.hadoop.fs.statistics.IOStatisticsContext;
import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.fs.store.LogExactlyOnce;
import org.apache.hadoop.fs.store.audit.AuditEntryPoint;
import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource;
import org.apache.hadoop.fs.store.audit.AuditSpan;
@@ -213,7 +225,8 @@
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.Statistic.*;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.INITIALIZE_SPAN;
-import static org.apache.hadoop.fs.s3a.auth.RolePolicies.STATEMENT_ALLOW_SSE_KMS_RW;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSCredentialProviderList;
+import static org.apache.hadoop.fs.s3a.auth.RolePolicies.STATEMENT_ALLOW_KMS_RW;
import static org.apache.hadoop.fs.s3a.auth.RolePolicies.allowS3Operations;
import static org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.TokenIssuingPolicy.NoTokensAvailable;
import static org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.hasDelegationTokenBinding;
@@ -224,14 +237,13 @@
import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_OVERWRITE;
import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isObjectNotFound;
import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_INACCESSIBLE;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_REQUIRED_EXCEPTION;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.ARN_BUCKET_OPTION;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_UPLOAD_PART_COUNT_LIMIT;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DELETE_CONSIDERED_IDEMPOTENT;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_403;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_403_FORBIDDEN;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404_NOT_FOUND;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.UPLOAD_PART_COUNT_LIMIT;
import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.fixBucketRegion;
import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.logDnsLookup;
@@ -246,6 +258,7 @@
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfSupplier;
import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
import static org.apache.hadoop.util.Preconditions.checkArgument;
+import static org.apache.hadoop.util.functional.RemoteIterators.foreach;
import static org.apache.hadoop.util.functional.RemoteIterators.typeCastingRemoteIterator;
/**
@@ -275,7 +288,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private URI uri;
private Path workingDir;
private String username;
- private AmazonS3 s3;
+ private S3Client s3Client;
+ /** Async client is used for transfer manager and s3 select. */
+ private S3AsyncClient s3AsyncClient;
// initial callback policy is fail-once; it's there just to assist
// some mock tests and other codepaths trying to call the low level
// APIs on an uninitialized filesystem.
@@ -294,7 +309,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private Listing listing;
private long partSize;
private boolean enableMultiObjectsDelete;
- private TransferManager transfers;
+ private S3TransferManager transferManager;
private ExecutorService boundedThreadPool;
private ThreadPoolExecutor unboundedThreadPool;
@@ -313,10 +328,14 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private int executorCapacity;
private long multiPartThreshold;
public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class);
+
+ /** Log to warn of storage class configuration problems. */
+ private static final LogExactlyOnce STORAGE_CLASS_WARNING = new LogExactlyOnce(LOG);
+
private static final Logger PROGRESS =
LoggerFactory.getLogger("org.apache.hadoop.fs.s3a.S3AFileSystem.Progress");
private LocalDirAllocator directoryAllocator;
- private CannedAccessControlList cannedACL;
+ private String cannedACL;
/**
* This must never be null; until initialized it just declares that there
@@ -364,6 +383,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
private AWSCredentialProviderList credentials;
private SignerManager signerManager;
+ private S3AInternals s3aInternals;
+
+ /**
+ * Do directory operations purge pending uploads?
+ */
+ private boolean dirOperationsPurgeUploads;
/**
* Page size for deletions.
@@ -418,6 +443,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*/
private boolean isMultipartUploadEnabled = DEFAULT_MULTIPART_UPLOAD_ENABLED;
+ /**
+ * Should file copy operations use the S3 transfer manager?
+ * True unless multipart upload is disabled.
+ */
+ private boolean isMultipartCopyEnabled;
+
/**
* A cache of files that should be deleted when the FileSystem is closed
* or the JVM is exited.
@@ -510,6 +541,8 @@ public void initialize(URI name, Configuration originalConf)
super.initialize(uri, conf);
setConf(conf);
+ s3aInternals = createS3AInternals();
+
// look for encryption data
// DT Bindings may override this
setEncryptionSecrets(
@@ -538,6 +571,9 @@ public void initialize(URI name, Configuration originalConf)
//check but do not store the block size
longBytesOption(conf, FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE, 1);
enableMultiObjectsDelete = conf.getBoolean(ENABLE_MULTI_DELETE, true);
+ // should the delete also purge uploads.
+ dirOperationsPurgeUploads = conf.getBoolean(DIRECTORY_OPERATIONS_PURGE_UPLOADS,
+ DIRECTORY_OPERATIONS_PURGE_UPLOADS_DEFAULT);
this.isMultipartUploadEnabled = conf.getBoolean(MULTIPART_UPLOADS_ENABLED,
DEFAULT_MULTIPART_UPLOAD_ENABLED);
@@ -550,6 +586,10 @@ public void initialize(URI name, Configuration originalConf)
this.prefetchBlockSize = (int) prefetchBlockSizeLong;
this.prefetchBlockCount =
intOption(conf, PREFETCH_BLOCK_COUNT_KEY, PREFETCH_BLOCK_DEFAULT_COUNT, 1);
+ this.isMultipartUploadEnabled = conf.getBoolean(MULTIPART_UPLOADS_ENABLED,
+ DEFAULT_MULTIPART_UPLOAD_ENABLED);
+ // multipart copy and upload are the same; this just makes it explicit
+ this.isMultipartCopyEnabled = isMultipartUploadEnabled;
initThreadPools(conf);
@@ -584,9 +624,6 @@ public void initialize(URI name, Configuration originalConf)
// the encryption algorithms)
bindAWSClient(name, delegationTokensEnabled);
- initTransferManager();
-
-
// This initiates a probe against S3 for the bucket existing.
doBucketProbing();
@@ -654,7 +691,7 @@ public void initialize(URI name, Configuration originalConf)
AWS_S3_VECTOR_ACTIVE_RANGE_READS, DEFAULT_AWS_S3_VECTOR_ACTIVE_RANGE_READS, 1);
vectoredIOContext = populateVectoredIOContext(conf);
scheme = (this.uri != null && this.uri.getScheme() != null) ? this.uri.getScheme() : FS_S3A;
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
// amazon client exception: stop all services then throw the translation
cleanupWithLogger(LOG, span);
stopAllServices();
@@ -706,7 +743,7 @@ private void setCSEGauge() {
* bucket existence check is not done to improve performance of
* S3AFileSystem initialization. When set to 1 or 2, bucket existence check
* will be performed which is potentially slow.
- * If 3 or higher: warn and use the v2 check.
+ * If 3 or higher: warn and skip check.
* Also logging DNS address of the s3 endpoint if the bucket probe value is
* greater than 0 else skipping it for increased performance.
* @throws UnknownStoreException the bucket is absent
@@ -723,18 +760,14 @@ private void doBucketProbing() throws IOException {
LOG.debug("skipping check for bucket existence");
break;
case 1:
- logDnsLookup(getConf());
- verifyBucketExists();
- break;
case 2:
logDnsLookup(getConf());
- verifyBucketExistsV2();
+ verifyBucketExists();
break;
default:
// we have no idea what this is, assume it is from a later release.
- LOG.warn("Unknown bucket probe option {}: {}; falling back to check #2",
+ LOG.warn("Unknown bucket probe option {}: {}; skipping check for bucket existence",
S3A_BUCKET_PROBE, bucketProbe);
- verifyBucketExistsV2();
break;
}
}
@@ -827,54 +860,34 @@ protected static S3AStorageStatistics createStorageStatistics(
}
/**
- * Verify that the bucket exists. This does not check permissions,
- * not even read access.
+ * Verify that the bucket exists.
* Retry policy: retrying, translated.
* @throws UnknownStoreException the bucket is absent
* @throws IOException any other problem talking to S3
*/
@Retries.RetryTranslated
- protected void verifyBucketExists()
- throws UnknownStoreException, IOException {
- if (!invoker.retry("doesBucketExist", bucket, true,
- trackDurationOfOperation(getDurationTrackerFactory(),
- STORE_EXISTS_PROBE.getSymbol(),
- () -> s3.doesBucketExist(bucket)))) {
- throw new UnknownStoreException("s3a://" + bucket + "/", " Bucket does "
- + "not exist");
- }
- }
+ protected void verifyBucketExists() throws UnknownStoreException, IOException {
- /**
- * Verify that the bucket exists. This will correctly throw an exception
- * when credentials are invalid.
- * Retry policy: retrying, translated.
- * @throws UnknownStoreException the bucket is absent
- * @throws IOException any other problem talking to S3
- */
- @Retries.RetryTranslated
- protected void verifyBucketExistsV2()
- throws UnknownStoreException, IOException {
- if (!invoker.retry("doesBucketExistV2", bucket, true,
- trackDurationOfOperation(getDurationTrackerFactory(),
- STORE_EXISTS_PROBE.getSymbol(),
- () -> {
- // Bug in SDK always returns `true` for AccessPoint ARNs with `doesBucketExistV2()`
- // expanding implementation to use ARNs and buckets correctly
+ if(!trackDurationAndSpan(
+ STORE_EXISTS_PROBE, bucket, null, () ->
+ invoker.retry("doesBucketExist", bucket, true, () -> {
try {
- s3.getBucketAcl(bucket);
- } catch (AmazonServiceException ex) {
- int statusCode = ex.getStatusCode();
- if (statusCode == SC_404 ||
- (statusCode == SC_403 && ex.getMessage().contains(AP_INACCESSIBLE))) {
+ s3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build());
+ return true;
+ } catch (AwsServiceException ex) {
+ int statusCode = ex.statusCode();
+ if (statusCode == SC_404_NOT_FOUND ||
+ (statusCode == SC_403_FORBIDDEN && accessPoint != null)) {
return false;
}
}
return true;
}))) {
- throw new UnknownStoreException("s3a://" + bucket + "/", " Bucket does "
- + "not exist");
+
+ throw new UnknownStoreException("s3a://" + bucket + "/",
+ " Bucket does " + "not exist. " + "Accessing with " + ENDPOINT + " set to "
+ + getConf().getTrimmed(ENDPOINT, null));
}
}
@@ -915,7 +928,6 @@ public Listing getListing() {
* @param dtEnabled are delegation tokens enabled?
* @throws IOException failure.
*/
- @SuppressWarnings("deprecation")
private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
Configuration conf = getConf();
credentials = null;
@@ -928,7 +940,6 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
// with it if so.
LOG.debug("Using delegation tokens");
- V2Migration.v1DelegationTokenCredentialProvidersUsed();
S3ADelegationTokens tokens = new S3ADelegationTokens();
this.delegationTokens = Optional.of(tokens);
tokens.bindToFileSystem(getCanonicalUri(),
@@ -955,7 +966,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
uaSuffix = tokens.getUserAgentField();
} else {
// DT support is disabled, so create the normal credential chain
- credentials = createAWSCredentialProviderSet(name, conf);
+ credentials = createAWSCredentialProviderList(name, conf);
}
LOG.debug("Using credential provider {}", credentials);
Class s3ClientFactoryClass = conf.getClass(
@@ -966,8 +977,12 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
? conf.getTrimmed(ENDPOINT, DEFAULT_ENDPOINT)
: accessPoint.getEndpoint();
- S3ClientFactory.S3ClientCreationParameters parameters = null;
- parameters = new S3ClientFactory.S3ClientCreationParameters()
+ String configuredRegion = accessPoint == null
+ ? conf.getTrimmed(AWS_REGION)
+ : accessPoint.getRegion();
+
+ S3ClientFactory.S3ClientCreationParameters parameters =
+ new S3ClientFactory.S3ClientCreationParameters()
.withCredentialSet(credentials)
.withPathUri(name)
.withEndpoint(endpoint)
@@ -975,11 +990,30 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
.withPathStyleAccess(conf.getBoolean(PATH_STYLE_ACCESS, false))
.withUserAgentSuffix(uaSuffix)
.withRequesterPays(conf.getBoolean(ALLOW_REQUESTER_PAYS, DEFAULT_ALLOW_REQUESTER_PAYS))
- .withRequestHandlers(auditManager.createRequestHandlers());
+ .withExecutionInterceptors(auditManager.createExecutionInterceptors())
+ .withMinimumPartSize(partSize)
+ .withMultipartCopyEnabled(isMultipartCopyEnabled)
+ .withMultipartThreshold(multiPartThreshold)
+ .withTransferManagerExecutor(unboundedThreadPool)
+ .withRegion(configuredRegion);
- s3 = ReflectionUtils.newInstance(s3ClientFactoryClass, conf)
- .createS3Client(getUri(),
- parameters);
+ S3ClientFactory clientFactory = ReflectionUtils.newInstance(s3ClientFactoryClass, conf);
+ s3Client = clientFactory.createS3Client(getUri(), parameters);
+ createS3AsyncClient(clientFactory, parameters);
+ transferManager = clientFactory.createS3TransferManager(getS3AsyncClient());
+ }
+
+ /**
+ * Creates and configures the S3AsyncClient.
+ * Uses synchronized method to suppress spotbugs error.
+ *
+ * @param clientFactory factory used to create S3AsyncClient
+ * @param parameters parameter object
+ * @throws IOException on any IO problem
+ */
+ private void createS3AsyncClient(S3ClientFactory clientFactory,
+ S3ClientFactory.S3ClientCreationParameters parameters) throws IOException {
+ s3AsyncClient = clientFactory.createS3AsyncClient(getUri(), parameters);
}
/**
@@ -1073,18 +1107,24 @@ protected RequestFactory createRequestFactory() {
// Any encoding type
String contentEncoding = getConf().getTrimmed(CONTENT_ENCODING, null);
+ if (contentEncoding != null) {
+ LOG.debug("Using content encoding set in {} = {}", CONTENT_ENCODING, contentEncoding);
+ }
String storageClassConf = getConf()
.getTrimmed(STORAGE_CLASS, "")
.toUpperCase(Locale.US);
StorageClass storageClass = null;
if (!storageClassConf.isEmpty()) {
- try {
- storageClass = StorageClass.fromValue(storageClassConf);
- } catch (IllegalArgumentException e) {
- LOG.warn("Unknown storage class property {}: {}; falling back to default storage class",
- STORAGE_CLASS, storageClassConf);
+ storageClass = StorageClass.fromValue(storageClassConf);
+ LOG.debug("Using storage class {}", storageClass);
+ if (storageClass.equals(StorageClass.UNKNOWN_TO_SDK_VERSION)) {
+ STORAGE_CLASS_WARNING.warn("Unknown storage class \"{}\" from option: {};"
+ + " falling back to default storage class",
+ storageClassConf, STORAGE_CLASS);
+ storageClass = null;
}
+
} else {
LOG.debug("Unset storage class property {}; falling back to default storage class",
STORAGE_CLASS);
@@ -1110,6 +1150,14 @@ public RequestFactory getRequestFactory() {
return requestFactory;
}
+ /**
+ * Get the S3 Async client.
+ * @return the async s3 client.
+ */
+ private S3AsyncClient getS3AsyncClient() {
+ return s3AsyncClient;
+ }
+
/**
* Implementation of all operations used by delegation tokens.
*/
@@ -1150,22 +1198,10 @@ public EncryptionSecrets getEncryptionSecrets() {
return encryptionSecrets;
}
- private void initTransferManager() {
- TransferManagerConfiguration transferConfiguration =
- new TransferManagerConfiguration();
- transferConfiguration.setMinimumUploadPartSize(partSize);
- transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
- transferConfiguration.setMultipartCopyPartSize(partSize);
- transferConfiguration.setMultipartCopyThreshold(multiPartThreshold);
-
- transfers = new TransferManager(s3, unboundedThreadPool);
- transfers.setConfiguration(transferConfiguration);
- }
-
private void initCannedAcls(Configuration conf) {
String cannedACLName = conf.get(CANNED_ACL, DEFAULT_CANNED_ACL);
if (!cannedACLName.isEmpty()) {
- cannedACL = CannedAccessControlList.valueOf(cannedACLName);
+ cannedACL = AWSCannedACL.valueOf(cannedACLName).toString();
} else {
cannedACL = null;
}
@@ -1198,12 +1234,22 @@ private void initMultipartUploads(Configuration conf) throws IOException {
public void abortOutstandingMultipartUploads(long seconds)
throws IOException {
Preconditions.checkArgument(seconds >= 0);
- Date purgeBefore =
- new Date(new Date().getTime() - seconds * 1000);
+ Instant purgeBefore =
+ Instant.now().minusSeconds(seconds);
LOG.debug("Purging outstanding multipart uploads older than {}",
purgeBefore);
invoker.retry("Purging multipart uploads", bucket, true,
- () -> transfers.abortMultipartUploads(bucket, purgeBefore));
+ () -> {
+ RemoteIterator uploadIterator =
+ MultipartUtils.listMultipartUploads(createStoreContext(), s3Client, null, maxKeys);
+
+ while (uploadIterator.hasNext()) {
+ MultipartUpload upload = uploadIterator.next();
+ if (upload.initiated().compareTo(purgeBefore) < 0) {
+ abortMultipartUpload(upload);
+ }
+ }
+ });
}
/**
@@ -1252,81 +1298,127 @@ public int getDefaultPort() {
}
/**
- * Returns the S3 client used by this filesystem.
- * This is for internal use within the S3A code itself.
- * @return AmazonS3Client
+ * Set the client -used in mocking tests to force in a different client.
+ * @param client client.
*/
- private AmazonS3 getAmazonS3Client() {
- return s3;
+ protected void setAmazonS3Client(S3Client client) {
+ Preconditions.checkNotNull(client, "clientV2");
+ LOG.debug("Setting S3V2 client to {}", client);
+ s3Client = client;
}
/**
- * Returns the S3 client used by this filesystem.
- * Warning: this must only be used for testing, as it bypasses core
- * S3A operations.
- * @param reason a justification for requesting access.
- * @return AmazonS3Client
+ * S3AInternals method.
+ * {@inheritDoc}.
*/
- @VisibleForTesting
- public AmazonS3 getAmazonS3ClientForTesting(String reason) {
- LOG.warn("Access to S3A client requested, reason {}", reason);
- V2Migration.v1S3ClientRequested();
- return s3;
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public String getBucketLocation() throws IOException {
+ return s3aInternals.getBucketLocation(bucket);
}
/**
- * Set the client -used in mocking tests to force in a different client.
- * @param client client.
+ * Create the S3AInternals; left as something mocking
+ * subclasses may want to override.
+ * @return the internal implementation
*/
- protected void setAmazonS3Client(AmazonS3 client) {
- Preconditions.checkNotNull(client, "client");
- LOG.debug("Setting S3 client to {}", client);
- s3 = client;
-
- // Need to use a new TransferManager that uses the new client.
- // Also, using a new TransferManager requires a new threadpool as the old
- // TransferManager will shut the thread pool down when it is garbage
- // collected.
- initThreadPools(getConf());
- initTransferManager();
+ protected S3AInternals createS3AInternals() {
+ return new S3AInternalsImpl();
}
/**
- * Get the region of a bucket.
- * Invoked from StoreContext; consider an entry point.
- * @return the region in which a bucket is located
- * @throws AccessDeniedException if the caller lacks permission.
- * @throws IOException on any failure.
+ * Get the S3AInternals.
+ * @return the internal implementation
*/
- @Retries.RetryTranslated
- @InterfaceAudience.LimitedPrivate("diagnostics")
- public String getBucketLocation() throws IOException {
- return getBucketLocation(bucket);
+ public S3AInternals getS3AInternals() {
+ return s3aInternals;
}
/**
- * Get the region of a bucket; fixing up the region so it can be used
- * in the builders of other AWS clients.
- * Requires the caller to have the AWS role permission
- * {@code s3:GetBucketLocation}.
- * Retry policy: retrying, translated.
- * @param bucketName the name of the bucket
- * @return the region in which a bucket is located
- * @throws AccessDeniedException if the caller lacks permission.
- * @throws IOException on any failure.
+ * Implementation of the S3A Internals operations; pulled out of S3AFileSystem to
+ * force code accessing it to call {@link #getS3AInternals()}.
*/
- @VisibleForTesting
- @AuditEntryPoint
- @Retries.RetryTranslated
- public String getBucketLocation(String bucketName) throws IOException {
- final String region = trackDurationAndSpan(
- STORE_EXISTS_PROBE, bucketName, null, () ->
- invoker.retry("getBucketLocation()", bucketName, true, () ->
- // If accessPoint then region is known from Arn
- accessPoint != null
- ? accessPoint.getRegion()
- : s3.getBucketLocation(bucketName)));
- return fixBucketRegion(region);
+ private final class S3AInternalsImpl implements S3AInternals {
+
+ @Override
+ public S3Client getAmazonS3Client(String reason) {
+ LOG.debug("Access to S3 client requested, reason {}", reason);
+ return s3Client;
+ }
+
+ /**
+ * S3AInternals method.
+ * {@inheritDoc}.
+ */
+ @Override
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public String getBucketLocation() throws IOException {
+ return s3aInternals.getBucketLocation(bucket);
+ }
+
+ /**
+ * S3AInternals method.
+ * {@inheritDoc}.
+ */
+ @Override
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public String getBucketLocation(String bucketName) throws IOException {
+ final String region = trackDurationAndSpan(
+ STORE_EXISTS_PROBE, bucketName, null, () ->
+ invoker.retry("getBucketLocation()", bucketName, true, () ->
+ // If accessPoint then region is known from Arn
+ accessPoint != null
+ ? accessPoint.getRegion()
+ : s3Client.getBucketLocation(GetBucketLocationRequest.builder()
+ .bucket(bucketName)
+ .build())
+ .locationConstraintAsString()));
+ return fixBucketRegion(region);
+ }
+
+ /**
+ * S3AInternals method.
+ * {@inheritDoc}.
+ */
+ @Override
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public HeadObjectResponse getObjectMetadata(Path path) throws IOException {
+ return trackDurationAndSpan(INVOCATION_GET_FILE_STATUS, path, () ->
+ S3AFileSystem.this.getObjectMetadata(makeQualified(path), null, invoker,
+ "getObjectMetadata"));
+ }
+
+ /**
+ * S3AInternals method.
+ * {@inheritDoc}.
+ */
+ @Override
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ public HeadBucketResponse getBucketMetadata() throws IOException {
+ return S3AFileSystem.this.getBucketMetadata();
+ }
+
+ /**
+ * Get a shared copy of the AWS credentials, with its reference
+ * counter updated.
+ * Caller is required to call {@code close()} on this after
+ * they have finished using it.
+ * @param purpose what is this for? This is initially for logging
+ * @return a reference to shared credentials.
+ */
+ public AWSCredentialProviderList shareCredentials(final String purpose) {
+ LOG.debug("Sharing credentials for: {}", purpose);
+ return credentials.share();
+ }
+
+ @Override
+ public boolean isMultipartCopyEnabled() {
+ return S3AFileSystem.this.isMultipartUploadEnabled;
+ }
}
/**
@@ -1349,7 +1441,7 @@ public ChangeDetectionPolicy getChangeDetectionPolicy() {
}
/**
- * Get the encryption algorithm of this endpoint.
+ * Get the encryption algorithm of this connector.
* @return the encryption algorithm.
*/
public S3AEncryptionMethods getS3EncryptionAlgorithm() {
@@ -1396,6 +1488,8 @@ private void initLocalDirAllocatorIfNotInitialized(Configuration conf) {
* Get the bucket of this filesystem.
* @return the bucket
*/
+ @InterfaceAudience.Public
+ @InterfaceStability.Stable
public String getBucket() {
return bucket;
}
@@ -1413,7 +1507,7 @@ protected void setBucket(String bucket) {
* Get the canned ACL of this FS.
* @return an ACL, if any
*/
- CannedAccessControlList getCannedACL() {
+ String getCannedACL() {
return cannedACL;
}
@@ -1643,18 +1737,18 @@ public void close() {
}
@Override
- public GetObjectRequest newGetRequest(final String key) {
+ public GetObjectRequest.Builder newGetRequestBuilder(final String key) {
// active the audit span used for the operation
try (AuditSpan span = auditSpan.activate()) {
- return getRequestFactory().newGetObjectRequest(key);
+ return getRequestFactory().newGetObjectRequestBuilder(key);
}
}
@Override
- public S3Object getObject(GetObjectRequest request) {
+ public ResponseInputStream getObject(GetObjectRequest request) {
// active the audit span used for the operation
try (AuditSpan span = auditSpan.activate()) {
- return s3.getObject(request);
+ return s3Client.getObject(request);
}
}
@@ -1681,18 +1775,19 @@ private final class WriteOperationHelperCallbacksImpl
implements WriteOperationHelper.WriteOperationHelperCallbacks {
@Override
- public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) {
- return s3.selectObjectContent(request);
+ public CompletableFuture selectObjectContent(
+ SelectObjectContentRequest request,
+ SelectObjectContentResponseHandler responseHandler) {
+ return getS3AsyncClient().selectObjectContent(request, responseHandler);
}
@Override
- public CompleteMultipartUploadResult completeMultipartUpload(
+ public CompleteMultipartUploadResponse completeMultipartUpload(
CompleteMultipartUploadRequest request) {
- return s3.completeMultipartUpload(request);
+ return s3Client.completeMultipartUpload(request);
}
}
-
/**
* Create the read context for reading from the referenced file,
* using FS state as well as the status.
@@ -2054,7 +2149,7 @@ public boolean rename(Path src, Path dst) throws IOException {
innerRename(src, dst));
LOG.debug("Copied {} bytes", bytesCopied);
return true;
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("rename(" + src +", " + dst + ")", src, e);
} catch (RenameFailedException e) {
LOG.info("{}", e.getMessage());
@@ -2165,7 +2260,7 @@ private Pair initiateRename(
* This operation throws an exception on any failure which needs to be
* reported and downgraded to a failure.
* Retries: retry translated, assuming all operations it is called do
- * so. For safely, consider catch and handle AmazonClientException
+ * so. For safely, consider catch and handle SdkException
* because this is such a complex method there's a risk it could surface.
* @param source path to be renamed
* @param dest new path after rename
@@ -2176,12 +2271,12 @@ private Pair initiateRename(
* @return the number of bytes copied.
* @throws FileNotFoundException there's no source file.
* @throws IOException on IO failure.
- * @throws AmazonClientException on failures inside the AWS SDK
+ * @throws SdkException on failures inside the AWS SDK
*/
@Retries.RetryMixed
private long innerRename(Path source, Path dest)
throws RenameFailedException, FileNotFoundException, IOException,
- AmazonClientException {
+ SdkException {
Path src = qualify(source);
Path dst = qualify(dest);
@@ -2194,12 +2289,14 @@ private long innerRename(Path source, Path dest)
// Initiate the rename.
// this will call back into this class via the rename callbacks
+ final StoreContext storeContext = createStoreContext();
RenameOperation renameOperation = new RenameOperation(
- createStoreContext(),
+ storeContext,
src, srcKey, p.getLeft(),
dst, dstKey, p.getRight(),
- new OperationCallbacksImpl(),
- pageSize);
+ new OperationCallbacksImpl(storeContext),
+ pageSize,
+ dirOperationsPurgeUploads);
return renameOperation.execute();
}
@@ -2220,8 +2317,19 @@ private final class OperationCallbacksImpl implements OperationCallbacks {
/** Audit Span at time of creation. */
private final AuditSpan auditSpan;
- private OperationCallbacksImpl() {
- auditSpan = getActiveAuditSpan();
+ private final StoreContext storeContext;
+
+ private OperationCallbacksImpl(final StoreContext storeContext) {
+ this.storeContext = requireNonNull(storeContext);
+ this.auditSpan = storeContext.getActiveAuditSpan();
+ }
+
+ /**
+ * Get the audit span.
+ * @return the span
+ */
+ private AuditSpan getAuditSpan() {
+ return auditSpan;
}
@Override
@@ -2276,7 +2384,7 @@ public RemoteIterator listFilesAndDirectoryMarkers(
}
@Override
- public CopyResult copyFile(final String srcKey,
+ public CopyObjectResponse copyFile(final String srcKey,
final String destKey,
final S3ObjectAttributes srcAttributes,
final S3AReadOpContext readContext) throws IOException {
@@ -2287,9 +2395,9 @@ public CopyResult copyFile(final String srcKey,
@Override
public void removeKeys(
- final List keysToDelete,
+ final List keysToDelete,
final boolean deleteFakeDir)
- throws MultiObjectDeleteException, AmazonClientException, IOException {
+ throws MultiObjectDeleteException, SdkException, IOException {
auditSpan.activate();
S3AFileSystem.this.removeKeys(keysToDelete, deleteFakeDir);
}
@@ -2321,7 +2429,29 @@ public RemoteIterator listObjects(
Listing.ACCEPT_ALL_BUT_S3N,
auditSpan));
}
- }
+
+ /**
+ * Abort multipart uploads under a path.
+ * @param prefix prefix for uploads to abort
+ * @return a count of aborts
+ * @throws IOException trouble; FileNotFoundExceptions are swallowed.
+ */
+ @Override
+ @Retries.RetryTranslated
+ public long abortMultipartUploadsUnderPrefix(String prefix)
+ throws IOException {
+ getAuditSpan().activate();
+ // this deactivates the audit span somehow
+ final RemoteIterator uploads =
+ S3AFileSystem.this.listUploadsUnderPrefix(storeContext, prefix);
+ // so reactivate it.
+ getAuditSpan().activate();
+ return foreach(uploads, upload ->
+ invoker.retry("Aborting multipart commit", upload.key(), true, () ->
+ S3AFileSystem.this.abortMultipartUpload(upload)));
+ }
+
+ } // end OperationCallbacksImpl
/**
* Callbacks from {@link Listing}.
@@ -2388,21 +2518,17 @@ public int getMaxKeys() {
* Low-level call to get at the object metadata.
* This method is used in some external applications and so
* must be viewed as a public entry point.
- * Auditing: An audit entry point.
+ * @deprecated use S3AInternals API.
* @param path path to the object. This will be qualified.
* @return metadata
* @throws IOException IO and object access problems.
*/
- @VisibleForTesting
@AuditEntryPoint
@InterfaceAudience.LimitedPrivate("utilities")
@Retries.RetryTranslated
- @InterfaceStability.Evolving
- public ObjectMetadata getObjectMetadata(Path path) throws IOException {
- V2Migration.v1GetObjectMetadataCalled();
- return trackDurationAndSpan(INVOCATION_GET_FILE_STATUS, path, () ->
- getObjectMetadata(makeQualified(path), null, invoker,
- "getObjectMetadata"));
+ @Deprecated
+ public HeadObjectResponse getObjectMetadata(Path path) throws IOException {
+ return getS3AInternals().getObjectMetadata(path);
}
/**
@@ -2415,7 +2541,7 @@ public ObjectMetadata getObjectMetadata(Path path) throws IOException {
* @throws IOException IO and object access problems.
*/
@Retries.RetryTranslated
- private ObjectMetadata getObjectMetadata(Path path,
+ private HeadObjectResponse getObjectMetadata(Path path,
ChangeTracker changeTracker, Invoker changeInvoker, String operation)
throws IOException {
String key = pathToKey(path);
@@ -2628,7 +2754,7 @@ protected DurationTrackerFactory nonNullDurationTrackerFactory(
@Retries.RetryRaw
@VisibleForTesting
@InterfaceAudience.LimitedPrivate("external utilities")
- ObjectMetadata getObjectMetadata(String key) throws IOException {
+ HeadObjectResponse getObjectMetadata(String key) throws IOException {
return getObjectMetadata(key, null, invoker, "getObjectMetadata");
}
@@ -2645,28 +2771,28 @@ ObjectMetadata getObjectMetadata(String key) throws IOException {
* @throws RemoteFileChangedException if an unexpected version is detected
*/
@Retries.RetryRaw
- protected ObjectMetadata getObjectMetadata(String key,
+ protected HeadObjectResponse getObjectMetadata(String key,
ChangeTracker changeTracker,
Invoker changeInvoker,
String operation) throws IOException {
- ObjectMetadata meta = changeInvoker.retryUntranslated("GET " + key, true,
+ HeadObjectResponse response = changeInvoker.retryUntranslated("GET " + key, true,
() -> {
- GetObjectMetadataRequest request
- = getRequestFactory().newGetObjectMetadataRequest(key);
+ HeadObjectRequest.Builder requestBuilder =
+ getRequestFactory().newHeadObjectRequestBuilder(key);
incrementStatistic(OBJECT_METADATA_REQUESTS);
DurationTracker duration = getDurationTrackerFactory()
.trackDuration(ACTION_HTTP_HEAD_REQUEST.getSymbol());
try {
LOG.debug("HEAD {} with change tracker {}", key, changeTracker);
if (changeTracker != null) {
- changeTracker.maybeApplyConstraint(request);
+ changeTracker.maybeApplyConstraint(requestBuilder);
}
- ObjectMetadata objectMetadata = s3.getObjectMetadata(request);
+ HeadObjectResponse headObjectResponse = s3Client.headObject(requestBuilder.build());
if (changeTracker != null) {
- changeTracker.processMetadata(objectMetadata, operation);
+ changeTracker.processMetadata(headObjectResponse, operation);
}
- return objectMetadata;
- } catch(AmazonServiceException ase) {
+ return headObjectResponse;
+ } catch (AwsServiceException ase) {
if (!isObjectNotFound(ase)) {
// file not found is not considered a failure of the call,
// so only switch the duration tracker to update failure
@@ -2680,7 +2806,28 @@ protected ObjectMetadata getObjectMetadata(String key,
}
});
incrementReadOperations();
- return meta;
+ return response;
+ }
+
+ /**
+ * Request bucket metadata.
+ * @return the metadata
+ * @throws UnknownStoreException the bucket is absent
+ * @throws IOException any other problem talking to S3
+ */
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ protected HeadBucketResponse getBucketMetadata() throws IOException {
+ final HeadBucketResponse response = trackDurationAndSpan(STORE_EXISTS_PROBE, bucket, null,
+ () -> invoker.retry("getBucketMetadata()", bucket, true, () -> {
+ try {
+ return s3Client.headBucket(
+ getRequestFactory().newHeadBucketRequestBuilder(bucket).build());
+ } catch (NoSuchBucketException e) {
+ throw new UnknownStoreException("s3a://" + bucket + "/", " Bucket does " + "not exist");
+ }
+ }));
+ return response;
}
/**
@@ -2709,9 +2856,9 @@ protected S3ListResult listObjects(S3ListRequest request,
OBJECT_LIST_REQUEST,
() -> {
if (useListV1) {
- return S3ListResult.v1(s3.listObjects(request.getV1()));
+ return S3ListResult.v1(s3Client.listObjects(request.getV1()));
} else {
- return S3ListResult.v2(s3.listObjectsV2(request.getV2()));
+ return S3ListResult.v2(s3Client.listObjectsV2(request.getV2()));
}
}));
}
@@ -2754,15 +2901,21 @@ protected S3ListResult continueListObjects(S3ListRequest request,
OBJECT_CONTINUE_LIST_REQUEST,
() -> {
if (useListV1) {
- return S3ListResult.v1(
- s3.listNextBatchOfObjects(
- getRequestFactory()
- .newListNextBatchOfObjectsRequest(
- prevResult.getV1())));
+ List prevListResult = prevResult.getV1().contents();
+
+ // Next markers are only present when a delimiter is specified.
+ String nextMarker;
+ if (prevResult.getV1().nextMarker() != null) {
+ nextMarker = prevResult.getV1().nextMarker();
+ } else {
+ nextMarker = prevListResult.get(prevListResult.size() - 1).key();
+ }
+
+ return S3ListResult.v1(s3Client.listObjects(
+ request.getV1().toBuilder().marker(nextMarker).build()));
} else {
- request.getV2().setContinuationToken(prevResult.getV2()
- .getNextContinuationToken());
- return S3ListResult.v2(s3.listObjectsV2(request.getV2()));
+ return S3ListResult.v2(s3Client.listObjectsV2(request.getV2().toBuilder()
+ .continuationToken(prevResult.getV2().nextContinuationToken()).build()));
}
}));
}
@@ -2792,14 +2945,14 @@ public void incrementWriteOperations() {
*
* Retry policy: retry untranslated; delete considered idempotent.
* @param key key to blob to delete.
- * @throws AmazonClientException problems working with S3
+ * @throws SdkException problems working with S3
* @throws InvalidRequestException if the request was rejected due to
* a mistaken attempt to delete the root directory.
*/
@VisibleForTesting
@Retries.RetryRaw
protected void deleteObject(String key)
- throws AmazonClientException, IOException {
+ throws SdkException, IOException {
blockRootDelete(key);
incrementWriteOperations();
try (DurationInfo ignored =
@@ -2807,14 +2960,21 @@ protected void deleteObject(String key)
"deleting %s", key)) {
invoker.retryUntranslated(String.format("Delete %s:/%s", bucket, key),
DELETE_CONSIDERED_IDEMPOTENT,
- ()-> {
+ () -> {
incrementStatistic(OBJECT_DELETE_OBJECTS);
trackDurationOfInvocation(getDurationTrackerFactory(),
OBJECT_DELETE_REQUEST.getSymbol(),
- () -> s3.deleteObject(getRequestFactory()
- .newDeleteObjectRequest(key)));
+ () -> s3Client.deleteObject(getRequestFactory()
+ .newDeleteObjectRequestBuilder(key)
+ .build()));
return null;
});
+ } catch (AwsServiceException ase) {
+ // 404 errors get swallowed; this can be raised by
+ // third party stores (GCS).
+ if (!isObjectNotFound(ase)) {
+ throw ase;
+ }
}
}
@@ -2825,14 +2985,14 @@ protected void deleteObject(String key)
* @param f path path to delete
* @param key key of entry
* @param isFile is the path a file (used for instrumentation only)
- * @throws AmazonClientException problems working with S3
+ * @throws SdkException problems working with S3
* @throws IOException from invoker signature only -should not be raised.
*/
@Retries.RetryRaw
void deleteObjectAtPath(Path f,
String key,
boolean isFile)
- throws AmazonClientException, IOException {
+ throws SdkException, IOException {
if (isFile) {
instrumentation.fileDeleted(1);
} else {
@@ -2874,66 +3034,58 @@ private void blockRootDelete(String key) throws InvalidRequestException {
* @return the AWS response
* @throws MultiObjectDeleteException one or more of the keys could not
* be deleted.
- * @throws AmazonClientException amazon-layer failure.
+ * @throws SdkException amazon-layer failure.
*/
@Retries.RetryRaw
- private DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteRequest)
- throws MultiObjectDeleteException, AmazonClientException, IOException {
+ private DeleteObjectsResponse deleteObjects(DeleteObjectsRequest deleteRequest)
+ throws MultiObjectDeleteException, SdkException, IOException {
incrementWriteOperations();
BulkDeleteRetryHandler retryHandler =
new BulkDeleteRetryHandler(createStoreContext());
- int keyCount = deleteRequest.getKeys().size();
- try(DurationInfo ignored =
+ int keyCount = deleteRequest.delete().objects().size();
+ try (DurationInfo ignored =
new DurationInfo(LOG, false, "DELETE %d keys",
keyCount)) {
- return invoker.retryUntranslated("delete",
- DELETE_CONSIDERED_IDEMPOTENT,
- (text, e, r, i) -> {
- // handle the failure
- retryHandler.bulkDeleteRetried(deleteRequest, e);
- },
- // duration is tracked in the bulk delete counters
- trackDurationOfOperation(getDurationTrackerFactory(),
- OBJECT_BULK_DELETE_REQUEST.getSymbol(), () -> {
- incrementStatistic(OBJECT_DELETE_OBJECTS, keyCount);
- return s3.deleteObjects(deleteRequest);
- }));
- } catch (MultiObjectDeleteException e) {
- // one or more of the keys could not be deleted.
- // log and rethrow
- List errors = e.getErrors();
- LOG.debug("Partial failure of delete, {} errors", errors.size(), e);
- for (MultiObjectDeleteException.DeleteError error : errors) {
- LOG.debug("{}: \"{}\" - {}",
- error.getKey(), error.getCode(), error.getMessage());
+ DeleteObjectsResponse response =
+ invoker.retryUntranslated("delete", DELETE_CONSIDERED_IDEMPOTENT,
+ (text, e, r, i) -> {
+ // handle the failure
+ retryHandler.bulkDeleteRetried(deleteRequest, e);
+ },
+ // duration is tracked in the bulk delete counters
+ trackDurationOfOperation(getDurationTrackerFactory(),
+ OBJECT_BULK_DELETE_REQUEST.getSymbol(), () -> {
+ incrementStatistic(OBJECT_DELETE_OBJECTS, keyCount);
+ return s3Client.deleteObjects(deleteRequest);
+ }));
+
+ if (!response.errors().isEmpty()) {
+ // one or more of the keys could not be deleted.
+ // log and then throw
+ List errors = response.errors();
+ LOG.debug("Partial failure of delete, {} errors", errors.size());
+ for (S3Error error : errors) {
+ LOG.debug("{}: \"{}\" - {}", error.key(), error.code(), error.message());
+ }
+ throw new MultiObjectDeleteException(errors);
}
- throw e;
+
+ return response;
}
}
/**
- * Create a putObject request.
+ * Create a putObject request builder.
* Adds the ACL and metadata
* @param key key of object
- * @param metadata metadata header
- * @param srcfile source file
+ * @param length length of object to be uploaded
+ * @param isDirectoryMarker true if object to be uploaded is a directory marker
* @return the request
*/
- public PutObjectRequest newPutObjectRequest(String key,
- ObjectMetadata metadata, File srcfile) {
- return requestFactory.newPutObjectRequest(key, metadata, null, srcfile);
- }
-
- /**
- * Create a new object metadata instance.
- * Any standard metadata headers are added here, for example:
- * encryption.
- *
- * @param length length of data to set in header.
- * @return a new metadata instance
- */
- public ObjectMetadata newObjectMetadata(long length) {
- return requestFactory.newObjectMetadata(length);
+ public PutObjectRequest.Builder newPutObjectRequestBuilder(String key,
+ long length,
+ boolean isDirectoryMarker) {
+ return requestFactory.newPutObjectRequestBuilder(key, null, length, isDirectoryMarker);
}
/**
@@ -2950,15 +3102,24 @@ public ObjectMetadata newObjectMetadata(long length) {
* Retry policy: N/A: the transfer manager is performing the upload.
* Auditing: must be inside an audit span.
* @param putObjectRequest the request
+ * @param file the file to be uploaded
+ * @param listener the progress listener for the request
* @return the upload initiated
*/
@Retries.OnceRaw
- public UploadInfo putObject(PutObjectRequest putObjectRequest) {
+ public UploadInfo putObject(PutObjectRequest putObjectRequest, File file,
+ ProgressableProgressListener listener) {
long len = getPutRequestLength(putObjectRequest);
- LOG.debug("PUT {} bytes to {} via transfer manager ",
- len, putObjectRequest.getKey());
+ LOG.debug("PUT {} bytes to {} via transfer manager ", len, putObjectRequest.key());
incrementPutStartStatistics(len);
- Upload upload = transfers.upload(putObjectRequest);
+
+ FileUpload upload = transferManager.uploadFile(
+ UploadFileRequest.builder()
+ .putObjectRequest(putObjectRequest)
+ .source(file)
+ .addTransferListener(listener)
+ .build());
+
return new UploadInfo(upload, len);
}
@@ -2973,30 +3134,37 @@ public UploadInfo putObject(PutObjectRequest putObjectRequest) {
* @param putObjectRequest the request
* @param putOptions put object options
* @param durationTrackerFactory factory for duration tracking
+ * @param uploadData data to be uploaded
+ * @param isFile represents if data to be uploaded is a file
* @return the upload initiated
- * @throws AmazonClientException on problems
+ * @throws SdkException on problems
*/
@VisibleForTesting
@Retries.OnceRaw("For PUT; post-PUT actions are RetryExceptionsSwallowed")
- PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest,
+ PutObjectResponse putObjectDirect(PutObjectRequest putObjectRequest,
PutObjectOptions putOptions,
+ S3ADataBlocks.BlockUploadData uploadData, boolean isFile,
DurationTrackerFactory durationTrackerFactory)
- throws AmazonClientException {
+ throws SdkException {
long len = getPutRequestLength(putObjectRequest);
- LOG.debug("PUT {} bytes to {}", len, putObjectRequest.getKey());
+ LOG.debug("PUT {} bytes to {}", len, putObjectRequest.key());
incrementPutStartStatistics(len);
try {
- PutObjectResult result = trackDurationOfSupplier(
- nonNullDurationTrackerFactory(durationTrackerFactory),
- OBJECT_PUT_REQUESTS.getSymbol(), () ->
- s3.putObject(putObjectRequest));
+ PutObjectResponse response =
+ trackDurationOfSupplier(nonNullDurationTrackerFactory(durationTrackerFactory),
+ OBJECT_PUT_REQUESTS.getSymbol(),
+ () -> isFile ?
+ s3Client.putObject(putObjectRequest, RequestBody.fromFile(uploadData.getFile())) :
+ s3Client.putObject(putObjectRequest,
+ RequestBody.fromInputStream(uploadData.getUploadStream(),
+ putObjectRequest.contentLength())));
incrementPutCompletedStatistics(true, len);
// apply any post-write actions.
- finishedWrite(putObjectRequest.getKey(), len,
- result.getETag(), result.getVersionId(),
+ finishedWrite(putObjectRequest.key(), len,
+ response.eTag(), response.versionId(),
putOptions);
- return result;
- } catch (SdkBaseException e) {
+ return response;
+ } catch (SdkException e) {
incrementPutCompletedStatistics(false, len);
throw e;
}
@@ -3009,12 +3177,8 @@ PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest,
* @throws IllegalArgumentException if the length is negative
*/
private long getPutRequestLength(PutObjectRequest putObjectRequest) {
- long len;
- if (putObjectRequest.getFile() != null) {
- len = putObjectRequest.getFile().length();
- } else {
- len = putObjectRequest.getMetadata().getContentLength();
- }
+ long len = putObjectRequest.contentLength();
+
Preconditions.checkState(len >= 0, "Cannot PUT object of unknown length");
return len;
}
@@ -3022,28 +3186,29 @@ private long getPutRequestLength(PutObjectRequest putObjectRequest) {
/**
* Upload part of a multi-partition file.
* Increments the write and put counters.
- * Important: this call does not close any input stream in the request.
+ * Important: this call does not close any input stream in the body.
*
* Retry Policy: none.
- * @param request request
* @param durationTrackerFactory duration tracker factory for operation
+ * @param request the upload part request.
+ * @param body the request body.
* @return the result of the operation.
- * @throws AmazonClientException on problems
+ * @throws AwsServiceException on problems
*/
@Retries.OnceRaw
- UploadPartResult uploadPart(UploadPartRequest request,
+ UploadPartResponse uploadPart(UploadPartRequest request, RequestBody body,
final DurationTrackerFactory durationTrackerFactory)
- throws AmazonClientException {
- long len = request.getPartSize();
+ throws AwsServiceException {
+ long len = request.contentLength();
incrementPutStartStatistics(len);
try {
- UploadPartResult uploadPartResult = trackDurationOfSupplier(
+ UploadPartResponse uploadPartResponse = trackDurationOfSupplier(
nonNullDurationTrackerFactory(durationTrackerFactory),
MULTIPART_UPLOAD_PART_PUT.getSymbol(), () ->
- s3.uploadPart(request));
+ s3Client.uploadPart(request, body));
incrementPutCompletedStatistics(true, len);
- return uploadPartResult;
- } catch (AmazonClientException e) {
+ return uploadPartResponse;
+ } catch (AwsServiceException e) {
incrementPutCompletedStatistics(false, len);
throw e;
}
@@ -3108,56 +3273,57 @@ public void incrementPutProgressStatistics(String key, long bytes) {
* be deleted in a multiple object delete operation.
* The number of rejected objects will be added to the metric
* {@link Statistic#FILES_DELETE_REJECTED}.
- * @throws AmazonClientException other amazon-layer failure.
+ * @throws AwsServiceException other amazon-layer failure.
*/
@Retries.RetryRaw
private void removeKeysS3(
- List keysToDelete,
+ List keysToDelete,
boolean deleteFakeDir)
- throws MultiObjectDeleteException, AmazonClientException,
- IOException {
+ throws MultiObjectDeleteException, AwsServiceException, IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Initiating delete operation for {} objects",
keysToDelete.size());
- for (DeleteObjectsRequest.KeyVersion key : keysToDelete) {
- LOG.debug(" {} {}", key.getKey(),
- key.getVersion() != null ? key.getVersion() : "");
+ for (ObjectIdentifier objectIdentifier : keysToDelete) {
+ LOG.debug(" {} {}", objectIdentifier.key(),
+ objectIdentifier.versionId() != null ? objectIdentifier.versionId() : "");
}
}
if (keysToDelete.isEmpty()) {
// exit fast if there are no keys to delete
return;
}
- for (DeleteObjectsRequest.KeyVersion keyVersion : keysToDelete) {
- blockRootDelete(keyVersion.getKey());
+ for (ObjectIdentifier objectIdentifier : keysToDelete) {
+ blockRootDelete(objectIdentifier.key());
}
try {
if (enableMultiObjectsDelete) {
if (keysToDelete.size() <= pageSize) {
deleteObjects(getRequestFactory()
- .newBulkDeleteRequest(keysToDelete));
+ .newBulkDeleteRequestBuilder(keysToDelete)
+ .build());
} else {
// Multi object deletion of more than 1000 keys is not supported
// by s3. So we are paging the keys by page size.
LOG.debug("Partitioning the keys to delete as it is more than " +
"page size. Number of keys: {}, Page size: {}",
keysToDelete.size(), pageSize);
- for (List batchOfKeysToDelete :
+ for (List batchOfKeysToDelete :
Lists.partition(keysToDelete, pageSize)) {
deleteObjects(getRequestFactory()
- .newBulkDeleteRequest(batchOfKeysToDelete));
+ .newBulkDeleteRequestBuilder(batchOfKeysToDelete)
+ .build());
}
}
} else {
- for (DeleteObjectsRequest.KeyVersion keyVersion : keysToDelete) {
- deleteObject(keyVersion.getKey());
+ for (ObjectIdentifier objectIdentifier : keysToDelete) {
+ deleteObject(objectIdentifier.key());
}
}
} catch (MultiObjectDeleteException ex) {
// partial delete.
// Update the stats with the count of the actual number of successful
// deletions.
- int rejected = ex.getErrors().size();
+ int rejected = ex.errors().size();
noteDeleted(keysToDelete.size() - rejected, deleteFakeDir);
incrementStatistic(FILES_DELETE_REJECTED, rejected);
throw ex;
@@ -3190,15 +3356,15 @@ private void noteDeleted(final int count, final boolean deleteFakeDir) {
* a mistaken attempt to delete the root directory.
* @throws MultiObjectDeleteException one or more of the keys could not
* be deleted in a multiple object delete operation.
- * @throws AmazonClientException amazon-layer failure.
+ * @throws AwsServiceException amazon-layer failure.
* @throws IOException other IO Exception.
*/
@VisibleForTesting
@Retries.RetryRaw
public void removeKeys(
- final List keysToDelete,
+ final List keysToDelete,
final boolean deleteFakeDir)
- throws MultiObjectDeleteException, AmazonClientException,
+ throws MultiObjectDeleteException, AwsServiceException,
IOException {
try (DurationInfo ignored = new DurationInfo(LOG, false,
"Deleting %d keys", keysToDelete.size())) {
@@ -3246,14 +3412,17 @@ protected boolean deleteWithoutCloseCheck(Path f, boolean recursive) throws IOEx
// span covers delete, getFileStatus, fake directory operations.
try (AuditSpan span = createSpan(INVOCATION_DELETE.getSymbol(),
path.toString(), null)) {
+ // SC will include active span
+ final StoreContext storeContext = createStoreContext();
boolean outcome = trackDuration(getDurationTrackerFactory(),
INVOCATION_DELETE.getSymbol(),
new DeleteOperation(
- createStoreContext(),
+ storeContext,
innerGetFileStatus(path, true, StatusProbeEnum.ALL),
recursive,
- new OperationCallbacksImpl(),
- pageSize));
+ new OperationCallbacksImpl(storeContext),
+ pageSize,
+ dirOperationsPurgeUploads));
if (outcome) {
try {
maybeCreateFakeParentDirectory(path);
@@ -3268,7 +3437,7 @@ protected boolean deleteWithoutCloseCheck(Path f, boolean recursive) throws IOEx
LOG.debug("Couldn't delete {} - does not exist: {}", path, e.toString());
instrumentation.errorIgnored();
return false;
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("delete", path, e);
}
}
@@ -3282,7 +3451,7 @@ protected boolean deleteWithoutCloseCheck(Path f, boolean recursive) throws IOEx
*/
@Retries.RetryTranslated
private void createFakeDirectoryIfNecessary(Path f)
- throws IOException, AmazonClientException {
+ throws IOException, SdkException {
String key = pathToKey(f);
// we only make the LIST call; the codepaths to get here should not
// be reached if there is an empty dir marker -and if they do, it
@@ -3302,7 +3471,7 @@ private void createFakeDirectoryIfNecessary(Path f)
@Retries.RetryTranslated
@VisibleForTesting
protected void maybeCreateFakeParentDirectory(Path path)
- throws IOException, AmazonClientException {
+ throws IOException, SdkException {
Path parent = path.getParent();
if (parent != null && !parent.isRoot() && !isUnderMagicCommitPath(parent)) {
createFakeDirectoryIfNecessary(parent);
@@ -3356,11 +3525,11 @@ public FileStatus[] listStatus(Path f) throws FileNotFoundException,
* @return the statuses of the files/directories in the given patch
* @throws FileNotFoundException when the path does not exist;
* @throws IOException due to an IO problem.
- * @throws AmazonClientException on failures inside the AWS SDK
+ * @throws SdkException on failures inside the AWS SDK
*/
private RemoteIterator innerListStatus(Path f)
throws FileNotFoundException,
- IOException, AmazonClientException {
+ IOException, SdkException {
Path path = qualify(f);
LOG.debug("List status for path: {}", path);
@@ -3424,15 +3593,15 @@ public S3ListRequest createListObjectsRequest(String key,
private S3ListRequest createListObjectsRequest(String key,
String delimiter, int limit) {
if (!useListV1) {
- ListObjectsV2Request request =
- getRequestFactory().newListObjectsV2Request(
+ ListObjectsV2Request.Builder requestBuilder =
+ getRequestFactory().newListObjectsV2RequestBuilder(
key, delimiter, limit);
- return S3ListRequest.v2(request);
+ return S3ListRequest.v2(requestBuilder.build());
} else {
- ListObjectsRequest request =
- getRequestFactory().newListObjectsV1Request(
+ ListObjectsRequest.Builder requestBuilder =
+ getRequestFactory().newListObjectsV1RequestBuilder(
key, delimiter, limit);
- return S3ListRequest.v1(request);
+ return S3ListRequest.v1(requestBuilder.build());
}
}
@@ -3720,31 +3889,31 @@ S3AFileStatus s3GetFileStatus(final Path path,
&& probes.contains(StatusProbeEnum.Head)) {
try {
// look for the simple file
- ObjectMetadata meta = getObjectMetadata(key);
+ HeadObjectResponse meta = getObjectMetadata(key);
LOG.debug("Found exact file: normal file {}", key);
- long contentLength = meta.getContentLength();
+ long contentLength = meta.contentLength();
// check if CSE is enabled, then strip padded length.
- if (isCSEEnabled
- && meta.getUserMetaDataOf(Headers.CRYPTO_CEK_ALGORITHM) != null
+ if (isCSEEnabled &&
+ meta.metadata().get(AWSHeaders.CRYPTO_CEK_ALGORITHM) != null
&& contentLength >= CSE_PADDING_LENGTH) {
contentLength -= CSE_PADDING_LENGTH;
}
return new S3AFileStatus(contentLength,
- dateToLong(meta.getLastModified()),
+ meta.lastModified().toEpochMilli(),
path,
getDefaultBlockSize(path),
username,
- meta.getETag(),
- meta.getVersionId());
- } catch (AmazonServiceException e) {
+ meta.eTag(),
+ meta.versionId());
+ } catch (AwsServiceException e) {
// if the response is a 404 error, it just means that there is
// no file at that path...the remaining checks will be needed.
// But: an empty bucket is also a 404, so check for that
// and fail.
- if (e.getStatusCode() != SC_404 || isUnknownBucket(e)) {
+ if (e.statusCode() != SC_404_NOT_FOUND || isUnknownBucket(e)) {
throw translateException("getFileStatus", path, e);
}
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("getFileStatus", path, e);
}
}
@@ -3787,11 +3956,11 @@ S3AFileStatus s3GetFileStatus(final Path path,
LOG.debug("Found root directory");
return new S3AFileStatus(Tristate.TRUE, path, username);
}
- } catch (AmazonServiceException e) {
- if (e.getStatusCode() != SC_404 || isUnknownBucket(e)) {
+ } catch (AwsServiceException e) {
+ if (e.statusCode() != SC_404_NOT_FOUND || isUnknownBucket(e)) {
throw translateException("getFileStatus", path, e);
}
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("getFileStatus", path, e);
}
}
@@ -3835,7 +4004,7 @@ private boolean s3Exists(final Path path, final Set probes)
* @throws IOException IO problem
* @throws FileAlreadyExistsException the destination file exists and
* overwrite==false
- * @throws AmazonClientException failure in the AWS SDK
+ * @throws SdkException failure in the AWS SDK
*/
@Override
@AuditEntryPoint
@@ -3890,13 +4059,12 @@ public void copyLocalFileFromTo(File file, Path from, Path to) throws IOExceptio
to,
() -> {
final String key = pathToKey(to);
- final ObjectMetadata om = newObjectMetadata(file.length());
Progressable progress = null;
- PutObjectRequest putObjectRequest = newPutObjectRequest(key, om, file);
- S3AFileSystem.this.invoker.retry(
- "putObject(" + "" + ")", to.toString(),
- true,
- () -> executePut(putObjectRequest, progress, putOptionsForPath(to)));
+ PutObjectRequest.Builder putObjectRequestBuilder =
+ newPutObjectRequestBuilder(key, file.length(), false);
+ S3AFileSystem.this.invoker.retry("putObject(" + "" + ")", to.toString(), true,
+ () -> executePut(putObjectRequestBuilder.build(), progress, putOptionsForPath(to),
+ file));
return null;
});
@@ -3921,40 +4089,35 @@ public boolean createEmptyDir(Path path, StoreContext storeContext)
/**
* Execute a PUT via the transfer manager, blocking for completion.
- * If the waiting for completion is interrupted, the upload will be
- * aborted before an {@code InterruptedIOException} is thrown.
* @param putObjectRequest request
* @param progress optional progress callback
* @param putOptions put object options
* @return the upload result
- * @throws InterruptedIOException if the blocking was interrupted.
+ * @throws IOException IO failure
*/
@Retries.OnceRaw("For PUT; post-PUT actions are RetrySwallowed")
- UploadResult executePut(
+ PutObjectResponse executePut(
final PutObjectRequest putObjectRequest,
final Progressable progress,
- final PutObjectOptions putOptions)
- throws InterruptedIOException {
- String key = putObjectRequest.getKey();
+ final PutObjectOptions putOptions,
+ final File file)
+ throws IOException {
+ String key = putObjectRequest.key();
long len = getPutRequestLength(putObjectRequest);
- UploadInfo info = putObject(putObjectRequest);
- Upload upload = info.getUpload();
- ProgressableProgressListener listener = new ProgressableProgressListener(
- this, key, upload, progress);
- upload.addProgressListener(listener);
- UploadResult result = waitForUploadCompletion(key, info);
- listener.uploadCompleted();
+ ProgressableProgressListener listener =
+ new ProgressableProgressListener(this, putObjectRequest.key(), progress);
+ UploadInfo info = putObject(putObjectRequest, file, listener);
+ PutObjectResponse result = waitForUploadCompletion(key, info).response();
+ listener.uploadCompleted(info.getFileUpload());
// post-write actions
finishedWrite(key, len,
- result.getETag(), result.getVersionId(), putOptions);
+ result.eTag(), result.versionId(), putOptions);
return result;
}
/**
* Wait for an upload to complete.
- * If the waiting for completion is interrupted, the upload will be
- * aborted before an {@code InterruptedIOException} is thrown.
* If the upload (or its result collection) failed, this is where
* the failure is raised as an AWS exception.
* Calls {@link #incrementPutCompletedStatistics(boolean, long)}
@@ -3962,24 +4125,20 @@ UploadResult executePut(
* @param key destination key
* @param uploadInfo upload to wait for
* @return the upload result
- * @throws InterruptedIOException if the blocking was interrupted.
+ * @throws IOException IO failure
*/
@Retries.OnceRaw
- UploadResult waitForUploadCompletion(String key, UploadInfo uploadInfo)
- throws InterruptedIOException {
- Upload upload = uploadInfo.getUpload();
+ CompletedFileUpload waitForUploadCompletion(String key, UploadInfo uploadInfo)
+ throws IOException {
+ FileUpload upload = uploadInfo.getFileUpload();
try {
- UploadResult result = upload.waitForUploadResult();
+ CompletedFileUpload result = upload.completionFuture().join();
incrementPutCompletedStatistics(true, uploadInfo.getLength());
return result;
- } catch (InterruptedException e) {
+ } catch (CompletionException e) {
LOG.info("Interrupted: aborting upload");
incrementPutCompletedStatistics(false, uploadInfo.getLength());
- upload.abort();
- throw (InterruptedIOException)
- new InterruptedIOException("Interrupted in PUT to "
- + keyToQualifiedPath(key))
- .initCause(e);
+ throw extractException("upload", key, e);
}
}
@@ -4072,17 +4231,13 @@ public void close() throws IOException {
* both the expected state of this FS and of failures while being stopped.
*/
protected synchronized void stopAllServices() {
- // shutting down the transfer manager also shuts
- // down the S3 client it is bonded to.
- if (transfers != null) {
- try {
- transfers.shutdownNow(true);
- } catch (RuntimeException e) {
- // catch and swallow for resilience.
- LOG.debug("When shutting down", e);
- }
- transfers = null;
- }
+ closeAutocloseables(LOG, transferManager,
+ s3Client,
+ getS3AsyncClient());
+ transferManager = null;
+ s3Client = null;
+ s3AsyncClient = null;
+
// At this point the S3A client is shut down,
// now the executor pools are closed
HadoopExecutors.shutdown(boundedThreadPool, LOG,
@@ -4108,13 +4263,13 @@ protected synchronized void stopAllServices() {
}
/**
- * Verify that the input stream is open. Non blocking; this gives
+ * Verify that the filesystem has not been closed. Non blocking; this gives
* the last state of the volatile {@link #closed} field.
- * @throws IOException if the connection is closed.
+ * @throws PathIOException if the FS is closed.
*/
- private void checkNotClosed() throws IOException {
+ private void checkNotClosed() throws PathIOException {
if (isClosed) {
- throw new IOException(uri + ": " + E_FS_CLOSED);
+ throw new PathIOException(uri.toString(), E_FS_CLOSED);
}
}
@@ -4218,7 +4373,7 @@ public List listAWSPolicyRules(
// no attempt is made to qualify KMS access; there's no
// way to predict read keys, and not worried about granting
// too much encryption access.
- statements.add(STATEMENT_ALLOW_SSE_KMS_RW);
+ statements.add(STATEMENT_ALLOW_KMS_RW);
return statements;
}
@@ -4237,21 +4392,11 @@ public List listAWSPolicyRules(
* @throws IOException Other IO problems
*/
@Retries.RetryTranslated
- private CopyResult copyFile(String srcKey, String dstKey, long size,
+ private CopyObjectResponse copyFile(String srcKey, String dstKey, long size,
S3ObjectAttributes srcAttributes, S3AReadOpContext readContext)
- throws IOException, InterruptedIOException {
+ throws IOException {
LOG.debug("copyFile {} -> {} ", srcKey, dstKey);
- ProgressListener progressListener = progressEvent -> {
- switch (progressEvent.getEventType()) {
- case TRANSFER_PART_COMPLETED_EVENT:
- incrementWriteOperations();
- break;
- default:
- break;
- }
- };
-
ChangeTracker changeTracker = new ChangeTracker(
keyToQualifiedPath(srcKey).toString(),
changeDetectionPolicy,
@@ -4263,7 +4408,7 @@ private CopyResult copyFile(String srcKey, String dstKey, long size,
String action = "copyFile(" + srcKey + ", " + dstKey + ")";
Invoker readInvoker = readContext.getReadInvoker();
- ObjectMetadata srcom;
+ HeadObjectResponse srcom;
try {
srcom = once(action, srcKey,
() ->
@@ -4274,7 +4419,6 @@ private CopyResult copyFile(String srcKey, String dstKey, long size,
// This means the File was deleted since LIST enumerated it.
LOG.debug("getObjectMetadata({}) failed to find an expected file",
srcKey, e);
- // We create an exception, but the text depends on the S3Guard state
throw new RemoteFileChangedException(
keyToQualifiedPath(srcKey).toString(),
action,
@@ -4282,38 +4426,68 @@ private CopyResult copyFile(String srcKey, String dstKey, long size,
e);
}
- return readInvoker.retry(
- action, srcKey,
- true,
- () -> {
- CopyObjectRequest copyObjectRequest =
- getRequestFactory().newCopyObjectRequest(srcKey, dstKey, srcom);
- changeTracker.maybeApplyConstraint(copyObjectRequest);
- incrementStatistic(OBJECT_COPY_REQUESTS);
- Copy copy = transfers.copy(copyObjectRequest,
- getAuditManager().createStateChangeListener());
- copy.addProgressListener(progressListener);
- CopyOutcome copyOutcome = CopyOutcome.waitForCopy(copy);
- InterruptedException interruptedException =
- copyOutcome.getInterruptedException();
- if (interruptedException != null) {
- // copy interrupted: convert to an IOException.
- throw (IOException)new InterruptedIOException(
- "Interrupted copying " + srcKey
- + " to " + dstKey + ", cancelling")
- .initCause(interruptedException);
- }
- SdkBaseException awsException = copyOutcome.getAwsException();
- if (awsException != null) {
- changeTracker.processException(awsException, "copy");
- throw awsException;
- }
- CopyResult result = copyOutcome.getCopyResult();
- changeTracker.processResponse(result);
- incrementWriteOperations();
- instrumentation.filesCopied(1, size);
- return result;
- });
+ CopyObjectRequest.Builder copyObjectRequestBuilder =
+ getRequestFactory().newCopyObjectRequestBuilder(srcKey, dstKey, srcom);
+ changeTracker.maybeApplyConstraint(copyObjectRequestBuilder);
+ final CopyObjectRequest copyRequest = copyObjectRequestBuilder.build();
+ LOG.debug("Copy Request: {}", copyRequest);
+ CopyObjectResponse response;
+
+ // transfer manager is skipped if disabled or the file is too small to worry about
+ final boolean useTransferManager = isMultipartCopyEnabled && size >= multiPartThreshold;
+ if (useTransferManager) {
+ // use transfer manager
+ response = readInvoker.retry(
+ action, srcKey,
+ true,
+ () -> {
+ incrementStatistic(OBJECT_COPY_REQUESTS);
+
+ Copy copy = transferManager.copy(
+ CopyRequest.builder()
+ .copyObjectRequest(copyRequest)
+ .build());
+
+ try {
+ CompletedCopy completedCopy = copy.completionFuture().join();
+ return completedCopy.response();
+ } catch (CompletionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof SdkException) {
+ // if this is a 412 precondition failure, it may
+ // be converted to a RemoteFileChangedException
+ SdkException awsException = (SdkException)cause;
+ changeTracker.processException(awsException, "copy");
+ throw awsException;
+ }
+ throw extractException(action, srcKey, e);
+ }
+ });
+ } else {
+ // single part copy bypasses transfer manager
+ // note, this helps with some mock testing, e.g. HBoss. as there is less to mock.
+ response = readInvoker.retry(
+ action, srcKey,
+ true,
+ () -> {
+ LOG.debug("copyFile: single part copy {} -> {} of size {}", srcKey, dstKey, size);
+ incrementStatistic(OBJECT_COPY_REQUESTS);
+ try {
+ return s3Client.copyObject(copyRequest);
+ } catch (SdkException awsException) {
+ // if this is a 412 precondition failure, it may
+ // be converted to a RemoteFileChangedException
+ changeTracker.processException(awsException, "copy");
+ // otherwise, rethrow
+ throw awsException;
+ }
+ });
+ }
+
+ changeTracker.processResponse(response);
+ incrementWriteOperations();
+ instrumentation.filesCopied(1, size);
+ return response;
}
/**
@@ -4321,16 +4495,16 @@ private CopyResult copyFile(String srcKey, String dstKey, long size,
* Retry policy: none + untranslated.
* @param request request to initiate
* @return the result of the call
- * @throws AmazonClientException on failures inside the AWS SDK
+ * @throws SdkException on failures inside the AWS SDK
* @throws IOException Other IO problems
*/
@Retries.OnceRaw
- InitiateMultipartUploadResult initiateMultipartUpload(
- InitiateMultipartUploadRequest request) throws IOException {
- LOG.debug("Initiate multipart upload to {}", request.getKey());
+ CreateMultipartUploadResponse initiateMultipartUpload(
+ CreateMultipartUploadRequest request) throws IOException {
+ LOG.debug("Initiate multipart upload to {}", request.key());
return trackDurationOfSupplier(getDurationTrackerFactory(),
OBJECT_MULTIPART_UPLOAD_INITIATED.getSymbol(),
- () -> getAmazonS3Client().initiateMultipartUpload(request));
+ () -> s3Client.createMultipartUpload(request));
}
/**
@@ -4403,22 +4577,22 @@ private PutObjectOptions putOptionsForPath(Path path) {
*/
@Retries.RetryExceptionsSwallowed
private void deleteUnnecessaryFakeDirectories(Path path) {
- List keysToRemove = new ArrayList<>();
+ List keysToRemove = new ArrayList<>();
while (!path.isRoot()) {
String key = pathToKey(path);
key = (key.endsWith("/")) ? key : (key + "/");
LOG.trace("To delete unnecessary fake directory {} for {}", key, path);
- keysToRemove.add(new DeleteObjectsRequest.KeyVersion(key));
+ keysToRemove.add(ObjectIdentifier.builder().key(key).build());
path = path.getParent();
}
try {
removeKeys(keysToRemove, true);
- } catch(AmazonClientException | IOException e) {
+ } catch (AwsServiceException | IOException e) {
instrumentation.errorIgnored();
if (LOG.isDebugEnabled()) {
StringBuilder sb = new StringBuilder();
- for(DeleteObjectsRequest.KeyVersion kv : keysToRemove) {
- sb.append(kv.getKey()).append(",");
+ for (ObjectIdentifier objectIdentifier : keysToRemove) {
+ sb.append(objectIdentifier.key()).append(",");
}
LOG.debug("While deleting keys {} ", sb.toString(), e);
}
@@ -4451,11 +4625,18 @@ private void createFakeDirectory(final String objectName,
@Retries.RetryTranslated
private void createEmptyObject(final String objectName, PutObjectOptions putOptions)
throws IOException {
- invoker.retry("PUT 0-byte object ", objectName,
- true, () ->
- putObjectDirect(getRequestFactory().newDirectoryMarkerRequest(objectName),
- putOptions,
- getDurationTrackerFactory()));
+ final InputStream im = new InputStream() {
+ @Override
+ public int read() throws IOException {
+ return -1;
+ }
+ };
+
+ S3ADataBlocks.BlockUploadData uploadData = new S3ADataBlocks.BlockUploadData(im);
+
+ invoker.retry("PUT 0-byte object ", objectName, true,
+ () -> putObjectDirect(getRequestFactory().newDirectoryMarkerRequest(objectName).build(),
+ putOptions, uploadData, false, getDurationTrackerFactory()));
incrementPutProgressStatistics(objectName, 0);
instrumentation.directoryCreated();
}
@@ -4712,10 +4893,10 @@ public EtagChecksum getFileChecksum(Path f, final long length)
ETAG_CHECKSUM_ENABLED_DEFAULT)) {
return trackDurationAndSpan(INVOCATION_GET_FILE_CHECKSUM, path, () -> {
LOG.debug("getFileChecksum({})", path);
- ObjectMetadata headers = getObjectMetadata(path, null,
+ HeadObjectResponse headers = getObjectMetadata(path, null,
invoker,
"getFileChecksum are");
- String eTag = headers.getETag();
+ String eTag = headers.eTag();
return eTag != null ? new EtagChecksum(eTag) : null;
});
} else {
@@ -4797,11 +4978,18 @@ protected final class HeaderProcessingCallbacksImpl implements
HeaderProcessing.HeaderProcessingCallbacks {
@Override
- public ObjectMetadata getObjectMetadata(final String key)
+ public HeadObjectResponse getObjectMetadata(final String key)
throws IOException {
return once("getObjectMetadata", key, () ->
S3AFileSystem.this.getObjectMetadata(key));
}
+
+ @Override
+ public HeadBucketResponse getBucketMetadata()
+ throws IOException {
+ return once("getBucketMetadata", bucket, () ->
+ S3AFileSystem.this.getBucketMetadata());
+ }
}
/**
* {@inheritDoc}.
@@ -4914,7 +5102,7 @@ private RemoteIterator innerListFiles(
// If we have reached here, it means either there are files
// in this directory or it is empty.
return listFilesAssumingDir;
- } catch (AmazonClientException e) {
+ } catch (SdkException e) {
throw translateException("listFiles", path, e);
}
}
@@ -5007,14 +5195,39 @@ S3ALocatedFileStatus toLocatedFileStatus(S3AFileStatus status)
@InterfaceAudience.Private
@Retries.RetryTranslated
@AuditEntryPoint
- public MultipartUtils.UploadIterator listUploads(@Nullable String prefix)
+ public RemoteIterator listUploads(@Nullable String prefix)
throws IOException {
// span is picked up retained in the listing.
- return trackDurationAndSpan(MULTIPART_UPLOAD_LIST, prefix, null, () ->
- MultipartUtils.listMultipartUploads(
- createStoreContext(),
- s3, prefix, maxKeys
- ));
+ checkNotClosed();
+ try (AuditSpan span = createSpan(MULTIPART_UPLOAD_LIST.getSymbol(),
+ prefix, null)) {
+ return listUploadsUnderPrefix(createStoreContext(), prefix);
+ }
+ }
+
+ /**
+ * List any pending multipart uploads whose keys begin with prefix, using
+ * an iterator that can handle an unlimited number of entries.
+ * See {@link #listMultipartUploads(String)} for a non-iterator version of
+ * this.
+ * @param storeContext store conext.
+ * @param prefix optional key prefix to search
+ * @return Iterator over multipart uploads.
+ * @throws IOException on failure
+ */
+ @InterfaceAudience.Private
+ @Retries.RetryTranslated
+ public RemoteIterator listUploadsUnderPrefix(
+ final StoreContext storeContext,
+ final @Nullable String prefix)
+ throws IOException {
+ // span is picked up retained in the listing.
+ String p = prefix;
+ if (prefix != null && !prefix.isEmpty() && !prefix.endsWith("/")) {
+ p = prefix + "/";
+ }
+ // duration tracking is done in iterator.
+ return MultipartUtils.listMultipartUploads(storeContext, s3Client, p, maxKeys);
}
/**
@@ -5024,7 +5237,7 @@ public MultipartUtils.UploadIterator listUploads(@Nullable String prefix)
* Retry policy: retry, translated.
* @return a listing of multipart uploads.
* @param prefix prefix to scan for, "" for none
- * @throws IOException IO failure, including any uprated AmazonClientException
+ * @throws IOException IO failure, including any uprated SdkException
*/
@InterfaceAudience.Private
@Retries.RetryTranslated
@@ -5036,9 +5249,10 @@ public List listMultipartUploads(String prefix)
}
String p = prefix;
return invoker.retry("listMultipartUploads", p, true, () -> {
- ListMultipartUploadsRequest request = getRequestFactory()
- .newListMultipartUploadsRequest(p);
- return s3.listMultipartUploads(request).getMultipartUploads();
+ final ListMultipartUploadsRequest request = getRequestFactory()
+ .newListMultipartUploadsRequestBuilder(p).build();
+ return trackDuration(getInstrumentation(), MULTIPART_UPLOAD_LIST.getSymbol(), () ->
+ s3Client.listMultipartUploads(request).uploads());
});
}
@@ -5047,37 +5261,35 @@ public List listMultipartUploads(String prefix)
* Retry policy: none.
* @param destKey destination key
* @param uploadId Upload ID
+ * @throws IOException IO failure, including any uprated SdkException
*/
- @Retries.OnceRaw
- void abortMultipartUpload(String destKey, String uploadId) {
- LOG.info("Aborting multipart upload {} to {}", uploadId, destKey);
- getAmazonS3Client().abortMultipartUpload(
- getRequestFactory().newAbortMultipartUploadRequest(
- destKey,
- uploadId));
+ @Retries.OnceTranslated
+ public void abortMultipartUpload(String destKey, String uploadId) throws IOException {
+ LOG.debug("Aborting multipart upload {} to {}", uploadId, destKey);
+ trackDuration(getInstrumentation(), OBJECT_MULTIPART_UPLOAD_ABORTED.getSymbol(), () ->
+ s3Client.abortMultipartUpload(
+ getRequestFactory().newAbortMultipartUploadRequestBuilder(
+ destKey,
+ uploadId).build()));
}
/**
* Abort a multipart upload.
* Retry policy: none.
* @param upload the listed upload to abort.
+ * @throws IOException IO failure, including any uprated SdkException
*/
- @Retries.OnceRaw
- void abortMultipartUpload(MultipartUpload upload) {
- String destKey;
- String uploadId;
- destKey = upload.getKey();
- uploadId = upload.getUploadId();
- if (LOG.isInfoEnabled()) {
+ @Retries.OnceTranslated
+ public void abortMultipartUpload(MultipartUpload upload) throws IOException {
+ String destKey = upload.key();
+ String uploadId = upload.uploadId();
+ if (LOG.isDebugEnabled()) {
DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
LOG.debug("Aborting multipart upload {} to {} initiated by {} on {}",
- uploadId, destKey, upload.getInitiator(),
- df.format(upload.getInitiated()));
+ uploadId, destKey, upload.initiator(),
+ df.format(Date.from(upload.initiated())));
}
- getAmazonS3Client().abortMultipartUpload(
- getRequestFactory().newAbortMultipartUploadRequest(
- destKey,
- uploadId));
+ abortMultipartUpload(destKey, uploadId);
}
/**
@@ -5123,13 +5335,17 @@ public boolean hasPathCapability(final Path path, final String capability)
case STORE_CAPABILITY_DIRECTORY_MARKER_AWARE:
return true;
+ // Do directory operations purge uploads.
+ case DIRECTORY_OPERATIONS_PURGE_UPLOADS:
+ return dirOperationsPurgeUploads;
+
// etags are avaialable in listings, but they
// are not consistent across renames.
// therefore, only availability is declared
case CommonPathCapabilities.ETAGS_AVAILABLE:
return true;
- /*
+ /*
* Marker policy capabilities are handed off.
*/
case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_KEEP:
@@ -5399,7 +5615,7 @@ public MarkerToolOperations createMarkerToolOperations(final String target)
throws IOException {
createSpan("marker-tool-scan", target,
null);
- return new MarkerToolOperationsImpl(new OperationCallbacksImpl());
+ return new MarkerToolOperationsImpl(new OperationCallbacksImpl(createStoreContext()));
}
/**
@@ -5464,4 +5680,5 @@ public boolean isCSEEnabled() {
public boolean isMultipartUploadEnabled() {
return isMultipartUploadEnabled;
}
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index 4b50ab2c04bd9..2ed9083efcddd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -22,6 +22,7 @@
import java.io.Closeable;
import java.io.EOFException;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InterruptedIOException;
import java.net.SocketTimeoutException;
import java.nio.ByteBuffer;
@@ -31,9 +32,9 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.IntFunction;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,7 +47,6 @@
import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileRange;
-import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.fs.impl.CombinedFileRange;
import org.apache.hadoop.fs.VectoredReadUtils;
@@ -61,6 +61,7 @@
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.functional.CallableRaisingIOE;
+
import static java.util.Objects.requireNonNull;
import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import static org.apache.hadoop.fs.VectoredReadUtils.isOrderedDisjoint;
@@ -125,14 +126,9 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead,
*/
private volatile boolean closed;
/**
- * wrappedStream is associated with an object (instance of S3Object). When
- * the object is garbage collected, the associated wrappedStream will be
- * closed. Keep a reference to this object to prevent the wrapperStream
- * still in use from being closed unexpectedly due to garbage collection.
- * See HADOOP-17338 for details.
+ * Input stream returned by a getObject call.
*/
- private S3Object object;
- private S3ObjectInputStream wrappedStream;
+ private ResponseInputStream wrappedStream;
private final S3AReadOpContext context;
private final InputStreamCallbacks client;
@@ -271,28 +267,22 @@ private synchronized void reopen(String reason, long targetPos, long length,
uri, reason, targetPos, contentRangeFinish, length, pos, nextReadPos,
inputPolicy);
+ GetObjectRequest request = client.newGetRequestBuilder(key)
+ .range(S3AUtils.formatRange(targetPos, contentRangeFinish - 1))
+ .applyMutation(changeTracker::maybeApplyConstraint)
+ .build();
long opencount = streamStatistics.streamOpened();
- GetObjectRequest request = client.newGetRequest(key)
- .withRange(targetPos, contentRangeFinish - 1);
String operation = opencount == 0 ? OPERATION_OPEN : OPERATION_REOPEN;
String text = String.format("%s %s at %d",
operation, uri, targetPos);
- changeTracker.maybeApplyConstraint(request);
-
- object = onceTrackingDuration(text, uri,
+ wrappedStream = onceTrackingDuration(text, uri,
streamStatistics.initiateGetRequest(), () ->
client.getObject(request));
-
- changeTracker.processResponse(object, operation,
+ changeTracker.processResponse(wrappedStream.response(), operation,
targetPos);
- wrappedStream = object.getObjectContent();
- contentRangeStart = targetPos;
- if (wrappedStream == null) {
- throw new PathIOException(uri,
- "Null IO stream from " + operation + " of (" + reason + ") ");
- }
+ contentRangeStart = targetPos;
this.pos = targetPos;
}
@@ -505,14 +495,15 @@ public synchronized int read() throws IOException {
*/
@Retries.OnceTranslated
private void onReadFailure(IOException ioe, boolean forceAbort) {
+ GetObjectResponse objectResponse = wrappedStream == null ? null : wrappedStream.response();
if (LOG.isDebugEnabled()) {
LOG.debug("Got exception while trying to read from stream {}, " +
"client: {} object: {}, trying to recover: ",
- uri, client, object, ioe);
+ uri, client, objectResponse, ioe);
} else {
LOG.info("Got exception while trying to read from stream {}, " +
"client: {} object: {}, trying to recover: " + ioe,
- uri, client, object);
+ uri, client, objectResponse);
}
streamStatistics.readException();
closeStream("failure recovery", forceAbort, false);
@@ -672,7 +663,6 @@ private CompletableFuture closeStream(
CompletableFuture operation;
SDKStreamDrainer drainer = new SDKStreamDrainer(
uri,
- object,
wrappedStream,
shouldAbort,
(int) remaining,
@@ -694,7 +684,6 @@ private CompletableFuture closeStream(
// either the stream is closed in the blocking call or the async call is
// submitted with its own copy of the references
wrappedStream = null;
- object = null;
return operation;
}
@@ -910,23 +899,19 @@ public void readVectored(List ranges,
private void readCombinedRangeAndUpdateChildren(CombinedFileRange combinedFileRange,
IntFunction allocate) {
LOG.debug("Start reading combined range {} from path {} ", combinedFileRange, pathStr);
- // This reference must be kept till all buffers are populated as this is a
- // finalizable object which closes the internal stream when gc triggers.
- S3Object objectRange = null;
- S3ObjectInputStream objectContent = null;
+ ResponseInputStream rangeContent = null;
try {
- objectRange = getS3ObjectAndValidateNotNull("readCombinedFileRange",
+ rangeContent = getS3ObjectInputStream("readCombinedFileRange",
combinedFileRange.getOffset(),
combinedFileRange.getLength());
- objectContent = objectRange.getObjectContent();
- populateChildBuffers(combinedFileRange, objectContent, allocate);
+ populateChildBuffers(combinedFileRange, rangeContent, allocate);
} catch (Exception ex) {
LOG.debug("Exception while reading a range {} from path {} ", combinedFileRange, pathStr, ex);
for(FileRange child : combinedFileRange.getUnderlying()) {
child.getData().completeExceptionally(ex);
}
} finally {
- IOUtils.cleanupWithLogger(LOG, objectRange, objectContent);
+ IOUtils.cleanupWithLogger(LOG, rangeContent);
}
LOG.debug("Finished reading range {} from path {} ", combinedFileRange, pathStr);
}
@@ -939,7 +924,7 @@ private void readCombinedRangeAndUpdateChildren(CombinedFileRange combinedFileRa
* @throws IOException any IOE.
*/
private void populateChildBuffers(CombinedFileRange combinedFileRange,
- S3ObjectInputStream objectContent,
+ InputStream objectContent,
IntFunction allocate) throws IOException {
// If the combined file range just contains a single child
// range, we only have to fill that one child buffer else
@@ -971,7 +956,7 @@ private void populateChildBuffers(CombinedFileRange combinedFileRange,
* @param drainQuantity how many bytes to drain.
* @throws IOException any IOE.
*/
- private void drainUnnecessaryData(S3ObjectInputStream objectContent, long drainQuantity)
+ private void drainUnnecessaryData(InputStream objectContent, long drainQuantity)
throws IOException {
int drainBytes = 0;
int readCount;
@@ -1013,28 +998,24 @@ private void validateRangeRequest(FileRange range) throws EOFException {
*/
private void readSingleRange(FileRange range, ByteBuffer buffer) {
LOG.debug("Start reading range {} from path {} ", range, pathStr);
- // This reference must be kept till all buffers are populated as this is a
- // finalizable object which closes the internal stream when gc triggers.
- S3Object objectRange = null;
- S3ObjectInputStream objectContent = null;
+ ResponseInputStream objectRange = null;
try {
long position = range.getOffset();
int length = range.getLength();
- objectRange = getS3ObjectAndValidateNotNull("readSingleRange", position, length);
- objectContent = objectRange.getObjectContent();
- populateBuffer(length, buffer, objectContent);
+ objectRange = getS3ObjectInputStream("readSingleRange", position, length);
+ populateBuffer(length, buffer, objectRange);
range.getData().complete(buffer);
} catch (Exception ex) {
LOG.warn("Exception while reading a range {} from path {} ", range, pathStr, ex);
range.getData().completeExceptionally(ex);
} finally {
- IOUtils.cleanupWithLogger(LOG, objectRange, objectContent);
+ IOUtils.cleanupWithLogger(LOG, objectRange);
}
LOG.debug("Finished reading range {} from path {} ", range, pathStr);
}
/**
- * Get the s3 object for S3 server for a specified range.
+ * Get the s3 object input stream for S3 server for a specified range.
* Also checks if the vectored io operation has been stopped before and after
* the http get request such that we don't waste time populating the buffers.
* @param operationName name of the operation for which get object on S3 is called.
@@ -1043,15 +1024,11 @@ private void readSingleRange(FileRange range, ByteBuffer buffer) {
* @return result s3 object.
* @throws IOException exception if any.
*/
- private S3Object getS3ObjectAndValidateNotNull(final String operationName,
- final long position,
- final int length) throws IOException {
+ private ResponseInputStream getS3ObjectInputStream(
+ final String operationName, final long position, final int length) throws IOException {
checkIfVectoredIOStopped();
- S3Object objectRange = getS3Object(operationName, position, length);
- if (objectRange.getObjectContent() == null) {
- throw new PathIOException(uri,
- "Null IO stream received during " + operationName);
- }
+ ResponseInputStream objectRange =
+ getS3Object(operationName, position, length);
checkIfVectoredIOStopped();
return objectRange;
}
@@ -1066,7 +1043,7 @@ private S3Object getS3ObjectAndValidateNotNull(final String operationName,
*/
private void populateBuffer(int length,
ByteBuffer buffer,
- S3ObjectInputStream objectContent) throws IOException {
+ InputStream objectContent) throws IOException {
if (buffer.isDirect()) {
VectoredReadUtils.readInDirectBuffer(length, buffer,
@@ -1091,7 +1068,7 @@ private void populateBuffer(int length,
* @param length number of bytes to fill in dest.
* @throws IOException any IOE.
*/
- private void readByteArray(S3ObjectInputStream objectContent,
+ private void readByteArray(InputStream objectContent,
byte[] dest,
int offset,
int length) throws IOException {
@@ -1118,13 +1095,16 @@ private void readByteArray(S3ObjectInputStream objectContent,
* @return S3Object result s3 object.
* @throws IOException exception if any.
*/
- private S3Object getS3Object(String operationName, long position,
- int length) throws IOException {
- final GetObjectRequest request = client.newGetRequest(key)
- .withRange(position, position + length - 1);
- changeTracker.maybeApplyConstraint(request);
+ private ResponseInputStream getS3Object(String operationName,
+ long position,
+ int length)
+ throws IOException {
+ final GetObjectRequest request = client.newGetRequestBuilder(key)
+ .range(S3AUtils.formatRange(position, position + length - 1))
+ .applyMutation(changeTracker::maybeApplyConstraint)
+ .build();
DurationTracker tracker = streamStatistics.initiateGetRequest();
- S3Object objectRange;
+ ResponseInputStream objectRange;
Invoker invoker = context.getReadInvoker();
try {
objectRange = invoker.retry(operationName, pathStr, true,
@@ -1139,7 +1119,7 @@ private S3Object getS3Object(String operationName, long position,
} finally {
tracker.close();
}
- changeTracker.processResponse(objectRange, operationName,
+ changeTracker.processResponse(objectRange.response(), operationName,
position);
return objectRange;
}
@@ -1293,11 +1273,11 @@ public IOStatistics getIOStatistics() {
public interface InputStreamCallbacks extends Closeable {
/**
- * Create a GET request.
+ * Create a GET request builder.
* @param key object key
- * @return the request
+ * @return the request builder
*/
- GetObjectRequest newGetRequest(String key);
+ GetObjectRequest.Builder newGetRequestBuilder(String key);
/**
* Execute the request.
@@ -1305,7 +1285,7 @@ public interface InputStreamCallbacks extends Closeable {
* @return the response
*/
@Retries.OnceRaw
- S3Object getObject(GetObjectRequest request);
+ ResponseInputStream getObject(GetObjectRequest request);
/**
* Submit some asynchronous work, for example, draining a stream.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java
new file mode 100644
index 0000000000000..18d6c1af586ff
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+import java.nio.file.AccessDeniedException;
+
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.store.audit.AuditEntryPoint;
+
+/**
+ * This is an unstable interface for access to S3A Internal state, S3 operations
+ * and the S3 client connector itself.
+ */
+@InterfaceStability.Unstable
+@InterfaceAudience.LimitedPrivate("testing/diagnostics")
+public interface S3AInternals {
+
+ /**
+ * Returns the S3 client used by this filesystem.
+ * Will log at debug.
+ *
+ * Warning
+ * This bypasses core S3A operations, including auditing.
+ * It is intended for use in testing, diagnostics and for accessing
+ * operations not available through the S3A connector itself.
+ *
+ * Unless audit spans are created through the S3AFileSystem, make
+ * sure that {@code fs.s3a.audit.reject.out.of.span.operations} is
+ * set to false.
+ *
+ * Mocking note: this is the same S3Client as is used by the owning
+ * filesystem; changes to this client will be reflected by changes
+ * in the behavior of that filesystem.
+ * @param reason a justification for requesting access.
+ * @return S3Client
+ */
+ S3Client getAmazonS3Client(String reason);
+
+ /**
+ * Get the region of a bucket.
+ * Invoked from StoreContext; consider an entry point.
+ * @return the region in which a bucket is located
+ * @throws AccessDeniedException if the caller lacks permission.
+ * @throws IOException on any failure.
+ */
+ @Retries.RetryTranslated
+ @AuditEntryPoint
+ String getBucketLocation() throws IOException;
+
+ /**
+ * Get the region of a bucket; fixing up the region so it can be used
+ * in the builders of other AWS clients.
+ * Requires the caller to have the AWS role permission
+ * {@code s3:GetBucketLocation}.
+ * Retry policy: retrying, translated.
+ * @param bucketName the name of the bucket
+ * @return the region in which a bucket is located
+ * @throws AccessDeniedException if the caller lacks permission.
+ * @throws IOException on any failure.
+ */
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ String getBucketLocation(String bucketName) throws IOException;
+
+ /**
+ * Low-level call to get at the object metadata.
+ * Auditing: An audit entry point.
+ * @param path path to the object. This will be qualified.
+ * @return metadata
+ * @throws IOException IO and object access problems.
+ */
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ HeadObjectResponse getObjectMetadata(Path path) throws IOException;
+
+ /**
+ * Get a shared copy of the AWS credentials, with its reference
+ * counter updated.
+ * Caller is required to call {@code close()} on this after
+ * they have finished using it.
+ * @param purpose what is this for? This is for logging
+ * @return a reference to shared credentials.
+ */
+ AWSCredentialProviderList shareCredentials(String purpose);
+
+ /**
+ * Request bucket metadata.
+ * @return the metadata
+ * @throws UnknownStoreException the bucket is absent
+ * @throws IOException any other problem talking to S3
+ */
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ HeadBucketResponse getBucketMetadata() throws IOException;
+
+ /**
+ * Is multipart copy enabled?
+ * @return true if the transfer manager is used to copy files.
+ */
+ boolean isMultipartCopyEnabled();
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
index 528a99f5e0966..cc30da4fbe04e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
@@ -22,7 +22,10 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
+import java.net.BindException;
+import java.net.ConnectException;
import java.net.NoRouteToHostException;
+import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.nio.file.AccessDeniedException;
@@ -30,7 +33,7 @@
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import com.amazonaws.AmazonClientException;
+import software.amazon.awssdk.core.exception.SdkException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,15 +45,16 @@
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.net.ConnectTimeoutException;
-import org.apache.hadoop.util.Preconditions;
-import static org.apache.hadoop.io.retry.RetryPolicies.*;
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.io.retry.RetryPolicies.*;
import static org.apache.hadoop.fs.s3a.Constants.*;
+
/**
* The S3A request retry policy.
- *
+ *
* This uses the retry options in the configuration file to determine retry
* count and delays for "normal" retries and separately, for throttling;
* the latter is best handled for longer with an exponential back-off.
@@ -65,20 +69,25 @@
* For non-idempotent operations, only failures due to throttling or
* from failures which are known to only arise prior to talking to S3
* are retried.
- *
+ *
* The retry policy is all built around that of the normal IO exceptions,
* particularly those extracted from
- * {@link S3AUtils#translateException(String, Path, AmazonClientException)}.
+ * {@link S3AUtils#translateException(String, Path, SdkException)}.
* Because the {@link #shouldRetry(Exception, int, int, boolean)} method
- * does this translation if an {@code AmazonClientException} is processed,
+ * does this translation if an {@code SdkException} is processed,
* the policy defined for the IOEs also applies to the original exceptions.
- *
+ *
* Put differently: this retry policy aims to work for handlers of the
* untranslated exceptions, as well as the translated ones.
+ *
+ * Note that because delete is considered idempotent, all s3a operations currently
+ * declare themselves idempotent.
+ * This means the retry policy here is more complex than it needs to be -but it
+ * does force us to consider when retrying operations would not be safe.
* @see S3 Error responses
- * @see Amazon S3 Error Best Practices
+ * @see Amazon S3 Error Best Practices
*/
-@SuppressWarnings("visibilitymodifier") // I want a struct of finals, for real.
+@SuppressWarnings("visibilitymodifier") // we want a struct of finals, for real.
public class S3ARetryPolicy implements RetryPolicy {
private static final Logger LOG = LoggerFactory.getLogger(
@@ -111,13 +120,17 @@ public class S3ARetryPolicy implements RetryPolicy {
*/
protected final RetryPolicy connectivityFailure;
+ /**
+ * Handling of AWSClientIOException and subclasses.
+ */
+ protected final RetryPolicy retryAwsClientExceptions;
+
/**
* Instantiate.
* @param conf configuration to read.
*/
public S3ARetryPolicy(Configuration conf) {
- Preconditions.checkArgument(conf != null, "Null configuration");
- this.configuration = conf;
+ this.configuration = requireNonNull(conf, "Null configuration");
// base policy from configuration
int limit = conf.getInt(RETRY_LIMIT, RETRY_LIMIT_DEFAULT);
@@ -138,6 +151,11 @@ public S3ARetryPolicy(Configuration conf) {
retryIdempotentCalls = new FailNonIOEs(
new IdempotencyRetryFilter(baseExponentialRetry));
+ // retry on AWSClientIOException and possibly subclasses;
+ // See: HADOOP-18871. S3ARetryPolicy to use sdk exception retryable() if it is valid
+ // currently the normal retryIdempotentCalls policy is used.
+ retryAwsClientExceptions = retryIdempotentCalls;
+
// and a separate policy for throttle requests, which are considered
// repeatable, even for non-idempotent calls, as the service
// rejected the call entirely
@@ -177,35 +195,57 @@ protected Map, RetryPolicy> createExceptionMap() {
// inherit policies.
Map, RetryPolicy> policyMap = new HashMap<>();
- // failfast exceptions which we consider unrecoverable
- policyMap.put(UnknownHostException.class, fail);
- policyMap.put(NoRouteToHostException.class, fail);
- policyMap.put(InterruptedException.class, fail);
- // note this does not pick up subclasses (like socket timeout)
- policyMap.put(InterruptedIOException.class, fail);
// Access denial and auth exceptions are not retried
policyMap.put(AccessDeniedException.class, fail);
- policyMap.put(NoAuthWithAWSException.class, fail);
- policyMap.put(FileNotFoundException.class, fail);
- policyMap.put(UnknownStoreException.class, fail);
- policyMap.put(InvalidRequestException.class, fail);
- // once the file has changed, trying again is not going to help
- policyMap.put(RemoteFileChangedException.class, fail);
+ // Treated as an immediate failure
+ policyMap.put(AWSBadRequestException.class, fail);
- // likely only recovered by changing the policy configuration or s3
- // implementation
- policyMap.put(NoVersionAttributeException.class, fail);
+ // use specific retry policy for aws client exceptions
+ // nested IOExceptions will already have been extracted and used
+ // in this map.
+ policyMap.put(AWSClientIOException.class, retryAwsClientExceptions);
+
+ // server didn't respond.
+ policyMap.put(AWSNoResponseException.class, retryIdempotentCalls);
// should really be handled by resubmitting to new location;
// that's beyond the scope of this retry policy
policyMap.put(AWSRedirectException.class, fail);
+ // generic exception from the service
+ policyMap.put(AWSServiceIOException.class, retryAwsClientExceptions);
+
// throttled requests are can be retried, always
policyMap.put(AWSServiceThrottledException.class, throttlePolicy);
+ // Status 5xx error code is an immediate failure
+ // this is sign of a server-side problem, and while
+ // rare in AWS S3, it does happen on third party stores.
+ // (out of disk space, etc).
+ // by the time we get here, the aws sdk will have
+ // already retried.
+ // there is specific handling for some 5XX codes (501, 503);
+ // this is for everything else
+ policyMap.put(AWSStatus500Exception.class, fail);
+
+ // subclass of AWSServiceIOException whose cause is always S3Exception
+ policyMap.put(AWSS3IOException.class, retryIdempotentCalls);
+
+ // server doesn't support a feature.
+ // raised from a number of HTTP error codes -mostly from
+ // third-party stores which only support a subset of AWS S3
+ // operations.
+ policyMap.put(AWSUnsupportedFeatureException.class, fail);
+
+ // socket exception subclass we consider unrecoverable
+ // though this is normally only found when opening a port for listening.
+ // which is never done in S3A.
+ policyMap.put(BindException.class, fail);
+
// connectivity problems are retried without worrying about idempotency
policyMap.put(ConnectTimeoutException.class, connectivityFailure);
+ policyMap.put(ConnectException.class, connectivityFailure);
// this can be a sign of an HTTP connection breaking early.
// which can be reacted to by another attempt if the request was idempotent.
@@ -213,25 +253,38 @@ protected Map, RetryPolicy> createExceptionMap() {
// which isn't going to be recovered from
policyMap.put(EOFException.class, retryIdempotentCalls);
- // policy on a 400/bad request still ambiguous.
- // Treated as an immediate failure
- policyMap.put(AWSBadRequestException.class, fail);
+ // object not found. 404 when not unknown bucket; 410 "gone"
+ policyMap.put(FileNotFoundException.class, fail);
- // Status 500 error code is also treated as a connectivity problem
- policyMap.put(AWSStatus500Exception.class, connectivityFailure);
+ // Interrupted, usually by other threads
+ policyMap.put(InterruptedException.class, fail);
+ // note this does not pick up subclasses (like socket timeout)
+ policyMap.put(InterruptedIOException.class, fail);
+ policyMap.put(InvalidRequestException.class, fail);
- // server didn't respond.
- policyMap.put(AWSNoResponseException.class, retryIdempotentCalls);
+ // auth failure. Possibly recoverable by reattempting with
+ // the credential provider, but not covered here.
+ policyMap.put(NoAuthWithAWSException.class, fail);
- // other operations
- policyMap.put(AWSClientIOException.class, retryIdempotentCalls);
- policyMap.put(AWSServiceIOException.class, retryIdempotentCalls);
- policyMap.put(AWSS3IOException.class, retryIdempotentCalls);
- policyMap.put(SocketTimeoutException.class, retryIdempotentCalls);
+ // network routing.
+ policyMap.put(NoRouteToHostException.class, fail);
+ // likely only recovered by changing the policy configuration or s3
+ // implementation
+ policyMap.put(NoVersionAttributeException.class, fail);
+ // once the file has changed, trying again is not going to help
+ policyMap.put(RemoteFileChangedException.class, fail);
+ // general socket exceptions
+ policyMap.put(SocketException.class, connectivityFailure);
+ policyMap.put(SocketTimeoutException.class, connectivityFailure);
+
+ // assume that DNS wil not recover; SDK is likely to have retried.
+ policyMap.put(UnknownHostException.class, fail);
+ policyMap.put(UnknownStoreException.class, fail);
// Unsupported requests do not work, however many times you try
policyMap.put(UnsupportedRequestException.class, fail);
+
return policyMap;
}
@@ -240,15 +293,19 @@ public RetryAction shouldRetry(Exception exception,
int retries,
int failovers,
boolean idempotent) throws Exception {
- Preconditions.checkArgument(exception != null, "Null exception");
- Exception ex = exception;
- if (exception instanceof AmazonClientException) {
- // uprate the amazon client exception for the purpose of exception
+ Exception ex = requireNonNull(exception, "Null exception");
+ if (exception instanceof SdkException) {
+ // update the sdk exception for the purpose of exception
// processing.
- ex = S3AUtils.translateException("", "",
- (AmazonClientException) exception);
+ ex = S3AUtils.translateException("", "", (SdkException) exception);
}
- return retryPolicy.shouldRetry(ex, retries, failovers, idempotent);
+ LOG.debug("Retry probe for {} with {} retries and {} failovers,"
+ + " idempotent={}, due to {}",
+ ex.getClass().getSimpleName(), retries, failovers, idempotent, ex, ex);
+ // look in the retry policy map
+ final RetryAction action = retryPolicy.shouldRetry(ex, retries, failovers, idempotent);
+ LOG.debug("Retry action is {}", action);
+ return action;
}
/**
@@ -315,4 +372,28 @@ public RetryAction shouldRetry(Exception e,
}
}
+ /**
+ * Policy where AWS SDK exceptions are retried if they state that they are retryable.
+ * See HADOOP-18871. S3ARetryPolicy to use sdk exception retryable() if it is valid.
+ */
+ private static final class RetryFromAWSClientExceptionPolicy implements RetryPolicy {
+
+ private final RetryPolicy next;
+
+ private RetryFromAWSClientExceptionPolicy(RetryPolicy next) {
+ this.next = next;
+ }
+
+ @Override
+ public RetryAction shouldRetry(Exception e,
+ int retries,
+ int failovers,
+ boolean isIdempotentOrAtMostOnce) throws Exception {
+ return
+ e instanceof AWSClientIOException ?
+ next.shouldRetry(e, retries, failovers, ((AWSClientIOException)e).retryable())
+ : RetryAction.FAIL;
+ }
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index dc0c211fcab50..f3e23f1cbf9fb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -18,20 +18,12 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AbortedException;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
-import com.amazonaws.retry.RetryUtils;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.MultiObjectDeleteException;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.core.exception.AbortedException;
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.core.retry.RetryUtils;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -44,16 +36,12 @@
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.util.functional.RemoteIterators;
-import org.apache.hadoop.fs.s3a.audit.AuditFailureException;
-import org.apache.hadoop.fs.s3a.audit.AuditIntegration;
import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
-import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider;
-import org.apache.hadoop.fs.s3a.impl.NetworkBinding;
-import org.apache.hadoop.fs.s3a.impl.V2Migration;
+import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
import org.apache.hadoop.net.ConnectTimeoutException;
import org.apache.hadoop.security.ProviderUtils;
-import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
import org.slf4j.Logger;
@@ -74,23 +62,25 @@
import java.net.URI;
import java.nio.file.AccessDeniedException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.Date;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
+import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
import static org.apache.commons.lang3.StringUtils.isEmpty;
+import static org.apache.hadoop.fs.s3a.AWSCredentialProviderList.maybeTranslateCredentialException;
import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.maybeTranslateAuditException;
import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH;
-import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.translateDeleteException;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.instantiationException;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isAbstract;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isNotInstanceOf;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unsupportedConstructor;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.*;
+import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractIOException;
import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator;
@@ -102,13 +92,7 @@
public final class S3AUtils {
private static final Logger LOG = LoggerFactory.getLogger(S3AUtils.class);
- static final String CONSTRUCTOR_EXCEPTION = "constructor exception";
- static final String INSTANTIATION_EXCEPTION
- = "instantiation exception";
- static final String NOT_AWS_PROVIDER =
- "does not implement AWSCredentialsProvider";
- static final String ABSTRACT_PROVIDER =
- "is abstract and therefore cannot be created";
+
static final String ENDPOINT_KEY = "Endpoint";
/** Filesystem is closed; kept here to keep the errors close. */
@@ -143,21 +127,13 @@ public final class S3AUtils {
private static final String BUCKET_PATTERN = FS_S3A_BUCKET_PREFIX + "%s.%s";
- /**
- * Error message when the AWS provider list built up contains a forbidden
- * entry.
- */
- @VisibleForTesting
- public static final String E_FORBIDDEN_AWS_PROVIDER
- = "AWS provider class cannot be used";
-
private S3AUtils() {
}
/**
* Translate an exception raised in an operation into an IOException.
* The specific type of IOException depends on the class of
- * {@link AmazonClientException} passed in, and any status codes included
+ * {@link SdkException} passed in, and any status codes included
* in the operation. That is: HTTP error codes are examined and can be
* used to build a more specific response.
*
@@ -170,14 +146,14 @@ private S3AUtils() {
*/
public static IOException translateException(String operation,
Path path,
- AmazonClientException exception) {
+ SdkException exception) {
return translateException(operation, path.toString(), exception);
}
/**
* Translate an exception raised in an operation into an IOException.
* The specific type of IOException depends on the class of
- * {@link AmazonClientException} passed in, and any status codes included
+ * {@link SdkException} passed in, and any status codes included
* in the operation. That is: HTTP error codes are examined and can be
* used to build a more specific response.
* @param operation operation
@@ -188,12 +164,14 @@ public static IOException translateException(String operation,
@SuppressWarnings("ThrowableInstanceNeverThrown")
public static IOException translateException(@Nullable String operation,
String path,
- SdkBaseException exception) {
+ SdkException exception) {
String message = String.format("%s%s: %s",
operation,
StringUtils.isNotEmpty(path)? (" on " + path) : "",
exception);
- if (!(exception instanceof AmazonServiceException)) {
+ if (!(exception instanceof AwsServiceException)) {
+ // exceptions raised client-side: connectivity, auth, network problems...
+
Exception innerCause = containsInterruptedException(exception);
if (innerCause != null) {
// interrupted IO, or a socket exception underneath that class
@@ -205,62 +183,72 @@ public static IOException translateException(@Nullable String operation,
}
// if the exception came from the auditor, hand off translation
// to it.
- if (exception instanceof AuditFailureException) {
- return AuditIntegration.translateAuditException(path, (AuditFailureException) exception);
+ IOException ioe = maybeTranslateAuditException(path, exception);
+ if (ioe != null) {
+ return ioe;
}
- if (exception instanceof CredentialInitializationException) {
- // the exception raised by AWSCredentialProvider list if the
- // credentials were not accepted,
- return (AccessDeniedException)new AccessDeniedException(path, null,
- exception.toString()).initCause(exception);
+ ioe = maybeTranslateCredentialException(path, exception);
+ if (ioe != null) {
+ return ioe;
}
+ // network problems covered by an IOE inside the exception chain.
+ ioe = maybeExtractIOException(path, exception);
+ if (ioe != null) {
+ return ioe;
+ }
+ // no custom handling.
return new AWSClientIOException(message, exception);
} else {
+ // "error response returned by an S3 or other service."
+ // These contain more details and should be translated based
+ // on the HTTP status code and other details.
IOException ioe;
- AmazonServiceException ase = (AmazonServiceException) exception;
+ AwsServiceException ase = (AwsServiceException) exception;
// this exception is non-null if the service exception is an s3 one
- AmazonS3Exception s3Exception = ase instanceof AmazonS3Exception
- ? (AmazonS3Exception) ase
+ S3Exception s3Exception = ase instanceof S3Exception
+ ? (S3Exception) ase
: null;
- int status = ase.getStatusCode();
- message = message + ":" + ase.getErrorCode();
+ int status = ase.statusCode();
+ if (ase.awsErrorDetails() != null) {
+ message = message + ":" + ase.awsErrorDetails().errorCode();
+ }
+
+ // big switch on the HTTP status code.
switch (status) {
- case 301:
- case 307:
+ case SC_301_MOVED_PERMANENTLY:
+ case SC_307_TEMPORARY_REDIRECT:
if (s3Exception != null) {
- if (s3Exception.getAdditionalDetails() != null &&
- s3Exception.getAdditionalDetails().containsKey(ENDPOINT_KEY)) {
- message = String.format("Received permanent redirect response to "
- + "endpoint %s. This likely indicates that the S3 endpoint "
- + "configured in %s does not match the AWS region containing "
- + "the bucket.",
- s3Exception.getAdditionalDetails().get(ENDPOINT_KEY), ENDPOINT);
- }
+ message = String.format("Received permanent redirect response to "
+ + "region %s. This likely indicates that the S3 region "
+ + "configured in %s does not match the AWS region containing " + "the bucket.",
+ s3Exception.awsErrorDetails().sdkHttpResponse().headers().get(BUCKET_REGION_HEADER),
+ AWS_REGION);
ioe = new AWSRedirectException(message, s3Exception);
} else {
ioe = new AWSRedirectException(message, ase);
}
break;
- case 400:
+ case SC_400_BAD_REQUEST:
ioe = new AWSBadRequestException(message, ase);
break;
// permissions
- case 401:
- case 403:
+ case SC_401_UNAUTHORIZED:
+ case SC_403_FORBIDDEN:
ioe = new AccessDeniedException(path, null, message);
ioe.initCause(ase);
break;
// the object isn't there
- case 404:
+ case SC_404_NOT_FOUND:
if (isUnknownBucket(ase)) {
// this is a missing bucket
ioe = new UnknownStoreException(path, message, ase);
} else {
- // a normal unknown object
+ // a normal unknown object.
+ // Can also be raised by third-party stores when aborting an unknown multipart upload
ioe = new FileNotFoundException(message);
ioe.initCause(ase);
}
@@ -268,20 +256,23 @@ public static IOException translateException(@Nullable String operation,
// this also surfaces sometimes and is considered to
// be ~ a not found exception.
- case 410:
+ case SC_410_GONE:
ioe = new FileNotFoundException(message);
ioe.initCause(ase);
break;
- // method not allowed; seen on S3 Select.
- // treated as a bad request
- case 405:
- ioe = new AWSBadRequestException(message, s3Exception);
+ // errors which stores can return from requests which
+ // the store does not support.
+ case SC_405_METHOD_NOT_ALLOWED:
+ case SC_412_PRECONDITION_FAILED:
+ case SC_415_UNSUPPORTED_MEDIA_TYPE:
+ case SC_501_NOT_IMPLEMENTED:
+ ioe = new AWSUnsupportedFeatureException(message, s3Exception);
break;
// out of range. This may happen if an object is overwritten with
// a shorter one while it is being read.
- case 416:
+ case SC_416_RANGE_NOT_SATISFIABLE:
ioe = new EOFException(message);
ioe.initCause(ase);
break;
@@ -289,32 +280,40 @@ public static IOException translateException(@Nullable String operation,
// this has surfaced as a "no response from server" message.
// so rare we haven't replicated it.
// Treating as an idempotent proxy error.
- case 443:
- case 444:
+ case SC_443_NO_RESPONSE:
+ case SC_444_NO_RESPONSE:
ioe = new AWSNoResponseException(message, ase);
break;
// throttling
- case 503:
+ case SC_429_TOO_MANY_REQUESTS_GCS: // google cloud through this connector
+ case SC_503_SERVICE_UNAVAILABLE: // AWS
ioe = new AWSServiceThrottledException(message, ase);
break;
// internal error
- case 500:
+ case SC_500_INTERNAL_SERVER_ERROR:
ioe = new AWSStatus500Exception(message, ase);
break;
- case 200:
+ case SC_200_OK:
if (exception instanceof MultiObjectDeleteException) {
// failure during a bulk delete
- return translateDeleteException(message,
- (MultiObjectDeleteException) exception);
+ return ((MultiObjectDeleteException) exception)
+ .translateException(message);
}
// other 200: FALL THROUGH
default:
- // no specific exit code. Choose an IOE subclass based on the class
- // of the caught exception
+ // no specifically handled exit code.
+
+ // convert all unknown 500+ errors to a 500 exception
+ if (status > SC_500_INTERNAL_SERVER_ERROR) {
+ ioe = new AWSStatus500Exception(message, ase);
+ break;
+ }
+
+ // Choose an IOE subclass based on the class of the caught exception
ioe = s3Exception != null
? new AWSS3IOException(message, s3Exception)
: new AWSServiceIOException(message, ase);
@@ -334,10 +333,35 @@ public static IOException translateException(@Nullable String operation,
public static IOException extractException(String operation,
String path,
ExecutionException ee) {
+ return convertExceptionCause(operation, path, ee.getCause());
+ }
+
+ /**
+ * Extract an exception from a failed future, and convert to an IOE.
+ * @param operation operation which failed
+ * @param path path operated on (may be null)
+ * @param ce completion exception
+ * @return an IOE which can be thrown
+ */
+ public static IOException extractException(String operation,
+ String path,
+ CompletionException ce) {
+ return convertExceptionCause(operation, path, ce.getCause());
+ }
+
+ /**
+ * Convert the cause of a concurrent exception to an IOE.
+ * @param operation operation which failed
+ * @param path path operated on (may be null)
+ * @param cause cause of a concurrent exception
+ * @return an IOE which can be thrown
+ */
+ private static IOException convertExceptionCause(String operation,
+ String path,
+ Throwable cause) {
IOException ioe;
- Throwable cause = ee.getCause();
- if (cause instanceof AmazonClientException) {
- ioe = translateException(operation, path, (AmazonClientException) cause);
+ if (cause instanceof SdkException) {
+ ioe = translateException(operation, path, (SdkException) cause);
} else if (cause instanceof IOException) {
ioe = (IOException) cause;
} else {
@@ -375,7 +399,7 @@ static Exception containsInterruptedException(Throwable thrown) {
* @return an IOE which can be rethrown
*/
private static InterruptedIOException translateInterruptedException(
- SdkBaseException exception,
+ SdkException exception,
final Exception innerCause,
String message) {
InterruptedIOException ioe;
@@ -386,6 +410,7 @@ private static InterruptedIOException translateInterruptedException(
if (name.endsWith(".ConnectTimeoutException")
|| name.endsWith(".ConnectionPoolTimeoutException")
|| name.endsWith("$ConnectTimeoutException")) {
+ // TODO: review in v2
// TCP connection http timeout from the shaded or unshaded filenames
// com.amazonaws.thirdparty.apache.http.conn.ConnectTimeoutException
ioe = new ConnectTimeoutException(message);
@@ -409,10 +434,10 @@ private static InterruptedIOException translateInterruptedException(
*/
public static boolean isThrottleException(Exception ex) {
return ex instanceof AWSServiceThrottledException
- || (ex instanceof AmazonServiceException
- && 503 == ((AmazonServiceException)ex).getStatusCode())
- || (ex instanceof SdkBaseException
- && RetryUtils.isThrottlingException((SdkBaseException) ex));
+ || (ex instanceof AwsServiceException
+ && 503 == ((AwsServiceException)ex).statusCode())
+ || (ex instanceof SdkException
+ && RetryUtils.isThrottlingException((SdkException) ex));
}
/**
@@ -422,7 +447,8 @@ public static boolean isThrottleException(Exception ex) {
* @param ex exception
* @return true if this is believed to be a sign the connection was broken.
*/
- public static boolean isMessageTranslatableToEOF(SdkBaseException ex) {
+ public static boolean isMessageTranslatableToEOF(SdkException ex) {
+ // TODO: review in v2
return ex.toString().contains(EOF_MESSAGE_IN_XML_PARSER) ||
ex.toString().contains(EOF_READ_DIFFERENT_LENGTH);
}
@@ -432,47 +458,26 @@ public static boolean isMessageTranslatableToEOF(SdkBaseException ex) {
* @param e exception
* @return string details
*/
- public static String stringify(AmazonServiceException e) {
+ public static String stringify(AwsServiceException e) {
StringBuilder builder = new StringBuilder(
- String.format("%s: %s error %d: %s; %s%s%n",
- e.getErrorType(),
- e.getServiceName(),
- e.getStatusCode(),
- e.getErrorCode(),
- e.getErrorMessage(),
- (e.isRetryable() ? " (retryable)": "")
+ String.format("%s error %d: %s; %s%s%n",
+ e.awsErrorDetails().serviceName(),
+ e.statusCode(),
+ e.awsErrorDetails().errorCode(),
+ e.awsErrorDetails().errorMessage(),
+ (e.retryable() ? " (retryable)": "")
));
- String rawResponseContent = e.getRawResponseContent();
+ String rawResponseContent = e.awsErrorDetails().rawResponse().asUtf8String();
if (rawResponseContent != null) {
builder.append(rawResponseContent);
}
return builder.toString();
}
- /**
- * Get low level details of an amazon exception for logging; multi-line.
- * @param e exception
- * @return string details
- */
- public static String stringify(AmazonS3Exception e) {
- // get the low level details of an exception,
- StringBuilder builder = new StringBuilder(
- stringify((AmazonServiceException) e));
- Map details = e.getAdditionalDetails();
- if (details != null) {
- builder.append('\n');
- for (Map.Entry d : details.entrySet()) {
- builder.append(d.getKey()).append('=')
- .append(d.getValue()).append('\n');
- }
- }
- return builder.toString();
- }
-
/**
* Create a files status instance from a listing.
* @param keyPath path to entry
- * @param summary summary from AWS
+ * @param s3Object s3Object entry
* @param blockSize block size to declare.
* @param owner owner of the file
* @param eTag S3 object eTag or null if unavailable
@@ -481,20 +486,20 @@ public static String stringify(AmazonS3Exception e) {
* @return a status entry
*/
public static S3AFileStatus createFileStatus(Path keyPath,
- S3ObjectSummary summary,
+ S3Object s3Object,
long blockSize,
String owner,
String eTag,
String versionId,
boolean isCSEEnabled) {
- long size = summary.getSize();
+ long size = s3Object.size();
// check if cse is enabled; strip out constant padding length.
if (isCSEEnabled && size >= CSE_PADDING_LENGTH) {
size -= CSE_PADDING_LENGTH;
}
return createFileStatus(keyPath,
- objectRepresentsDirectory(summary.getKey()),
- size, summary.getLastModified(), blockSize, owner, eTag, versionId);
+ objectRepresentsDirectory(s3Object.key()),
+ size, Date.from(s3Object.lastModified()), blockSize, owner, eTag, versionId);
}
/**
@@ -556,114 +561,7 @@ public static long dateToLong(final Date date) {
}
/**
- * The standard AWS provider list for AWS connections.
- */
- @SuppressWarnings("deprecation")
- public static final List>
- STANDARD_AWS_PROVIDERS = Collections.unmodifiableList(
- Arrays.asList(
- TemporaryAWSCredentialsProvider.class,
- SimpleAWSCredentialsProvider.class,
- EnvironmentVariableCredentialsProvider.class,
- IAMInstanceCredentialsProvider.class));
-
- /**
- * Create the AWS credentials from the providers, the URI and
- * the key {@link Constants#AWS_CREDENTIALS_PROVIDER} in the configuration.
- * @param binding Binding URI -may be null
- * @param conf filesystem configuration
- * @return a credentials provider list
- * @throws IOException Problems loading the providers (including reading
- * secrets from credential files).
- */
- public static AWSCredentialProviderList createAWSCredentialProviderSet(
- @Nullable URI binding,
- Configuration conf) throws IOException {
- // this will reject any user:secret entries in the URI
- S3xLoginHelper.rejectSecretsInURIs(binding);
- AWSCredentialProviderList credentials =
- buildAWSProviderList(binding,
- conf,
- AWS_CREDENTIALS_PROVIDER,
- STANDARD_AWS_PROVIDERS,
- new HashSet<>());
- // make sure the logging message strips out any auth details
- LOG.debug("For URI {}, using credentials {}",
- binding, credentials);
- return credentials;
- }
-
- /**
- * Load list of AWS credential provider/credential provider factory classes.
- * @param conf configuration
- * @param key key
- * @param defaultValue list of default values
- * @return the list of classes, possibly empty
- * @throws IOException on a failure to load the list.
- */
- public static List> loadAWSProviderClasses(Configuration conf,
- String key,
- Class... defaultValue) throws IOException {
- try {
- return Arrays.asList(conf.getClasses(key, defaultValue));
- } catch (RuntimeException e) {
- Throwable c = e.getCause() != null ? e.getCause() : e;
- throw new IOException("From option " + key + ' ' + c, c);
- }
- }
-
- /**
- * Load list of AWS credential provider/credential provider factory classes;
- * support a forbidden list to prevent loops, mandate full secrets, etc.
- * @param binding Binding URI -may be null
- * @param conf configuration
- * @param key key
- * @param forbidden a possibly empty set of forbidden classes.
- * @param defaultValues list of default providers.
- * @return the list of classes, possibly empty
- * @throws IOException on a failure to load the list.
- */
- public static AWSCredentialProviderList buildAWSProviderList(
- @Nullable final URI binding,
- final Configuration conf,
- final String key,
- final List> defaultValues,
- final Set> forbidden) throws IOException {
-
- // build up the base provider
- List> awsClasses = loadAWSProviderClasses(conf,
- key,
- defaultValues.toArray(new Class[defaultValues.size()]));
- // and if the list is empty, switch back to the defaults.
- // this is to address the issue that configuration.getClasses()
- // doesn't return the default if the config value is just whitespace.
- if (awsClasses.isEmpty()) {
- awsClasses = defaultValues;
- }
- // iterate through, checking for blacklists and then instantiating
- // each provider
- AWSCredentialProviderList providers = new AWSCredentialProviderList();
- for (Class aClass : awsClasses) {
-
- // List of V1 credential providers that will be migrated with V2 upgrade
- if (!Arrays.asList("EnvironmentVariableCredentialsProvider",
- "EC2ContainerCredentialsProviderWrapper", "InstanceProfileCredentialsProvider")
- .contains(aClass.getSimpleName()) && aClass.getName().contains(AWS_AUTH_CLASS_PREFIX)) {
- V2Migration.v1ProviderReferenced(aClass.getName());
- }
-
- if (forbidden.contains(aClass)) {
- throw new IOException(E_FORBIDDEN_AWS_PROVIDER
- + " in option " + key + ": " + aClass);
- }
- providers.add(createAWSCredentialProvider(conf,
- aClass, binding));
- }
- return providers;
- }
-
- /**
- * Create an AWS credential provider from its class by using reflection. The
+ * Creates an instance of a class using reflection. The
* class must implement one of the following means of construction, which are
* attempted in order:
*
@@ -672,92 +570,87 @@ public static AWSCredentialProviderList buildAWSProviderList(
* org.apache.hadoop.conf.Configuration
* a public constructor accepting
* org.apache.hadoop.conf.Configuration
- * a public static method named getInstance that accepts no
+ * a public static method named as per methodName, that accepts no
* arguments and returns an instance of
- * com.amazonaws.auth.AWSCredentialsProvider, or
+ * specified type, or
* a public default constructor.
*
*
+ * @param className name of class for which instance is to be created
* @param conf configuration
- * @param credClass credential class
* @param uri URI of the FS
- * @return the instantiated class
- * @throws IOException on any instantiation failure.
+ * @param interfaceImplemented interface that this class implements
+ * @param methodName name of factory method to be invoked
+ * @param configKey config key under which this class is specified
+ * @param Instance of class
+ * @return instance of the specified class
+ * @throws IOException on any problem
*/
- private static AWSCredentialsProvider createAWSCredentialProvider(
+ @SuppressWarnings("unchecked")
+ public static InstanceT getInstanceFromReflection(String className,
Configuration conf,
- Class credClass,
- @Nullable URI uri) throws IOException {
- AWSCredentialsProvider credentials = null;
- String className = credClass.getName();
- if (!AWSCredentialsProvider.class.isAssignableFrom(credClass)) {
- throw new IOException("Class " + credClass + " " + NOT_AWS_PROVIDER);
- }
- if (Modifier.isAbstract(credClass.getModifiers())) {
- throw new IOException("Class " + credClass + " " + ABSTRACT_PROVIDER);
- }
- LOG.debug("Credential provider class is {}", className);
-
+ @Nullable URI uri,
+ Class interfaceImplemented,
+ String methodName,
+ String configKey) throws IOException {
try {
- // new X(uri, conf)
- Constructor cons = getConstructor(credClass, URI.class,
- Configuration.class);
- if (cons != null) {
- credentials = (AWSCredentialsProvider)cons.newInstance(uri, conf);
- return credentials;
+ Class instanceClass = S3AUtils.class.getClassLoader().loadClass(className);
+ if (Modifier.isAbstract(instanceClass.getModifiers())) {
+ throw isAbstract(uri, className, configKey);
}
- // new X(conf)
- cons = getConstructor(credClass, Configuration.class);
- if (cons != null) {
- credentials = (AWSCredentialsProvider)cons.newInstance(conf);
- return credentials;
+ if (!interfaceImplemented.isAssignableFrom(instanceClass)) {
+ throw isNotInstanceOf(uri, className, interfaceImplemented.getName(), configKey);
+
+ }
+ Constructor cons;
+ if (conf != null) {
+ // new X(uri, conf)
+ cons = getConstructor(instanceClass, URI.class, Configuration.class);
+
+ if (cons != null) {
+ return (InstanceT) cons.newInstance(uri, conf);
+ }
+ // new X(conf)
+ cons = getConstructor(instanceClass, Configuration.class);
+ if (cons != null) {
+ return (InstanceT) cons.newInstance(conf);
+ }
}
- // X.getInstance()
- Method factory = getFactoryMethod(credClass, AWSCredentialsProvider.class,
- "getInstance");
+ // X.methodName()
+ Method factory = getFactoryMethod(instanceClass, interfaceImplemented, methodName);
if (factory != null) {
- credentials = (AWSCredentialsProvider)factory.invoke(null);
- return credentials;
+ return (InstanceT) factory.invoke(null);
}
// new X()
- cons = getConstructor(credClass);
+ cons = getConstructor(instanceClass);
if (cons != null) {
- credentials = (AWSCredentialsProvider)cons.newInstance();
- return credentials;
+ return (InstanceT) cons.newInstance();
}
// no supported constructor or factory method found
- throw new IOException(String.format("%s " + CONSTRUCTOR_EXCEPTION
- + ". A class specified in %s must provide a public constructor "
- + "of a supported signature, or a public factory method named "
- + "getInstance that accepts no arguments.",
- className, AWS_CREDENTIALS_PROVIDER));
+ throw unsupportedConstructor(uri, className, configKey);
} catch (InvocationTargetException e) {
Throwable targetException = e.getTargetException();
if (targetException == null) {
- targetException = e;
+ targetException = e;
}
if (targetException instanceof IOException) {
throw (IOException) targetException;
- } else if (targetException instanceof SdkBaseException) {
- throw translateException("Instantiate " + className, "",
- (SdkBaseException) targetException);
+ } else if (targetException instanceof SdkException) {
+ throw translateException("Instantiate " + className, "", (SdkException) targetException);
} else {
// supported constructor or factory method found, but the call failed
- throw new IOException(className + " " + INSTANTIATION_EXCEPTION
- + ": " + targetException,
- targetException);
+ throw instantiationException(uri, className, configKey, targetException);
}
} catch (ReflectiveOperationException | IllegalArgumentException e) {
// supported constructor or factory method found, but the call failed
- throw new IOException(className + " " + INSTANTIATION_EXCEPTION
- + ": " + e,
- e);
+ throw instantiationException(uri, className, configKey, e);
}
}
+
/**
* Set a key if the value is non-empty.
* @param config config to patch
@@ -941,13 +834,13 @@ static String lookupPassword(Configuration conf, String key, String defVal)
/**
* String information about a summary entry for debug messages.
- * @param summary summary object
+ * @param s3Object s3Object entry
* @return string value
*/
- public static String stringify(S3ObjectSummary summary) {
- StringBuilder builder = new StringBuilder(summary.getKey().length() + 100);
- builder.append(summary.getKey()).append(' ');
- builder.append("size=").append(summary.getSize());
+ public static String stringify(S3Object s3Object) {
+ StringBuilder builder = new StringBuilder(s3Object.key().length() + 100);
+ builder.append(s3Object.key()).append(' ');
+ builder.append("size=").append(s3Object.size());
return builder.toString();
}
@@ -1215,215 +1108,6 @@ public static void deleteWithWarning(FileSystem fs,
}
}
- /**
- * Create a new AWS {@code ClientConfiguration}.
- * All clients to AWS services MUST use this for consistent setup
- * of connectivity, UA, proxy settings.
- * @param conf The Hadoop configuration
- * @param bucket Optional bucket to use to look up per-bucket proxy secrets
- * @return new AWS client configuration
- * @throws IOException problem creating AWS client configuration
- *
- * @deprecated use {@link #createAwsConf(Configuration, String, String)}
- */
- @Deprecated
- public static ClientConfiguration createAwsConf(Configuration conf,
- String bucket)
- throws IOException {
- return createAwsConf(conf, bucket, null);
- }
-
- /**
- * Create a new AWS {@code ClientConfiguration}. All clients to AWS services
- * MUST use this or the equivalents for the specific service for
- * consistent setup of connectivity, UA, proxy settings.
- *
- * @param conf The Hadoop configuration
- * @param bucket Optional bucket to use to look up per-bucket proxy secrets
- * @param awsServiceIdentifier a string representing the AWS service (S3,
- * etc) for which the ClientConfiguration is being created.
- * @return new AWS client configuration
- * @throws IOException problem creating AWS client configuration
- */
- public static ClientConfiguration createAwsConf(Configuration conf,
- String bucket, String awsServiceIdentifier)
- throws IOException {
- final ClientConfiguration awsConf = new ClientConfiguration();
- initConnectionSettings(conf, awsConf);
- initProxySupport(conf, bucket, awsConf);
- initUserAgent(conf, awsConf);
- if (StringUtils.isNotEmpty(awsServiceIdentifier)) {
- String configKey = null;
- switch (awsServiceIdentifier) {
- case AWS_SERVICE_IDENTIFIER_S3:
- configKey = SIGNING_ALGORITHM_S3;
- break;
- case AWS_SERVICE_IDENTIFIER_STS:
- configKey = SIGNING_ALGORITHM_STS;
- break;
- default:
- // Nothing to do. The original signer override is already setup
- }
- if (configKey != null) {
- String signerOverride = conf.getTrimmed(configKey, "");
- if (!signerOverride.isEmpty()) {
- LOG.debug("Signer override for {}} = {}", awsServiceIdentifier,
- signerOverride);
- awsConf.setSignerOverride(signerOverride);
- }
- }
- }
- return awsConf;
- }
-
- /**
- * Initializes all AWS SDK settings related to connection management.
- *
- * @param conf Hadoop configuration
- * @param awsConf AWS SDK configuration
- *
- * @throws IOException if there was an error initializing the protocol
- * settings
- */
- public static void initConnectionSettings(Configuration conf,
- ClientConfiguration awsConf) throws IOException {
- awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS,
- DEFAULT_MAXIMUM_CONNECTIONS, 1));
- awsConf.setConnectionTTL(longOption(conf, CONNECTION_TTL,
- DEFAULT_CONNECTION_TTL, -1));
- initProtocolSettings(conf, awsConf);
- awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES,
- DEFAULT_MAX_ERROR_RETRIES, 0));
- awsConf.setConnectionTimeout(intOption(conf, ESTABLISH_TIMEOUT,
- DEFAULT_ESTABLISH_TIMEOUT, 0));
- awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT,
- DEFAULT_SOCKET_TIMEOUT, 0));
- int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER,
- DEFAULT_SOCKET_SEND_BUFFER, 2048);
- int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER,
- DEFAULT_SOCKET_RECV_BUFFER, 2048);
- long requestTimeoutMillis = conf.getTimeDuration(REQUEST_TIMEOUT,
- DEFAULT_REQUEST_TIMEOUT, TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
-
- if (requestTimeoutMillis > Integer.MAX_VALUE) {
- LOG.debug("Request timeout is too high({} ms). Setting to {} ms instead",
- requestTimeoutMillis, Integer.MAX_VALUE);
- requestTimeoutMillis = Integer.MAX_VALUE;
- }
- awsConf.setRequestTimeout((int) requestTimeoutMillis);
- awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
- String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
- if (!signerOverride.isEmpty()) {
- LOG.debug("Signer override = {}", signerOverride);
- awsConf.setSignerOverride(signerOverride);
- }
- }
-
- /**
- * Initializes the connection protocol settings when connecting to S3 (e.g.
- * either HTTP or HTTPS). If secure connections are enabled, this method
- * will load the configured SSL providers.
- *
- * @param conf Hadoop configuration
- * @param awsConf AWS SDK configuration
- *
- * @throws IOException if there is an error initializing the configured
- * {@link javax.net.ssl.SSLSocketFactory}
- */
- private static void initProtocolSettings(Configuration conf,
- ClientConfiguration awsConf) throws IOException {
- boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS,
- DEFAULT_SECURE_CONNECTIONS);
- awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
- if (secureConnections) {
- NetworkBinding.bindSSLChannelMode(conf, awsConf);
- }
- }
-
- /**
- * Initializes AWS SDK proxy support in the AWS client configuration
- * if the S3A settings enable it.
- *
- * @param conf Hadoop configuration
- * @param bucket Optional bucket to use to look up per-bucket proxy secrets
- * @param awsConf AWS SDK configuration to update
- * @throws IllegalArgumentException if misconfigured
- * @throws IOException problem getting username/secret from password source.
- */
- public static void initProxySupport(Configuration conf,
- String bucket,
- ClientConfiguration awsConf) throws IllegalArgumentException,
- IOException {
- String proxyHost = conf.getTrimmed(PROXY_HOST, "");
- int proxyPort = conf.getInt(PROXY_PORT, -1);
- if (!proxyHost.isEmpty()) {
- awsConf.setProxyHost(proxyHost);
- if (proxyPort >= 0) {
- awsConf.setProxyPort(proxyPort);
- } else {
- if (conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS)) {
- LOG.warn("Proxy host set without port. Using HTTPS default 443");
- awsConf.setProxyPort(443);
- } else {
- LOG.warn("Proxy host set without port. Using HTTP default 80");
- awsConf.setProxyPort(80);
- }
- }
- final String proxyUsername = lookupPassword(bucket, conf, PROXY_USERNAME,
- null, null);
- final String proxyPassword = lookupPassword(bucket, conf, PROXY_PASSWORD,
- null, null);
- if ((proxyUsername == null) != (proxyPassword == null)) {
- String msg = "Proxy error: " + PROXY_USERNAME + " or " +
- PROXY_PASSWORD + " set without the other.";
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
- boolean isProxySecured = conf.getBoolean(PROXY_SECURED, false);
- awsConf.setProxyUsername(proxyUsername);
- awsConf.setProxyPassword(proxyPassword);
- awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
- awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
- awsConf.setProxyProtocol(isProxySecured ? Protocol.HTTPS : Protocol.HTTP);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Using proxy server {}://{}:{} as user {} with password {} "
- + "on domain {} as workstation {}",
- awsConf.getProxyProtocol(),
- awsConf.getProxyHost(),
- awsConf.getProxyPort(),
- String.valueOf(awsConf.getProxyUsername()),
- awsConf.getProxyPassword(), awsConf.getProxyDomain(),
- awsConf.getProxyWorkstation());
- }
- } else if (proxyPort >= 0) {
- String msg =
- "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
- }
-
- /**
- * Initializes the User-Agent header to send in HTTP requests to AWS
- * services. We always include the Hadoop version number. The user also
- * may set an optional custom prefix to put in front of the Hadoop version
- * number. The AWS SDK internally appends its own information, which seems
- * to include the AWS SDK version, OS and JVM version.
- *
- * @param conf Hadoop configuration
- * @param awsConf AWS SDK configuration to update
- */
- private static void initUserAgent(Configuration conf,
- ClientConfiguration awsConf) {
- String userAgent = "Hadoop " + VersionInfo.getVersion();
- String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, "");
- if (!userAgentPrefix.isEmpty()) {
- userAgent = userAgentPrefix + ", " + userAgent;
- }
- LOG.debug("Using User-Agent: {}", userAgent);
- awsConf.setUserAgentPrefix(userAgent);
- }
-
/**
* Convert the data of an iterator of {@link S3AFileStatus} to
* an array.
@@ -1746,6 +1430,11 @@ public static EncryptionSecrets buildEncryptionSecrets(String bucket,
diagnostics);
break;
+ case DSSE_KMS:
+ LOG.debug("Using DSSE-KMS with {}",
+ diagnostics);
+ break;
+
case NONE:
default:
LOG.debug("Data is unencrypted");
@@ -1917,4 +1606,15 @@ public String toString() {
}
};
+ /**
+ * Format a byte range for a request header.
+ * See https://www.rfc-editor.org/rfc/rfc9110.html#section-14.1.2
+ *
+ * @param rangeStart the start byte offset
+ * @param rangeEnd the end byte offset (inclusive)
+ * @return a formatted byte range
+ */
+ public static String formatRange(long rangeStart, long rangeEnd) {
+ return String.format("bytes=%d-%d", rangeStart, rangeEnd);
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
index 9010f34dc259c..451eab8e9ad58 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
@@ -24,11 +24,13 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Executor;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.monitoring.MonitoringListener;
-import com.amazonaws.services.s3.AmazonS3;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.transfer.s3.S3TransferManager;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -37,7 +39,7 @@
import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ENDPOINT;
/**
- * Factory for creation of {@link AmazonS3} client instances.
+ * Factory for creation of {@link S3Client} client instances.
* Important: HBase's HBoss module implements this interface in its
* tests.
* Take care when updating this interface to ensure that a client
@@ -49,20 +51,44 @@
*/
@InterfaceAudience.LimitedPrivate("HBoss")
@InterfaceStability.Evolving
-@Deprecated
public interface S3ClientFactory {
/**
- * Creates a new {@link AmazonS3} client.
+ * Creates a new {@link S3Client}.
+ * The client returned supports synchronous operations. For
+ * asynchronous operations, use
+ * {@link #createS3AsyncClient(URI, S3ClientCreationParameters)}.
*
* @param uri S3A file system URI
* @param parameters parameter object
* @return S3 client
- * @throws IOException IO problem
+ * @throws IOException on any IO problem
*/
- AmazonS3 createS3Client(URI uri,
+ S3Client createS3Client(URI uri,
S3ClientCreationParameters parameters) throws IOException;
+ /**
+ * Creates a new {@link S3AsyncClient}.
+ * The client returned supports asynchronous operations. For
+ * synchronous operations, use
+ * {@link #createS3Client(URI, S3ClientCreationParameters)}.
+ *
+ * @param uri S3A file system URI
+ * @param parameters parameter object
+ * @return Async S3 client
+ * @throws IOException on any IO problem
+ */
+ S3AsyncClient createS3AsyncClient(URI uri,
+ S3ClientCreationParameters parameters) throws IOException;
+
+ /**
+ * Creates a new {@link S3TransferManager}.
+ *
+ * @param s3AsyncClient the async client to be used by the TM.
+ * @return S3 transfer manager
+ */
+ S3TransferManager createS3TransferManager(S3AsyncClient s3AsyncClient);
+
/**
* Settings for the S3 Client.
* Implemented as a class to pass in so that adding
@@ -74,7 +100,7 @@ final class S3ClientCreationParameters {
/**
* Credentials.
*/
- private AWSCredentialsProvider credentialSet;
+ private AwsCredentialsProvider credentialSet;
/**
* Endpoint.
@@ -86,11 +112,6 @@ final class S3ClientCreationParameters {
*/
private final Map headers = new HashMap<>();
- /**
- * Monitoring listener.
- */
- private MonitoringListener monitoringListener;
-
/**
* RequestMetricCollector metrics...if not-null will be wrapped
* with an {@code AwsStatisticsCollector} and passed to
@@ -109,9 +130,9 @@ final class S3ClientCreationParameters {
private boolean requesterPays;
/**
- * Request handlers; used for auditing, X-Ray etc.
- */
- private List requestHandlers;
+ * Execution interceptors; used for auditing, X-Ray etc.
+ * */
+ private List executionInterceptors;
/**
* Suffix to UA.
@@ -125,37 +146,48 @@ final class S3ClientCreationParameters {
private URI pathUri;
/**
- * List of request handlers to include in the chain
- * of request execution in the SDK.
- * @return the handler list
+ * Minimum part size for transfer parts.
*/
- public List getRequestHandlers() {
- return requestHandlers;
- }
+ private long minimumPartSize;
/**
- * List of request handlers.
- * @param handlers handler list.
- * @return this object
+ * Threshold for multipart operations.
*/
- public S3ClientCreationParameters withRequestHandlers(
- @Nullable final List handlers) {
- requestHandlers = handlers;
- return this;
- }
+ private long multiPartThreshold;
+
+ /**
+ * Multipart upload enabled.
+ */
+ private boolean multipartCopy = true;
- public MonitoringListener getMonitoringListener() {
- return monitoringListener;
+ /**
+ * Executor that the transfer manager will use to execute background tasks.
+ */
+ private Executor transferManagerExecutor;
+
+ /**
+ * Region of the S3 bucket.
+ */
+ private String region;
+
+
+ /**
+ * List of execution interceptors to include in the chain
+ * of interceptors in the SDK.
+ * @return the interceptors list
+ */
+ public List getExecutionInterceptors() {
+ return executionInterceptors;
}
/**
- * listener for AWS monitoring events.
- * @param listener listener
+ * List of execution interceptors.
+ * @param interceptors interceptors list.
* @return this object
*/
- public S3ClientCreationParameters withMonitoringListener(
- @Nullable final MonitoringListener listener) {
- monitoringListener = listener;
+ public S3ClientCreationParameters withExecutionInterceptors(
+ @Nullable final List interceptors) {
+ executionInterceptors = interceptors;
return this;
}
@@ -191,7 +223,7 @@ public boolean isRequesterPays() {
return requesterPays;
}
- public AWSCredentialsProvider getCredentialSet() {
+ public AwsCredentialsProvider getCredentialSet() {
return credentialSet;
}
@@ -202,7 +234,7 @@ public AWSCredentialsProvider getCredentialSet() {
*/
public S3ClientCreationParameters withCredentialSet(
- final AWSCredentialsProvider value) {
+ final AwsCredentialsProvider value) {
credentialSet = value;
return this;
}
@@ -294,5 +326,101 @@ public S3ClientCreationParameters withPathUri(
pathUri = value;
return this;
}
+
+ /**
+ * Get the minimum part size for transfer parts.
+ * @return part size
+ */
+ public long getMinimumPartSize() {
+ return minimumPartSize;
+ }
+
+ /**
+ * Set the minimum part size for transfer parts.
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withMinimumPartSize(
+ final long value) {
+ minimumPartSize = value;
+ return this;
+ }
+
+ /**
+ * Get the threshold for multipart operations.
+ * @return multipart threshold
+ */
+ public long getMultiPartThreshold() {
+ return multiPartThreshold;
+ }
+
+ /**
+ * Set the threshold for multipart operations.
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withMultipartThreshold(
+ final long value) {
+ multiPartThreshold = value;
+ return this;
+ }
+
+ /**
+ * Get the executor that the transfer manager will use to execute background tasks.
+ * @return part size
+ */
+ public Executor getTransferManagerExecutor() {
+ return transferManagerExecutor;
+ }
+
+ /**
+ * Set the executor that the transfer manager will use to execute background tasks.
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withTransferManagerExecutor(
+ final Executor value) {
+ transferManagerExecutor = value;
+ return this;
+ }
+
+ /**
+ * Set the multipart flag..
+ *
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withMultipartCopyEnabled(final boolean value) {
+ this.multipartCopy = value;
+ return this;
+ }
+
+ /**
+ * Get the multipart flag.
+ * @return multipart flag
+ */
+ public boolean isMultipartCopy() {
+ return multipartCopy;
+ }
+
+ /**
+ * Set region.
+ *
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withRegion(
+ final String value) {
+ region = value;
+ return this;
+ }
+
+ /**
+ * Get the region.
+ * @return invoker
+ */
+ public String getRegion() {
+ return region;
+ }
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java
index d51211516f251..c729f3de15f08 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
/**
* API version-independent container for S3 List requests.
@@ -78,14 +78,14 @@ public ListObjectsV2Request getV2() {
public String toString() {
if (isV1()) {
return String.format(DESCRIPTION,
- v1Request.getBucketName(), v1Request.getPrefix(),
- v1Request.getDelimiter(), v1Request.getMaxKeys(),
- v1Request.isRequesterPays());
+ v1Request.bucket(), v1Request.prefix(),
+ v1Request.delimiter(), v1Request.maxKeys(),
+ v1Request.requestPayerAsString());
} else {
return String.format(DESCRIPTION,
- v2Request.getBucketName(), v2Request.getPrefix(),
- v2Request.getDelimiter(), v2Request.getMaxKeys(),
- v2Request.isRequesterPays());
+ v2Request.bucket(), v2Request.prefix(),
+ v2Request.delimiter(), v2Request.maxKeys(),
+ v2Request.requestPayerAsString());
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
index 69c42bfe1471a..0300b78bec250 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
@@ -22,19 +22,23 @@
import java.util.List;
import java.util.stream.Collectors;
-import com.amazonaws.services.s3.model.ListObjectsV2Result;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
+import software.amazon.awssdk.services.s3.model.CommonPrefix;
+import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
import org.slf4j.Logger;
+import static java.util.Objects.requireNonNull;
+
/**
* API version-independent container for S3 List responses.
*/
public class S3ListResult {
- private ObjectListing v1Result;
- private ListObjectsV2Result v2Result;
+ private ListObjectsResponse v1Result;
+ private ListObjectsV2Response v2Result;
- protected S3ListResult(ObjectListing v1, ListObjectsV2Result v2) {
+ protected S3ListResult(ListObjectsResponse v1, ListObjectsV2Response v2) {
v1Result = v1;
v2Result = v2;
}
@@ -44,8 +48,8 @@ protected S3ListResult(ObjectListing v1, ListObjectsV2Result v2) {
* @param result v1 result
* @return new list result container
*/
- public static S3ListResult v1(ObjectListing result) {
- return new S3ListResult(result, null);
+ public static S3ListResult v1(ListObjectsResponse result) {
+ return new S3ListResult(requireNonNull(result), null);
}
/**
@@ -53,8 +57,8 @@ public static S3ListResult v1(ObjectListing result) {
* @param result v2 result
* @return new list result container
*/
- public static S3ListResult v2(ListObjectsV2Result result) {
- return new S3ListResult(null, result);
+ public static S3ListResult v2(ListObjectsV2Response result) {
+ return new S3ListResult(null, requireNonNull(result));
}
/**
@@ -65,19 +69,19 @@ public boolean isV1() {
return v1Result != null;
}
- public ObjectListing getV1() {
+ public ListObjectsResponse getV1() {
return v1Result;
}
- public ListObjectsV2Result getV2() {
+ public ListObjectsV2Response getV2() {
return v2Result;
}
- public List getObjectSummaries() {
+ public List getS3Objects() {
if (isV1()) {
- return v1Result.getObjectSummaries();
+ return v1Result.contents();
} else {
- return v2Result.getObjectSummaries();
+ return v2Result.contents();
}
}
@@ -89,21 +93,21 @@ public boolean isTruncated() {
}
}
- public List getCommonPrefixes() {
+ public List getCommonPrefixes() {
if (isV1()) {
- return v1Result.getCommonPrefixes();
+ return v1Result.commonPrefixes();
} else {
- return v2Result.getCommonPrefixes();
+ return v2Result.commonPrefixes();
}
}
/**
- * Get the list of keys in the object summary.
+ * Get the list of keys in the list result.
* @return a possibly empty list
*/
- private List objectSummaryKeys() {
- return getObjectSummaries().stream()
- .map(S3ObjectSummary::getKey)
+ private List objectKeys() {
+ return getS3Objects().stream()
+ .map(S3Object::key)
.collect(Collectors.toList());
}
@@ -112,9 +116,8 @@ private List objectSummaryKeys() {
* @return true if the result is non-empty
*/
public boolean hasPrefixesOrObjects() {
-
return !(getCommonPrefixes()).isEmpty()
- || !getObjectSummaries().isEmpty();
+ || !getS3Objects().isEmpty();
}
/**
@@ -128,7 +131,7 @@ public boolean representsEmptyDirectory(
// no children.
// So the listing must contain the marker entry only as an object,
// and prefixes is null
- List keys = objectSummaryKeys();
+ List keys = objectKeys();
return keys.size() == 1 && keys.contains(dirKey)
&& getCommonPrefixes().isEmpty();
}
@@ -138,15 +141,15 @@ public boolean representsEmptyDirectory(
* @param log log to use
*/
public void logAtDebug(Logger log) {
- Collection prefixes = getCommonPrefixes();
- Collection summaries = getObjectSummaries();
+ Collection prefixes = getCommonPrefixes();
+ Collection s3Objects = getS3Objects();
log.debug("Prefix count = {}; object count={}",
- prefixes.size(), summaries.size());
- for (S3ObjectSummary summary : summaries) {
- log.debug("Summary: {} {}", summary.getKey(), summary.getSize());
+ prefixes.size(), s3Objects.size());
+ for (S3Object s3Object : s3Objects) {
+ log.debug("Summary: {} {}", s3Object.key(), s3Object.size());
}
- for (String prefix : prefixes) {
- log.debug("Prefix: {}", prefix);
+ for (CommonPrefix prefix : prefixes) {
+ log.debug("Prefix: {}", prefix.prefix());
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
index 0a0454854b2ac..4fc5b8658b605 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.transfer.model.CopyResult;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
@@ -74,31 +72,6 @@ public S3ObjectAttributes(
this.len = len;
}
- /**
- * Construct from the result of a copy and those parameters
- * which aren't included in an AWS SDK response.
- * @param path path
- * @param copyResult copy result.
- * @param serverSideEncryptionAlgorithm current encryption algorithm
- * @param serverSideEncryptionKey any server side encryption key?
- * @param len object length
- */
- public S3ObjectAttributes(
- final Path path,
- final CopyResult copyResult,
- final S3AEncryptionMethods serverSideEncryptionAlgorithm,
- final String serverSideEncryptionKey,
- final long len) {
- this.bucket = copyResult.getDestinationBucketName();
- this.key = copyResult.getDestinationKey();
- this.path = path;
- this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm;
- this.serverSideEncryptionKey = serverSideEncryptionKey;
- this.eTag = copyResult.getETag();
- this.versionId = copyResult.getVersionId();
- this.len = len;
- }
-
public String getBucket() {
return bucket;
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
index 6579a2bc3e7d2..b7de937f5137a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
@@ -39,6 +39,5 @@
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-@SuppressWarnings("deprecation")
public final class SharedInstanceCredentialProvider extends IAMInstanceCredentialsProvider {
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
index 657bd437b5be9..087d034975adb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
@@ -18,9 +18,11 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
@@ -42,13 +44,10 @@
* property fs.s3a.aws.credentials.provider. Therefore, changing the class name
* would be a backward-incompatible change.
*
- * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
- * as part of upgrading S3A to SDK V2. See HADOOP-18073.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-@Deprecated
-public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider {
+public class SimpleAWSCredentialsProvider implements AwsCredentialsProvider {
public static final String NAME
= "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider";
@@ -63,7 +62,7 @@ public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider {
*/
public SimpleAWSCredentialsProvider(final URI uri, final Configuration conf)
throws IOException {
- this(getAWSAccessKeys(uri, conf));
+ this(getAWSAccessKeys(uri, conf));
}
/**
@@ -75,25 +74,25 @@ public SimpleAWSCredentialsProvider(final URI uri, final Configuration conf)
@VisibleForTesting
SimpleAWSCredentialsProvider(final S3xLoginHelper.Login login)
throws IOException {
- this.accessKey = login.getUser();
- this.secretKey = login.getPassword();
+ this.accessKey = login.getUser();
+ this.secretKey = login.getPassword();
}
@Override
- public AWSCredentials getCredentials() {
+ public AwsCredentials resolveCredentials() {
if (!StringUtils.isEmpty(accessKey) && !StringUtils.isEmpty(secretKey)) {
- return new BasicAWSCredentials(accessKey, secretKey);
+ return AwsBasicCredentials.create(accessKey, secretKey);
}
throw new NoAwsCredentialsException("SimpleAWSCredentialsProvider",
"No AWS credentials in the Hadoop configuration");
}
- @Override
- public void refresh() {}
-
@Override
public String toString() {
- return getClass().getSimpleName();
+ return "SimpleAWSCredentialsProvider{" +
+ "accessKey.empty=" + accessKey.isEmpty() +
+ ", secretKey.empty=" + secretKey.isEmpty() +
+ '}';
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
index f9694871d7304..72fc75b642415 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
@@ -242,7 +242,10 @@ public enum Statistic {
StoreStatisticNames.OBJECT_MULTIPART_UPLOAD_ABORTED,
"Object multipart upload aborted",
TYPE_DURATION),
- OBJECT_PUT_REQUESTS(
+ OBJECT_MULTIPART_UPLOAD_LIST(
+ StoreStatisticNames.OBJECT_MULTIPART_UPLOAD_LIST,
+ "Object multipart list request issued",
+ TYPE_DURATION), OBJECT_PUT_REQUESTS(
StoreStatisticNames.OBJECT_PUT_REQUEST,
"Object put/multipart upload count",
TYPE_DURATION),
@@ -577,7 +580,7 @@ public enum Statistic {
TYPE_COUNTER),
MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED(
StoreStatisticNames.MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED,
- "Multipart Upload Abort Unner Path Invoked",
+ "Multipart Upload Abort Under Path Invoked",
TYPE_COUNTER),
MULTIPART_UPLOAD_COMPLETED(
StoreStatisticNames.MULTIPART_UPLOAD_COMPLETED,
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
index db3d0bb13297c..5587b11a36f88 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
@@ -21,7 +21,7 @@
import javax.annotation.Nullable;
import java.io.IOException;
-import com.amazonaws.auth.AWSCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
import java.net.URI;
@@ -44,12 +44,9 @@
* This credential provider must not fail in creation because that will
* break a chain of credential providers.
*
- * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
- * as part of upgrading S3A to SDK V2. See HADOOP-18073.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-@Deprecated
public class TemporaryAWSCredentialsProvider extends AbstractSessionCredentialsProvider {
public static final String NAME
@@ -92,7 +89,7 @@ public TemporaryAWSCredentialsProvider(
* @throws NoAwsCredentialsException the credentials are actually empty.
*/
@Override
- protected AWSCredentials createCredentials(Configuration config)
+ protected AwsCredentials createCredentials(Configuration config)
throws IOException {
MarshalledCredentials creds = MarshalledCredentialBinding.fromFileSystem(
getUri(), config);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java
index 0462ccfd7cbbd..d7123787735a1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java
@@ -18,15 +18,58 @@
package org.apache.hadoop.fs.s3a;
+import java.util.Optional;
+
+import static java.util.Optional.empty;
+import static java.util.Optional.of;
+
/**
* Simple enum to express {true, false, don't know}.
*/
public enum Tristate {
+
// Do not add additional values here. Logic will assume there are exactly
// three possibilities.
- TRUE, FALSE, UNKNOWN;
+ TRUE(of(Boolean.TRUE)),
+ FALSE(of(Boolean.FALSE)),
+ UNKNOWN(empty());
+
+ /**
+ * Mapping to an optional boolean.
+ */
+ @SuppressWarnings("NonSerializableFieldInSerializableClass")
+ private final Optional mapping;
+
+ Tristate(final Optional t) {
+ mapping = t;
+ }
+
+ /**
+ * Get the boolean mapping, if present.
+ * @return the boolean value, if present.
+ */
+ public Optional getMapping() {
+ return mapping;
+ }
+
+ /**
+ * Does this value map to a boolean.
+ * @return true if the state is one of true or false.
+ */
+ public boolean isBoolean() {
+ return mapping.isPresent();
+ }
public static Tristate fromBool(boolean v) {
return v ? TRUE : FALSE;
}
+
+ /**
+ * Build a tristate from a boolean.
+ * @param b source optional
+ * @return a tristate derived from the argument.
+ */
+ public static Tristate from(Optional b) {
+ return b.map(Tristate::fromBool).orElse(UNKNOWN);
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java
index 238cd97ed8f48..85ed69e14bc79 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java
@@ -18,22 +18,22 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.services.s3.transfer.Upload;
+import software.amazon.awssdk.transfer.s3.model.FileUpload;
/**
* Simple struct that contains information about a S3 upload.
*/
public class UploadInfo {
- private final Upload upload;
+ private final FileUpload fileUpload;
private final long length;
- public UploadInfo(Upload upload, long length) {
- this.upload = upload;
+ public UploadInfo(FileUpload upload, long length) {
+ this.fileUpload = upload;
this.length = length;
}
- public Upload getUpload() {
- return upload;
+ public FileUpload getFileUpload() {
+ return fileUpload;
}
public long getLength() {
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
index 0bbb8a35f51cb..1eacc4cbe07b7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
@@ -19,29 +19,28 @@
package org.apache.hadoop.fs.s3a;
import javax.annotation.Nullable;
-import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
import java.util.List;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicInteger;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.SelectObjectContentResult;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -50,6 +49,8 @@
import org.apache.hadoop.fs.s3a.api.RequestFactory;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
import org.apache.hadoop.fs.s3a.impl.StoreContext;
+import org.apache.hadoop.fs.s3a.select.SelectEventStreamPublisher;
+import org.apache.hadoop.fs.s3a.select.SelectObjectContentHelper;
import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext;
import org.apache.hadoop.fs.s3a.select.SelectBinding;
import org.apache.hadoop.fs.statistics.DurationTrackerFactory;
@@ -239,49 +240,25 @@ private void deactivateAuditSpan() {
/**
* Create a {@link PutObjectRequest} request against the specific key.
* @param destKey destination key
- * @param inputStream source data.
* @param length size, if known. Use -1 for not known
* @param options options for the request
+ * @param isFile is data to be uploaded a file
* @return the request
*/
@Retries.OnceRaw
public PutObjectRequest createPutObjectRequest(String destKey,
- InputStream inputStream,
long length,
- final PutObjectOptions options) {
- activateAuditSpan();
- ObjectMetadata objectMetadata = newObjectMetadata(length);
- return getRequestFactory().newPutObjectRequest(
- destKey,
- objectMetadata,
- options,
- inputStream);
- }
+ final PutObjectOptions options,
+ boolean isFile) {
- /**
- * Create a {@link PutObjectRequest} request to upload a file.
- * @param dest key to PUT to.
- * @param sourceFile source file
- * @param options options for the request
- * @return the request
- */
- @Retries.OnceRaw
- public PutObjectRequest createPutObjectRequest(
- String dest,
- File sourceFile,
- final PutObjectOptions options) {
activateAuditSpan();
- final ObjectMetadata objectMetadata =
- newObjectMetadata((int) sourceFile.length());
-
- PutObjectRequest putObjectRequest = getRequestFactory().
- newPutObjectRequest(dest,
- objectMetadata,
- options,
- sourceFile);
- return putObjectRequest;
+
+ return getRequestFactory()
+ .newPutObjectRequestBuilder(destKey, options, length, false)
+ .build();
}
+
/**
* Callback on a successful write.
* @param length length of the write
@@ -297,17 +274,6 @@ public void writeFailed(Exception ex) {
LOG.debug("Write to {} failed", this, ex);
}
- /**
- * Create a new object metadata instance.
- * Any standard metadata headers are added here, for example:
- * encryption.
- * @param length size, if known. Use -1 for not known
- * @return a new metadata instance
- */
- public ObjectMetadata newObjectMetadata(long length) {
- return getRequestFactory().newObjectMetadata(length);
- }
-
/**
* {@inheritDoc}
*/
@@ -320,11 +286,11 @@ public String initiateMultiPartUpload(
try (AuditSpan span = activateAuditSpan()) {
return retry("initiate MultiPartUpload", destKey, true,
() -> {
- final InitiateMultipartUploadRequest initiateMPURequest =
- getRequestFactory().newMultipartUploadRequest(
+ final CreateMultipartUploadRequest.Builder initiateMPURequestBuilder =
+ getRequestFactory().newMultipartUploadRequestBuilder(
destKey, options);
- return owner.initiateMultipartUpload(initiateMPURequest)
- .getUploadId();
+ return owner.initiateMultipartUpload(initiateMPURequestBuilder.build())
+ .uploadId();
});
}
}
@@ -345,10 +311,10 @@ public String initiateMultiPartUpload(
* @throws IOException on problems.
*/
@Retries.RetryTranslated
- private CompleteMultipartUploadResult finalizeMultipartUpload(
+ private CompleteMultipartUploadResponse finalizeMultipartUpload(
String destKey,
String uploadId,
- List partETags,
+ List partETags,
long length,
PutObjectOptions putOptions,
Retried retrying) throws IOException {
@@ -357,18 +323,18 @@ private CompleteMultipartUploadResult finalizeMultipartUpload(
"No upload parts in multipart upload");
}
try (AuditSpan span = activateAuditSpan()) {
- CompleteMultipartUploadResult uploadResult;
+ CompleteMultipartUploadResponse uploadResult;
uploadResult = invoker.retry("Completing multipart upload", destKey,
true,
retrying,
() -> {
- final CompleteMultipartUploadRequest request =
- getRequestFactory().newCompleteMultipartUploadRequest(
+ final CompleteMultipartUploadRequest.Builder requestBuilder =
+ getRequestFactory().newCompleteMultipartUploadRequestBuilder(
destKey, uploadId, partETags);
- return writeOperationHelperCallbacks.completeMultipartUpload(request);
+ return writeOperationHelperCallbacks.completeMultipartUpload(requestBuilder.build());
});
- owner.finishedWrite(destKey, length, uploadResult.getETag(),
- uploadResult.getVersionId(),
+ owner.finishedWrite(destKey, length, uploadResult.eTag(),
+ uploadResult.versionId(),
putOptions);
return uploadResult;
}
@@ -391,10 +357,10 @@ private CompleteMultipartUploadResult finalizeMultipartUpload(
* the retry count was exceeded
*/
@Retries.RetryTranslated
- public CompleteMultipartUploadResult completeMPUwithRetries(
+ public CompleteMultipartUploadResponse completeMPUwithRetries(
String destKey,
String uploadId,
- List partETags,
+ List partETags,
long length,
AtomicInteger errorCount,
PutObjectOptions putOptions)
@@ -452,7 +418,7 @@ public void abortMultipartUpload(String destKey, String uploadId,
@Retries.RetryTranslated
public void abortMultipartUpload(MultipartUpload upload)
throws IOException {
- invoker.retry("Aborting multipart commit", upload.getKey(), true,
+ invoker.retry("Aborting multipart commit", upload.key(), true,
withinAuditSpan(getAuditSpan(),
() -> owner.abortMultipartUpload(upload)));
}
@@ -477,7 +443,7 @@ public int abortMultipartUploadsUnderPath(String prefix)
abortMultipartUpload(upload);
count++;
} catch (FileNotFoundException e) {
- LOG.debug("Already aborted: {}", upload.getKey(), e);
+ LOG.debug("Already aborted: {}", upload.key(), e);
}
}
return count;
@@ -506,45 +472,31 @@ public void abortMultipartCommit(String destKey, String uploadId)
}
/**
- * Create and initialize a part request of a multipart upload.
- * Exactly one of: {@code uploadStream} or {@code sourceFile}
- * must be specified.
- * A subset of the file may be posted, by providing the starting point
- * in {@code offset} and a length of block in {@code size} equal to
- * or less than the remaining bytes.
+ * Create and initialize a part request builder of a multipart upload.
* The part number must be less than 10000.
* Retry policy is once-translated; to much effort
* @param destKey destination key of ongoing operation
* @param uploadId ID of ongoing upload
* @param partNumber current part number of the upload
* @param size amount of data
- * @param uploadStream source of data to upload
- * @param sourceFile optional source file.
- * @param offset offset in file to start reading.
- * @return the request.
+ * @return the request builder.
* @throws IllegalArgumentException if the parameters are invalid.
* @throws PathIOException if the part number is out of range.
*/
@Override
@Retries.OnceTranslated
- public UploadPartRequest newUploadPartRequest(
+ public UploadPartRequest.Builder newUploadPartRequestBuilder(
String destKey,
String uploadId,
int partNumber,
- long size,
- InputStream uploadStream,
- File sourceFile,
- Long offset) throws IOException {
+ long size) throws IOException {
return once("upload part request", destKey,
withinAuditSpan(getAuditSpan(), () ->
- getRequestFactory().newUploadPartRequest(
+ getRequestFactory().newUploadPartRequestBuilder(
destKey,
uploadId,
partNumber,
- size,
- uploadStream,
- sourceFile,
- offset)));
+ size)));
}
/**
@@ -566,18 +518,20 @@ public String toString() {
* @param putObjectRequest the request
* @param putOptions put object options
* @param durationTrackerFactory factory for duration tracking
+ * @param uploadData data to be uploaded
+ * @param isFile is data to be uploaded a file
+ *
* @return the upload initiated
* @throws IOException on problems
*/
@Retries.RetryTranslated
- public PutObjectResult putObject(PutObjectRequest putObjectRequest,
- PutObjectOptions putOptions,
+ public PutObjectResponse putObject(PutObjectRequest putObjectRequest,
+ PutObjectOptions putOptions, S3ADataBlocks.BlockUploadData uploadData, boolean isFile,
DurationTrackerFactory durationTrackerFactory)
throws IOException {
- return retry("Writing Object",
- putObjectRequest.getKey(), true,
- withinAuditSpan(getAuditSpan(), () ->
- owner.putObjectDirect(putObjectRequest, putOptions, durationTrackerFactory)));
+ return retry("Writing Object", putObjectRequest.key(), true, withinAuditSpan(getAuditSpan(),
+ () -> owner.putObjectDirect(putObjectRequest, putOptions, uploadData, isFile,
+ durationTrackerFactory)));
}
/**
@@ -613,10 +567,10 @@ public void revertCommit(String destKey) throws IOException {
* the retry count was exceeded
*/
@Retries.RetryTranslated
- public CompleteMultipartUploadResult commitUpload(
+ public CompleteMultipartUploadResponse commitUpload(
String destKey,
String uploadId,
- List partETags,
+ List partETags,
long length)
throws IOException {
checkNotNull(uploadId);
@@ -635,19 +589,21 @@ public CompleteMultipartUploadResult commitUpload(
* Upload part of a multi-partition file.
* @param request request
* @param durationTrackerFactory duration tracker factory for operation
+ * @param request the upload part request.
+ * @param body the request body.
* @return the result of the operation.
* @throws IOException on problems
*/
@Retries.RetryTranslated
- public UploadPartResult uploadPart(UploadPartRequest request,
+ public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody body,
final DurationTrackerFactory durationTrackerFactory)
throws IOException {
- return retry("upload part #" + request.getPartNumber()
- + " upload ID " + request.getUploadId(),
- request.getKey(),
+ return retry("upload part #" + request.partNumber()
+ + " upload ID " + request.uploadId(),
+ request.key(),
true,
withinAuditSpan(getAuditSpan(),
- () -> owner.uploadPart(request, durationTrackerFactory)));
+ () -> owner.uploadPart(request, body, durationTrackerFactory)));
}
/**
@@ -659,15 +615,9 @@ public Configuration getConf() {
return conf;
}
- /**
- * Create a S3 Select request for the destination path.
- * This does not build the query.
- * @param path pre-qualified path for query
- * @return the request
- */
- public SelectObjectContentRequest newSelectRequest(Path path) {
+ public SelectObjectContentRequest.Builder newSelectRequestBuilder(Path path) {
try (AuditSpan span = getAuditSpan()) {
- return getRequestFactory().newSelectRequest(
+ return getRequestFactory().newSelectRequestBuilder(
storeContext.pathToKey(path));
}
}
@@ -676,26 +626,27 @@ public SelectObjectContentRequest newSelectRequest(Path path) {
* Execute an S3 Select operation.
* On a failure, the request is only logged at debug to avoid the
* select exception being printed.
- * @param source source for selection
+ *
+ * @param source source for selection
* @param request Select request to issue.
- * @param action the action for use in exception creation
+ * @param action the action for use in exception creation
* @return response
* @throws IOException failure
*/
@Retries.RetryTranslated
- public SelectObjectContentResult select(
+ public SelectEventStreamPublisher select(
final Path source,
final SelectObjectContentRequest request,
final String action)
throws IOException {
// no setting of span here as the select binding is (statically) created
// without any span.
- String bucketName = request.getBucketName();
+ String bucketName = request.bucket();
Preconditions.checkArgument(bucket.equals(bucketName),
"wrong bucket: %s", bucketName);
if (LOG.isDebugEnabled()) {
LOG.debug("Initiating select call {} {}",
- source, request.getExpression());
+ source, request.expression());
LOG.debug(SelectBinding.toString(request));
}
return invoker.retry(
@@ -706,8 +657,9 @@ public SelectObjectContentResult select(
try (DurationInfo ignored =
new DurationInfo(LOG, "S3 Select operation")) {
try {
- return writeOperationHelperCallbacks.selectObjectContent(request);
- } catch (AmazonS3Exception e) {
+ return SelectObjectContentHelper.select(
+ writeOperationHelperCallbacks, source, request, action);
+ } catch (Throwable e) {
LOG.error("Failure of S3 Select request against {}",
source);
LOG.debug("S3 Select request against {}:\n{}",
@@ -756,16 +708,18 @@ public interface WriteOperationHelperCallbacks {
/**
* Initiates a select request.
* @param request selectObjectContent request
+ * @param t selectObjectContent request handler
* @return selectObjectContentResult
*/
- SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request);
+ CompletableFuture selectObjectContent(SelectObjectContentRequest request,
+ SelectObjectContentResponseHandler t);
/**
* Initiates a complete multi-part upload request.
* @param request Complete multi-part upload request
* @return completeMultipartUploadResult
*/
- CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request);
+ CompleteMultipartUploadResponse completeMultipartUpload(CompleteMultipartUploadRequest request);
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
index 1c3d368857583..0fda4921a30da 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
@@ -20,29 +20,27 @@
import javax.annotation.Nullable;
import java.io.Closeable;
-import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.SelectObjectContentResult;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
import org.apache.hadoop.fs.statistics.DurationTrackerFactory;
+import org.apache.hadoop.fs.s3a.select.SelectEventStreamPublisher;
import org.apache.hadoop.fs.store.audit.AuditSpanSource;
import org.apache.hadoop.util.functional.CallableRaisingIOE;
@@ -77,27 +75,15 @@ T retry(String action,
/**
* Create a {@link PutObjectRequest} request against the specific key.
* @param destKey destination key
- * @param inputStream source data.
* @param length size, if known. Use -1 for not known
* @param options options for the request
+ * @param isFile is data to be uploaded a file
* @return the request
*/
PutObjectRequest createPutObjectRequest(String destKey,
- InputStream inputStream,
long length,
- @Nullable PutObjectOptions options);
-
- /**
- * Create a {@link PutObjectRequest} request to upload a file.
- * @param dest key to PUT to.
- * @param sourceFile source file
- * @param options options for the request
- * @return the request
- */
- PutObjectRequest createPutObjectRequest(
- String dest,
- File sourceFile,
- @Nullable PutObjectOptions options);
+ @Nullable PutObjectOptions options,
+ boolean isFile);
/**
* Callback on a successful write.
@@ -111,15 +97,6 @@ PutObjectRequest createPutObjectRequest(
*/
void writeFailed(Exception ex);
- /**
- * Create a new object metadata instance.
- * Any standard metadata headers are added here, for example:
- * encryption.
- * @param length size, if known. Use -1 for not known
- * @return a new metadata instance
- */
- ObjectMetadata newObjectMetadata(long length);
-
/**
* Start the multipart upload process.
* Retry policy: retrying, translated.
@@ -148,10 +125,10 @@ PutObjectRequest createPutObjectRequest(
* the retry count was exceeded
*/
@Retries.RetryTranslated
- CompleteMultipartUploadResult completeMPUwithRetries(
+ CompleteMultipartUploadResponse completeMPUwithRetries(
String destKey,
String uploadId,
- List partETags,
+ List partETags,
long length,
AtomicInteger errorCount,
PutObjectOptions putOptions)
@@ -213,31 +190,20 @@ void abortMultipartCommit(String destKey, String uploadId)
throws IOException;
/**
- * Create and initialize a part request of a multipart upload.
- * Exactly one of: {@code uploadStream} or {@code sourceFile}
- * must be specified.
- * A subset of the file may be posted, by providing the starting point
- * in {@code offset} and a length of block in {@code size} equal to
- * or less than the remaining bytes.
+ * Create and initialize a part request builder of a multipart upload.
* @param destKey destination key of ongoing operation
* @param uploadId ID of ongoing upload
* @param partNumber current part number of the upload
* @param size amount of data
- * @param uploadStream source of data to upload
- * @param sourceFile optional source file.
- * @param offset offset in file to start reading.
- * @return the request.
+ * @return the request builder.
* @throws IllegalArgumentException if the parameters are invalid
* @throws PathIOException if the part number is out of range.
*/
- UploadPartRequest newUploadPartRequest(
+ UploadPartRequest.Builder newUploadPartRequestBuilder(
String destKey,
String uploadId,
int partNumber,
- long size,
- InputStream uploadStream,
- File sourceFile,
- Long offset) throws IOException;
+ long size) throws IOException;
/**
* PUT an object directly (i.e. not via the transfer manager).
@@ -246,12 +212,14 @@ UploadPartRequest newUploadPartRequest(
* @param putObjectRequest the request
* @param putOptions put object options
* @param durationTrackerFactory factory for duration tracking
+ * @param uploadData data to be uploaded
+ * @param isFile is data to be uploaded a file
* @return the upload initiated
* @throws IOException on problems
*/
@Retries.RetryTranslated
- PutObjectResult putObject(PutObjectRequest putObjectRequest,
- PutObjectOptions putOptions,
+ PutObjectResponse putObject(PutObjectRequest putObjectRequest,
+ PutObjectOptions putOptions, S3ADataBlocks.BlockUploadData uploadData, boolean isFile,
DurationTrackerFactory durationTrackerFactory)
throws IOException;
@@ -279,22 +247,23 @@ PutObjectResult putObject(PutObjectRequest putObjectRequest,
* the retry count was exceeded
*/
@Retries.RetryTranslated
- CompleteMultipartUploadResult commitUpload(
+ CompleteMultipartUploadResponse commitUpload(
String destKey,
String uploadId,
- List partETags,
+ List partETags,
long length)
throws IOException;
/**
* Upload part of a multi-partition file.
- * @param request request
+ * @param request the upload part request.
+ * @param body the request body.
* @param durationTrackerFactory factory for duration tracking
* @return the result of the operation.
* @throws IOException on problems
*/
@Retries.RetryTranslated
- UploadPartResult uploadPart(UploadPartRequest request,
+ UploadPartResponse uploadPart(UploadPartRequest request, RequestBody body,
DurationTrackerFactory durationTrackerFactory)
throws IOException;
@@ -306,25 +275,26 @@ UploadPartResult uploadPart(UploadPartRequest request,
Configuration getConf();
/**
- * Create a S3 Select request for the destination path.
+ * Create a S3 Select request builder for the destination path.
* This does not build the query.
* @param path pre-qualified path for query
- * @return the request
+ * @return the request builder
*/
- SelectObjectContentRequest newSelectRequest(Path path);
+ SelectObjectContentRequest.Builder newSelectRequestBuilder(Path path);
/**
* Execute an S3 Select operation.
* On a failure, the request is only logged at debug to avoid the
* select exception being printed.
- * @param source source for selection
+ *
+ * @param source source for selection
* @param request Select request to issue.
- * @param action the action for use in exception creation
+ * @param action the action for use in exception creation
* @return response
* @throws IOException failure
*/
@Retries.RetryTranslated
- SelectObjectContentResult select(
+ SelectEventStreamPublisher select(
Path source,
SelectObjectContentRequest request,
String action)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java
new file mode 100644
index 0000000000000..177952cb9d22b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.adapter;
+
+import java.io.IOException;
+import java.net.URI;
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unavailable;
+
+/**
+ * Binding support; the sole way which the rest of the code should instantiate v1 SDK libraries.
+ * Uses this class's Classloader for its analysis/loading.
+ */
+@SuppressWarnings("StaticNonFinalField")
+public final class AwsV1BindingSupport {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ AwsV1BindingSupport.class);
+
+ /**
+ * V1 credential provider classname: {@code}.
+ */
+ public static final String CREDENTIAL_PROVIDER_CLASSNAME =
+ "com.amazonaws.auth.AWSCredentialsProvider";
+
+ /**
+ * SDK availability.
+ */
+ private static final boolean SDK_V1_FOUND = checkForAwsV1Sdk();
+
+ private AwsV1BindingSupport() {
+ }
+
+ /**
+ * Probe for the AWS v1 SDK being available by looking for
+ * the class {@link #CREDENTIAL_PROVIDER_CLASSNAME}.
+ * @return true if it was found in the classloader
+ */
+ private static boolean checkForAwsV1Sdk() {
+
+ try {
+ ClassLoader cl = AwsV1BindingSupport.class.getClassLoader();
+ cl.loadClass(CREDENTIAL_PROVIDER_CLASSNAME);
+ LOG.debug("v1 SDK class {} found", CREDENTIAL_PROVIDER_CLASSNAME);
+ return true;
+ } catch (Exception e) {
+ LOG.debug("v1 SDK class {} not found", CREDENTIAL_PROVIDER_CLASSNAME, e);
+ return false;
+ }
+ }
+
+ /**
+ * Is the AWS v1 SDK available?
+ * @return true if it was found in the classloader
+ */
+ public static synchronized boolean isAwsV1SdkAvailable() {
+ return SDK_V1_FOUND;
+ }
+
+
+ /**
+ * Create an AWS credential provider from its class by using reflection. The
+ * class must implement one of the following means of construction, which are
+ * attempted in order:
+ *
+ *
+ * - a public constructor accepting java.net.URI and
+ * org.apache.hadoop.conf.Configuration
+ * - a public constructor accepting
+ * org.apache.hadoop.conf.Configuration
+ * - a public static method named getInstance that accepts no
+ * arguments and returns an instance of
+ * com.amazonaws.auth.AWSCredentialsProvider, or
+ * - a public default constructor.
+ *
+ * @param conf configuration
+ * @param className credential classname
+ * @param uri URI of the FS
+ * @param key configuration key to use
+ * @return the instantiated class
+ * @throws InstantiationIOException on any instantiation failure, including v1 SDK not found
+ * @throws IOException anything else.
+ */
+ public static AwsCredentialsProvider createAWSV1CredentialProvider(
+ Configuration conf,
+ String className,
+ @Nullable URI uri,
+ final String key) throws IOException {
+ if (!isAwsV1SdkAvailable()) {
+ throw unavailable(uri, className, key, "No AWS v1 SDK available");
+ }
+ return V1ToV2AwsCredentialProviderAdapter.create(conf, className, uri);
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java
new file mode 100644
index 0000000000000..2cbd9836d3a43
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.adapter;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.URI;
+import javax.annotation.Nullable;
+
+import com.amazonaws.SdkClientException;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSSessionCredentials;
+import com.amazonaws.auth.AnonymousAWSCredentials;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.CredentialInitializationException;
+import org.apache.hadoop.fs.s3a.S3AUtils;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
+
+/**
+ * Adapts a V1 {@link AWSCredentialsProvider} to the V2 {@link AwsCredentialsProvider} interface.
+ */
+public final class V1ToV2AwsCredentialProviderAdapter
+ implements AwsCredentialsProvider, Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ V1ToV2AwsCredentialProviderAdapter.class);
+
+ /**
+ * The V1 credential provider constructed.
+ */
+ private final AWSCredentialsProvider v1CredentialsProvider;
+
+
+ private V1ToV2AwsCredentialProviderAdapter(AWSCredentialsProvider v1CredentialsProvider) {
+ this.v1CredentialsProvider = requireNonNull(v1CredentialsProvider);
+ }
+
+
+ /**
+ * Collect v1 credentials and convert to v2.
+ * @return v2 credentials
+ * @throws CredentialInitializationException if the inner retrieval raised an exception
+ */
+ @Override
+ public AwsCredentials resolveCredentials() {
+ try {
+ // get the wrapped credentials
+ AWSCredentials toAdapt = v1CredentialsProvider.getCredentials();
+ return convertToV2Credentials(toAdapt);
+ } catch (SdkClientException e) {
+ // wrap with a v2 exception so that code which adds a try/catch for v2 sdk exceptions
+ // gets a compatible exception.
+ throw new CredentialInitializationException(e.toString(), e);
+ }
+ }
+
+ /**
+ * Close the wrapped provider if it implements Closeable/AutoCloseable.
+ * @throws IOException failure
+ */
+ @Override
+ public void close() throws IOException {
+ if (v1CredentialsProvider instanceof Closeable) {
+ ((Closeable) v1CredentialsProvider).close();
+ } else if (v1CredentialsProvider instanceof AutoCloseable) {
+ S3AUtils.closeAutocloseables(LOG, (AutoCloseable) v1CredentialsProvider);
+ }
+ }
+
+ /**
+ * Convert v1 credentials to v2, including support for session and anonymous
+ * credentials.
+ * @param toAdapt credentials to adapt.
+ * @return v2 credentials.
+ */
+ static AwsCredentials convertToV2Credentials(final AWSCredentials toAdapt) {
+ if (toAdapt instanceof AWSSessionCredentials) {
+ return AwsSessionCredentials.create(toAdapt.getAWSAccessKeyId(),
+ toAdapt.getAWSSecretKey(),
+ ((AWSSessionCredentials) toAdapt).getSessionToken());
+ } else if (toAdapt instanceof AnonymousAWSCredentials) {
+ return AnonymousCredentialsProvider.create().resolveCredentials();
+ } else {
+ return AwsBasicCredentials.create(toAdapt.getAWSAccessKeyId(), toAdapt.getAWSSecretKey());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "V1ToV2AwsCredentialProviderAdapter{" +
+ "v1CredentialsProvider=" + v1CredentialsProvider +
+ '}';
+ }
+
+ /**
+ * @param v1CredentialsProvider V1 credential provider to adapt.
+ * @return A new instance of the credentials provider adapter.
+ */
+ static AwsCredentialsProvider create(AWSCredentialsProvider v1CredentialsProvider) {
+ return new V1ToV2AwsCredentialProviderAdapter(v1CredentialsProvider);
+ }
+
+ /**
+ * Create an AWS credential provider from its class by using reflection. The
+ * class must implement one of the following means of construction, which are
+ * attempted in order:
+ *
+ *
+ * - a public constructor accepting java.net.URI and
+ * org.apache.hadoop.conf.Configuration
+ * - a public constructor accepting
+ * org.apache.hadoop.conf.Configuration
+ * - a public static method named getInstance that accepts no
+ * arguments and returns an instance of
+ * com.amazonaws.auth.AWSCredentialsProvider, or
+ * - a public default constructor.
+ *
+ * @param conf configuration
+ * @param className classname
+ * @param uri URI of the FS
+ * @return the instantiated class
+ * @throws InstantiationIOException on construction and instantiation failures,
+ * including v1 SDK exceptions.
+ * @throws IOException if raised by a constructor/factory method.
+ */
+ static AwsCredentialsProvider create(
+ Configuration conf,
+ String className,
+ @Nullable URI uri) throws InstantiationIOException, IOException {
+
+ final AWSCredentialsProvider instance =
+ S3AUtils.getInstanceFromReflection(className, conf, uri, AWSCredentialsProvider.class,
+ "getInstance", AWS_CREDENTIALS_PROVIDER);
+ return create(instance);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/package-info.java
new file mode 100644
index 0000000000000..124534188bccc
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/package-info.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Adapter classes for allowing V1 credential providers to be used with SDKV2.
+ * This is the only package where use of aws v1 classes are permitted;
+ * all instantiations of objects here must use reflection to probe for
+ * availability or be prepared to catch exceptions which may be raised
+ * if the v1 SDK isn't found on the classpath
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.s3a.adapter;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
index 2a4771925f086..99a898f728166 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
@@ -19,41 +19,37 @@
package org.apache.hadoop.fs.s3a.api;
import javax.annotation.Nullable;
-import java.io.File;
-import java.io.InputStream;
import java.util.List;
-import java.util.Optional;
-
-import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CannedAccessControlList;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
-import com.amazonaws.services.s3.model.SSECustomerKey;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.StorageClass;
-import com.amazonaws.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.StorageClass;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
/**
- * Factory for S3 objects.
+ * Factory for S3 request objects.
*
* This is where the owner FS's {@code prepareRequest()}
* callback is invoked to mark up a request for this span.
@@ -67,6 +63,8 @@
* as there are no guarantees how they are processed.
* That is: no guarantees of retry or translation.
*/
+@InterfaceStability.Unstable
+@InterfaceAudience.LimitedPrivate("testing/diagnostics")
public interface RequestFactory {
/**
@@ -79,22 +77,7 @@ public interface RequestFactory {
* Get the canned ACL of this FS.
* @return an ACL, if any
*/
- CannedAccessControlList getCannedACL();
-
- /**
- * Create the AWS SDK structure used to configure SSE,
- * if the encryption secrets contain the information/settings for this.
- * @return an optional set of KMS Key settings
- */
- Optional generateSSEAwsKeyParams();
-
- /**
- * Create the SSE-C structure for the AWS SDK, if the encryption secrets
- * contain the information/settings for this.
- * This will contain a secret extracted from the bucket/configuration.
- * @return an optional customer key.
- */
- Optional generateSSECustomerKey();
+ String getCannedACL();
/**
* Get the encryption algorithm of this endpoint.
@@ -115,79 +98,58 @@ public interface RequestFactory {
StorageClass getStorageClass();
/**
- * Create a new object metadata instance.
- * Any standard metadata headers are added here, for example:
- * encryption.
- *
- * @param length length of data to set in header; Ignored if negative
- * @return a new metadata instance
- */
- ObjectMetadata newObjectMetadata(long length);
-
- /**
- * Create a copy request.
+ * Create a copy request builder.
* This includes the work of copying the relevant parts
* of the metadata from the source
* @param srcKey source
* @param dstKey destination
* @param srcom source object metadata.
- * @return the request
+ * @return the request builder
*/
- CopyObjectRequest newCopyObjectRequest(String srcKey,
+ CopyObjectRequest.Builder newCopyObjectRequestBuilder(String srcKey,
String dstKey,
- ObjectMetadata srcom);
+ HeadObjectResponse srcom);
- /**
- * Create a putObject request.
- * Adds the ACL and metadata
- * @param key key of object
- * @param metadata metadata header
- * @param options options for the request
- * @param srcfile source file
- * @return the request
- */
- PutObjectRequest newPutObjectRequest(String key,
- ObjectMetadata metadata, PutObjectOptions options, File srcfile);
/**
- * Create a {@link PutObjectRequest} request.
+ * Create a {@link PutObjectRequest} request builder.
* The metadata is assumed to have been configured with the size of the
* operation.
* @param key key of object
- * @param metadata metadata header
* @param options options for the request
- * @param inputStream source data.
- * @return the request
+ * @param length length of object to be uploaded
+ * @param isDirectoryMarker true if object to be uploaded is a directory marker
+ * @return the request builder
*/
- PutObjectRequest newPutObjectRequest(String key,
- ObjectMetadata metadata,
+ PutObjectRequest.Builder newPutObjectRequestBuilder(String key,
PutObjectOptions options,
- InputStream inputStream);
+ long length,
+ boolean isDirectoryMarker);
/**
* Create a {@link PutObjectRequest} request for creating
* an empty directory.
*
* @param directory destination directory.
- * @return request for a zero byte upload.
+ * @return request builder for a zero byte upload.
*/
- PutObjectRequest newDirectoryMarkerRequest(String directory);
+ PutObjectRequest.Builder newDirectoryMarkerRequest(String directory);
/**
* List all multipart uploads under a prefix.
* @param prefix prefix to list under
- * @return the request.
+ * @return the request builder.
*/
- ListMultipartUploadsRequest newListMultipartUploadsRequest(
+ ListMultipartUploadsRequest.Builder newListMultipartUploadsRequestBuilder(
@Nullable String prefix);
/**
* Abort a multipart upload.
* @param destKey destination object key
* @param uploadId ID of initiated upload
- * @return the request.
+ * @return the request builder.
*/
- AbortMultipartUploadRequest newAbortMultipartUploadRequest(
+ AbortMultipartUploadRequest.Builder newAbortMultipartUploadRequestBuilder(
String destKey,
String uploadId);
@@ -195,10 +157,10 @@ AbortMultipartUploadRequest newAbortMultipartUploadRequest(
* Start a multipart upload.
* @param destKey destination object key
* @param options options for the request
- * @return the request.
+ * @return the request builder.
* @throws PathIOException if multipart uploads are disabled
*/
- InitiateMultipartUploadRequest newMultipartUploadRequest(
+ CreateMultipartUploadRequest.Builder newMultipartUploadRequestBuilder(
String destKey,
@Nullable PutObjectOptions options) throws PathIOException;
@@ -207,107 +169,95 @@ InitiateMultipartUploadRequest newMultipartUploadRequest(
* @param destKey destination object key
* @param uploadId ID of initiated upload
* @param partETags ordered list of etags
- * @return the request.
+ * @return the request builder.
*/
- CompleteMultipartUploadRequest newCompleteMultipartUploadRequest(
+ CompleteMultipartUploadRequest.Builder newCompleteMultipartUploadRequestBuilder(
String destKey,
String uploadId,
- List partETags);
+ List partETags);
/**
- * Create a HEAD request.
+ * Create a HEAD object request builder.
* @param key key, may have trailing /
- * @return the request.
+ * @return the request builder.
+ */
+ HeadObjectRequest.Builder newHeadObjectRequestBuilder(String key);
+
+ /**
+ * Create a HEAD bucket request builder.
+ * @param bucket bucket to get metadata for
+ * @return the request builder.
*/
- GetObjectMetadataRequest newGetObjectMetadataRequest(String key);
+ HeadBucketRequest.Builder newHeadBucketRequestBuilder(String bucket);
+
/**
- * Create a GET request.
+ * Create a GET request builder.
* @param key object key
- * @return the request.
+ * @return the request builder.
*/
- GetObjectRequest newGetObjectRequest(String key);
+ GetObjectRequest.Builder newGetObjectRequestBuilder(String key);
/**
- * Create and initialize a part request of a multipart upload.
- * Exactly one of: {@code uploadStream} or {@code sourceFile}
- * must be specified.
- * A subset of the file may be posted, by providing the starting point
- * in {@code offset} and a length of block in {@code size} equal to
- * or less than the remaining bytes.
- * @param destKey destination key of ongoing operation
- * @param uploadId ID of ongoing upload
- * @param partNumber current part number of the upload
- * @param size amount of data
- * @param uploadStream source of data to upload
- * @param sourceFile optional source file.
- * @param offset offset in file to start reading.
- * @return the request.
+ * Create and initialize a part request builder of a multipart upload.
+ *
+ * @param destKey destination key of ongoing operation
+ * @param uploadId ID of ongoing upload
+ * @param partNumber current part number of the upload
+ * @param size amount of data
+ * @return the request builder.
* @throws PathIOException if the part number is out of range.
*/
- UploadPartRequest newUploadPartRequest(
+ UploadPartRequest.Builder newUploadPartRequestBuilder(
String destKey,
String uploadId,
int partNumber,
- long size,
- InputStream uploadStream,
- File sourceFile,
- long offset) throws PathIOException;
+ long size) throws PathIOException;
/**
- * Create a S3 Select request for the destination object.
+ * Create a S3 Select request builder for the destination object.
* This does not build the query.
* @param key object key
- * @return the request
+ * @return the request builder
*/
- SelectObjectContentRequest newSelectRequest(String key);
+ SelectObjectContentRequest.Builder newSelectRequestBuilder(String key);
/**
- * Create the (legacy) V1 list request.
+ * Create the (legacy) V1 list request builder.
* @param key key to list under
* @param delimiter delimiter for keys
* @param maxKeys maximum number in a list page.
- * @return the request
+ * @return the request builder.
*/
- ListObjectsRequest newListObjectsV1Request(String key,
+ ListObjectsRequest.Builder newListObjectsV1RequestBuilder(String key,
String delimiter,
int maxKeys);
/**
- * Create the next V1 page list request, following
- * on from the previous response.
- * @param prev previous response
- * @return the request
- */
-
- ListNextBatchOfObjectsRequest newListNextBatchOfObjectsRequest(
- ObjectListing prev);
-
- /**
- * Create a V2 list request.
+ * Create a V2 list request builder.
* This will be recycled for any subsequent requests.
* @param key key to list under
* @param delimiter delimiter for keys
* @param maxKeys maximum number in a list page.
- * @return the request
+ * @return the request builder.
*/
- ListObjectsV2Request newListObjectsV2Request(String key,
+ ListObjectsV2Request.Builder newListObjectsV2RequestBuilder(String key,
String delimiter,
int maxKeys);
/**
- * Create a request to delete a single object.
+ * Create a request builder to delete a single object.
* @param key object to delete
- * @return the request
+ * @return the request builder.
*/
- DeleteObjectRequest newDeleteObjectRequest(String key);
+ DeleteObjectRequest.Builder newDeleteObjectRequestBuilder(String key);
/**
- * Bulk delete request.
+ * Create a request builder to delete objects in bulk.
* @param keysToDelete list of keys to delete.
- * @return the request
+ * @return the request builder.
*/
- DeleteObjectsRequest newBulkDeleteRequest(
- List keysToDelete);
+ DeleteObjectsRequest.Builder newBulkDeleteRequestBuilder(
+ List keysToDelete);
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java
index 8134d5cea942e..712246c21961c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java
@@ -18,15 +18,9 @@
package org.apache.hadoop.fs.s3a.audit;
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.Request;
-import com.amazonaws.Response;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.handlers.HandlerAfterAttemptContext;
-import com.amazonaws.handlers.HandlerBeforeAttemptContext;
-import com.amazonaws.http.HttpResponse;
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
-import org.apache.hadoop.fs.s3a.Retries;
/**
* Callbacks for audit spans. This is implemented
@@ -37,10 +31,10 @@
* detect this and raise an exception.
*
* Look at the documentation for
- * {@code com.amazonaws.handlers.IRequestHandler2} for details
+ * {@code ExecutionInterceptor} for details
* on the callbacks.
*/
-public interface AWSAuditEventCallbacks {
+public interface AWSAuditEventCallbacks extends ExecutionInterceptor {
/**
* Return a span ID which must be unique for all spans within
@@ -66,95 +60,8 @@ public interface AWSAuditEventCallbacks {
* It is not invoked on any AWS requests created in the SDK.
* Avoid raising exceptions or talking to any remote service;
* this callback is for annotation rather than validation.
- * @param request request request.
- * @param type of request
- * @return the request, possibly modified.
+ * @param builder the request builder.
*/
- default T requestCreated(T request) {
- return request;
- }
+ default void requestCreated(SdkRequest.Builder builder) {}
- /**
- * Preflight preparation of AWS request.
- * @param request request
- * @param type of request
- * @return an updated request.
- * @throws AuditFailureException for generic audit failures
- * @throws SdkBaseException for other reasons.
- */
- @Retries.OnceRaw
- default T beforeExecution(T request)
- throws AuditFailureException, SdkBaseException {
- return request;
- }
-
- /**
- * Callback after S3 responded to a request.
- * @param request request
- * @param response response.
- * @throws AuditFailureException for generic audit failures
- * @throws SdkBaseException for other reasons.
- */
- default void afterResponse(Request request,
- Response response)
- throws AuditFailureException, SdkBaseException {
- }
-
- /**
- * Callback after a request resulted in an error.
- * @param request request
- * @param response response.
- * @param exception exception raised.
- * @throws AuditFailureException for generic audit failures
- * @throws SdkBaseException for other reasons.
- */
- default void afterError(Request request,
- Response response,
- Exception exception)
- throws AuditFailureException, SdkBaseException {
- }
-
- /**
- * Request before marshalling.
- * @param request request
- * @return possibly modified request.
- */
- default AmazonWebServiceRequest beforeMarshalling(
- AmazonWebServiceRequest request) {
- return request;
- }
-
- /**
- * Request before marshalling.
- * @param request request
- */
- default void beforeRequest(Request request) {
- }
-
- /**
- * Before any attempt is made.
- * @param context full context, including the request.
- */
- default void beforeAttempt(HandlerBeforeAttemptContext context) {
- }
-
- /**
- * After any attempt is made.
- * @param context full context, including the request.
- */
- default void afterAttempt(
- HandlerAfterAttemptContext context) {
- }
-
- /**
- * Before unmarshalling the response.
- * @param request request made.
- * @param httpResponse response received
- * @return updated response.
- */
- default HttpResponse beforeUnmarshalling(
- final Request request,
- final HttpResponse httpResponse) {
- return httpResponse;
- }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
index b4be341c912e0..8a24a4e14db1c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
@@ -20,24 +20,23 @@
import java.util.List;
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CopyPartRequest;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.GetBucketLocationRequest;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import com.amazonaws.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import static org.apache.hadoop.fs.statistics.StoreStatisticNames.ACTION_HTTP_GET_REQUEST;
import static org.apache.hadoop.fs.statistics.StoreStatisticNames.ACTION_HTTP_HEAD_REQUEST;
@@ -64,102 +63,85 @@ public class AWSRequestAnalyzer {
* read/write and path.
* @param request request.
* @return information about the request.
- * @param type of request.
*/
- public RequestInfo analyze(T request) {
+ public RequestInfo analyze(SdkRequest request) {
// this is where Scala's case statement would massively
// simplify life.
// Please Keep in Alphabetical Order.
if (request instanceof AbortMultipartUploadRequest) {
return writing(MULTIPART_UPLOAD_ABORTED,
- ((AbortMultipartUploadRequest) request).getKey(),
+ ((AbortMultipartUploadRequest) request).key(),
0);
} else if (request instanceof CompleteMultipartUploadRequest) {
CompleteMultipartUploadRequest r
= (CompleteMultipartUploadRequest) request;
return writing(MULTIPART_UPLOAD_COMPLETED,
- r.getKey(),
- r.getPartETags().size());
+ r.key(),
+ r.multipartUpload().parts().size());
+ } else if (request instanceof CreateMultipartUploadRequest) {
+ return writing(MULTIPART_UPLOAD_STARTED,
+ ((CreateMultipartUploadRequest) request).key(),
+ 0);
} else if (request instanceof DeleteObjectRequest) {
// DeleteObject: single object
return writing(OBJECT_DELETE_REQUEST,
- ((DeleteObjectRequest) request).getKey(),
+ ((DeleteObjectRequest) request).key(),
1);
} else if (request instanceof DeleteObjectsRequest) {
// DeleteObjects: bulk delete
// use first key as the path
DeleteObjectsRequest r = (DeleteObjectsRequest) request;
- List keys
- = r.getKeys();
+ List objectIdentifiers
+ = r.delete().objects();
return writing(OBJECT_BULK_DELETE_REQUEST,
- keys.isEmpty() ? null : keys.get(0).getKey(),
- keys.size());
+ objectIdentifiers.isEmpty() ? null : objectIdentifiers.get(0).key(),
+ objectIdentifiers.size());
} else if (request instanceof GetBucketLocationRequest) {
GetBucketLocationRequest r = (GetBucketLocationRequest) request;
return reading(STORE_EXISTS_PROBE,
- r.getBucketName(),
+ r.bucket(),
0);
- } else if (request instanceof GetObjectMetadataRequest) {
- return reading(ACTION_HTTP_HEAD_REQUEST,
- ((GetObjectMetadataRequest) request).getKey(), 0);
} else if (request instanceof GetObjectRequest) {
GetObjectRequest r = (GetObjectRequest) request;
- long[] range = r.getRange();
- long size = range == null
- ? -1
- : range[1] - range[0];
return reading(ACTION_HTTP_GET_REQUEST,
- r.getKey(),
- size);
- } else if (request instanceof InitiateMultipartUploadRequest) {
- return writing(MULTIPART_UPLOAD_STARTED,
- ((InitiateMultipartUploadRequest) request).getKey(),
- 0);
+ r.key(),
+ sizeFromRangeHeader(r.range()));
+ } else if (request instanceof HeadObjectRequest) {
+ return reading(ACTION_HTTP_HEAD_REQUEST,
+ ((HeadObjectRequest) request).key(), 0);
} else if (request instanceof ListMultipartUploadsRequest) {
ListMultipartUploadsRequest r
= (ListMultipartUploadsRequest) request;
return reading(MULTIPART_UPLOAD_LIST,
- r.getPrefix(),
- r.getMaxUploads());
+ r.prefix(),
+ r.maxUploads());
} else if (request instanceof ListObjectsRequest) {
ListObjectsRequest r = (ListObjectsRequest) request;
return reading(OBJECT_LIST_REQUEST,
- r.getPrefix(),
- r.getMaxKeys());
- } else if (request instanceof ListNextBatchOfObjectsRequest) {
- ListNextBatchOfObjectsRequest r = (ListNextBatchOfObjectsRequest) request;
- ObjectListing l = r.getPreviousObjectListing();
- String prefix = "";
- int size = 0;
- if (l != null) {
- prefix = l.getPrefix();
- size = l.getMaxKeys();
- }
- return reading(OBJECT_LIST_REQUEST,
- prefix,
- size);
+ r.prefix(),
+ r.maxKeys());
} else if (request instanceof ListObjectsV2Request) {
ListObjectsV2Request r = (ListObjectsV2Request) request;
return reading(OBJECT_LIST_REQUEST,
- r.getPrefix(),
- r.getMaxKeys());
+ r.prefix(),
+ r.maxKeys());
} else if (request instanceof PutObjectRequest) {
PutObjectRequest r = (PutObjectRequest) request;
return writing(OBJECT_PUT_REQUEST,
- r.getKey(),
+ r.key(),
0);
} else if (request instanceof SelectObjectContentRequest) {
SelectObjectContentRequest r =
(SelectObjectContentRequest) request;
return reading(OBJECT_SELECT_REQUESTS,
- r.getKey(),
+ r.key(),
1);
} else if (request instanceof UploadPartRequest) {
UploadPartRequest r = (UploadPartRequest) request;
return writing(MULTIPART_UPLOAD_PART_PUT,
- r.getKey(),
- r.getPartSize());
+ r.key(),
+ r.contentLength());
}
// no explicit support, return classname
return writing(request.getClass().getName(), null, 0);
@@ -212,7 +194,7 @@ private RequestInfo writing(final String verb,
*/
public static boolean
isRequestNotAlwaysInSpan(final Object request) {
- return request instanceof CopyPartRequest
+ return request instanceof UploadPartCopyRequest
|| request instanceof CompleteMultipartUploadRequest
|| request instanceof GetBucketLocationRequest;
}
@@ -225,9 +207,9 @@ private RequestInfo writing(final String verb,
* @return true if the transfer manager creates them.
*/
public static boolean isRequestMultipartIO(final Object request) {
- return request instanceof CopyPartRequest
+ return request instanceof UploadPartCopyRequest
|| request instanceof CompleteMultipartUploadRequest
- || request instanceof InitiateMultipartUploadRequest
+ || request instanceof CreateMultipartUploadRequest
|| request instanceof UploadPartRequest;
}
@@ -307,4 +289,23 @@ public String toString() {
private static long toSafeLong(final Number size) {
return size != null ? size.longValue() : 0;
}
+
+ private static final String BYTES_PREFIX = "bytes=";
+
+ private static Number sizeFromRangeHeader(String rangeHeader) {
+ if (rangeHeader != null && rangeHeader.startsWith(BYTES_PREFIX)) {
+ String[] values = rangeHeader
+ .substring(BYTES_PREFIX.length())
+ .split("-");
+ if (values.length == 2) {
+ try {
+ long start = Long.parseUnsignedLong(values[0]);
+ long end = Long.parseUnsignedLong(values[0]);
+ return end - start;
+ } catch(NumberFormatException e) {
+ }
+ }
+ }
+ return -1;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
index c66f45eb309ff..dfbe8e1a13502 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
@@ -23,7 +23,7 @@
import java.lang.reflect.InvocationTargetException;
import java.nio.file.AccessDeniedException;
-import com.amazonaws.HandlerContextAware;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,10 +34,11 @@
import org.apache.hadoop.fs.s3a.audit.impl.NoopAuditManagerS3A;
import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+
import static java.util.Objects.requireNonNull;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED_DEFAULT;
-import static org.apache.hadoop.fs.s3a.audit.impl.S3AInternalAuditConstants.AUDIT_SPAN_HANDLER_CONTEXT;
+import static org.apache.hadoop.fs.s3a.audit.impl.S3AInternalAuditConstants.AUDIT_SPAN_EXECUTION_ATTRIBUTE;
/**
* Support for integrating auditing within the S3A code.
@@ -123,25 +124,24 @@ public static OperationAuditor createAndInitAuditor(
}
/**
- * Get the span from a handler context.
- * @param request request
- * @param type of request.
+ * Get the span from the execution attributes.
+ * @param executionAttributes the execution attributes
* @return the span callbacks or null
*/
- public static AWSAuditEventCallbacks
- retrieveAttachedSpan(final T request) {
- return request.getHandlerContext(AUDIT_SPAN_HANDLER_CONTEXT);
+ public static AuditSpanS3A
+ retrieveAttachedSpan(final ExecutionAttributes executionAttributes) {
+ return executionAttributes.getAttribute(AUDIT_SPAN_EXECUTION_ATTRIBUTE);
}
/**
- * Attach a span to a handler context.
- * @param request request
+ * Attach a span to the execution attributes.
+ * @param executionAttributes the execution attributes
* @param span span to attach
- * @param type of request.
*/
- public static void attachSpanToRequest(
- final T request, final AWSAuditEventCallbacks span) {
- request.addHandlerContext(AUDIT_SPAN_HANDLER_CONTEXT, span);
+ public static void attachSpanToRequest(
+ final ExecutionAttributes executionAttributes,
+ final AuditSpanS3A span) {
+ executionAttributes.putAttribute(AUDIT_SPAN_EXECUTION_ATTRIBUTE, span);
}
/**
@@ -160,4 +160,30 @@ public static IOException translateAuditException(String path,
return (AccessDeniedException)new AccessDeniedException(path, null,
exception.toString()).initCause(exception);
}
+
+ /**
+ * Translate an exception if it or its inner exception is an
+ * {@link AuditFailureException}.
+ * If this condition is not met, null is returned.
+ * @param path path of operation.
+ * @param exception exception
+ * @return a translated exception or null.
+ */
+ public static IOException maybeTranslateAuditException(String path,
+ Exception exception) {
+ if (exception instanceof AuditFailureException) {
+ return translateAuditException(path, (AuditFailureException) exception);
+ } else if (exception.getCause() instanceof AuditFailureException) {
+ return translateAuditException(path,
+ (AuditFailureException) exception.getCause());
+ } else {
+ return null;
+ }
+ }
+
+ public static boolean containsAuditException(Exception exception) {
+ return exception instanceof AuditFailureException
+ || exception.getCause() instanceof AuditFailureException;
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java
index c1302d57454b1..bb7f94cfc206d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java
@@ -21,8 +21,8 @@
import java.io.IOException;
import java.util.List;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.transfer.s3.progress.TransferListener;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.Path;
@@ -32,6 +32,7 @@
import org.apache.hadoop.fs.store.audit.AuditSpanSource;
import org.apache.hadoop.service.Service;
+
/**
* Interface for Audit Managers auditing operations through the
* AWS libraries.
@@ -56,24 +57,24 @@ public interface AuditManagerS3A extends Service,
OperationAuditor getAuditor();
/**
- * Create the request handler(s) for this audit service.
- * The list returned is mutable; new handlers may be added.
- * @return list of handlers for the SDK.
+ * Create the execution interceptor(s) for this audit service.
+ * The list returned is mutable; new interceptors may be added.
+ * @return list of interceptors for the SDK.
* @throws IOException failure.
*/
- List createRequestHandlers() throws IOException;
+ List createExecutionInterceptors() throws IOException;
/**
- * Return a transfer state change callback which
+ * Return a transfer callback which
* fixes the active span context to be that in which
- * the state change listener was created.
+ * the transfer listener was created.
* This can be used to audit the creation of the multipart
* upload initiation request which the transfer manager
* makes when a file to be copied is split up.
* This must be invoked/used within the active span.
- * @return a state change listener.
+ * @return a transfer listener.
*/
- TransferStateChangeListener createStateChangeListener();
+ TransferListener createTransferListener();
/**
* Check for permission to access a path.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
index 1d76833f8ceab..55deb0a1989de 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
@@ -66,13 +66,20 @@ private S3AAuditConstants() {
"org.apache.hadoop.fs.s3a.audit.impl.NoopAuditor";
/**
- * List of extra AWS SDK request handlers: {@value}.
- * These are added to the SDK request chain after
- * any audit service.
+ * Deprecated list of extra AWS SDK request handlers: {@value}.
+ * Use {@link #AUDIT_EXECUTION_INTERCEPTORS} instead.
*/
public static final String AUDIT_REQUEST_HANDLERS =
"fs.s3a.audit.request.handlers";
+ /**
+ * List of extra AWS SDK execution interceptors: {@value}.
+ * These are added to the SDK request chain after
+ * any audit service.
+ */
+ public static final String AUDIT_EXECUTION_INTERCEPTORS =
+ "fs.s3a.audit.execution.interceptors";
+
/**
* Should operations outside spans be rejected?
* This is for testing coverage of the span code; if used
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
index ffd08d37e4aa2..1da203dc80249 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
@@ -25,16 +25,16 @@
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.HandlerContextAware;
-import com.amazonaws.Request;
-import com.amazonaws.Response;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.handlers.HandlerAfterAttemptContext;
-import com.amazonaws.handlers.HandlerBeforeAttemptContext;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.http.HttpResponse;
-import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.core.SdkResponse;
+import software.amazon.awssdk.core.interceptor.Context;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.http.SdkHttpRequest;
+import software.amazon.awssdk.http.SdkHttpResponse;
+import software.amazon.awssdk.transfer.s3.progress.TransferListener;
+
+import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,6 +56,7 @@
import org.apache.hadoop.fs.s3a.audit.OperationAuditor;
import org.apache.hadoop.fs.s3a.audit.OperationAuditorOptions;
import org.apache.hadoop.fs.s3a.audit.S3AAuditConstants;
+import org.apache.hadoop.fs.s3a.impl.V2Migration;
import org.apache.hadoop.fs.store.LogExactlyOnce;
import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
import org.apache.hadoop.service.CompositeService;
@@ -66,6 +67,7 @@
import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_REQUEST_EXECUTION;
import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.attachSpanToRequest;
import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.retrieveAttachedSpan;
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_EXECUTION_INTERCEPTORS;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_REQUEST_HANDLERS;
/**
@@ -82,10 +84,11 @@
* will deactivate the wrapped span and then
* switch the active span to the unbounded span.
*
- * The inner class {@link AWSAuditEventCallbacks} is returned
- * as a request handler in {@link #createRequestHandlers()};
- * this forwards all requests to the outer {@code ActiveAuditManagerS3A},
- * which then locates the active span and forwards the request.
+ * This class also implements {@link ExecutionInterceptor} and
+ * returns itself in {@link #createExecutionInterceptors()};
+ * once registered with the S3 client, the implemented methods
+ * will be called during different parts of an SDK request lifecycle,
+ * which then locate the active span and forward the request.
* If any such invocation raises an {@link AuditFailureException}
* then the IOStatistics counter for {@code AUDIT_FAILURE}
* is incremented.
@@ -390,25 +393,39 @@ public AuditSpanS3A createSpan(final String operation,
}
/**
- * Return a request handler for the AWS SDK which
+ * Return a list of execution interceptors for the AWS SDK which
* relays to this class.
- * @return a request handler.
+ * @return a list of execution interceptors.
*/
@Override
- public List createRequestHandlers()
+ public List createExecutionInterceptors()
throws IOException {
// wire up the AWS SDK To call back into this class when
// preparing to make S3 calls.
- List requestHandlers = new ArrayList<>();
- requestHandlers.add(new SdkRequestHandler());
- // now look for any more handlers
- final Class[] handlers = getConfig().getClasses(AUDIT_REQUEST_HANDLERS);
- if (handlers != null) {
- for (Class handler : handlers) {
+ List executionInterceptors = new ArrayList<>();
+ executionInterceptors.add(this);
+
+ final String handlers = getConfig().getTrimmed(AUDIT_REQUEST_HANDLERS, "");
+ if (!handlers.isEmpty()) {
+ // warn and ignore v1 handlers.
+ V2Migration.v1RequestHandlersUsed(handlers);
+ }
+
+ // V2 SDK supports global/service interceptors, but they need to be configured on the
+ // classpath and don't get the filesystem/job configuration passed down.
+ final Class[] interceptors = getConfig().getClasses(AUDIT_EXECUTION_INTERCEPTORS);
+ if (interceptors != null) {
+ for (Class handler : interceptors) {
try {
+ LOG.debug("Adding intercept of class {}", handler);
Constructor ctor = handler.getConstructor();
- requestHandlers.add((RequestHandler2)ctor.newInstance());
+ final ExecutionInterceptor interceptor = (ExecutionInterceptor) ctor.newInstance();
+ if (interceptor instanceof Configurable) {
+ // pass in the configuration.
+ ((Configurable) interceptor).setConf(getConfig());
+ }
+ executionInterceptors.add(interceptor);
} catch (ExceptionInInitializerError e) {
throw FutureIO.unwrapInnerException(e);
} catch (Exception e) {
@@ -416,13 +433,18 @@ public List createRequestHandlers()
}
}
}
- return requestHandlers;
+ return executionInterceptors;
}
@Override
- public TransferStateChangeListener createStateChangeListener() {
+ public TransferListener createTransferListener() {
final WrappingAuditSpan span = activeSpan();
- return (transfer, state) -> switchToActiveSpan(span);
+ return new TransferListener() {
+ @Override
+ public void transferInitiated(Context.TransferInitiated context) {
+ switchToActiveSpan(span);
+ }
+ };
}
@Override
@@ -434,20 +456,18 @@ public boolean checkAccess(final Path path,
}
/**
- * Attach a reference to the active thread span, then
- * invoke the same callback on that active thread.
+ * Audit the creation of a request and retrieve
+ * a reference to the active thread span.
*/
@Override
- public T requestCreated(
- final T request) {
+ public void requestCreated(final SdkRequest.Builder builder) {
AuditSpanS3A span = getActiveAuditSpan();
if (LOG.isTraceEnabled()) {
LOG.trace("Created Request {} in span {}",
- analyzer.analyze(request), span);
+ analyzer.analyze(builder.build()), span);
}
- attachSpanToRequest(request, span);
try {
- return span.requestCreated(request);
+ span.requestCreated(builder);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -463,14 +483,13 @@ public T requestCreated(
* {@inheritDoc}
*/
@Override
- public T beforeExecution(
- final T request) {
+ public void beforeExecution(Context.BeforeExecution context,
+ ExecutionAttributes executionAttributes) {
ioStatisticsStore.incrementCounter(AUDIT_REQUEST_EXECUTION.getSymbol());
-
- // identify the span and invoke the callback
+ AuditSpanS3A span = getActiveAuditSpan();
+ attachSpanToRequest(executionAttributes, span);
try {
- return extractAndActivateSpanFromRequest(request)
- .beforeExecution(request);
+ span.beforeExecution(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -479,16 +498,14 @@ public T beforeExecution(
/**
* Forward to active span.
- * @param request request
- * @param response response.
+ * {@inheritDoc}
*/
@Override
- public void afterResponse(final Request request,
- final Response response)
- throws AuditFailureException, SdkBaseException {
+ public void afterExecution(Context.AfterExecution context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(request)
- .afterResponse(request, response);
+ extractAndActivateSpanFromRequest(context.request(), executionAttributes)
+ .afterExecution(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -496,18 +513,19 @@ public void afterResponse(final Request request,
}
/**
- * Get the active span from the handler context,
+ * Get the active span from the execution attributes,
* falling back to the active thread span if there
- * is nothing in the context.
- * Provided the span is a wrapped span, the
+ * is nothing in the attributes.
+ * Provided the span is a wrapped span, the span is
+ * activated.
* @param request request
- * @param type of request.
- * @return the callbacks
+ * @param executionAttributes the execution attributes
+ * @return the active span
*/
- private AWSAuditEventCallbacks
- extractAndActivateSpanFromRequest(final T request) {
- AWSAuditEventCallbacks span;
- span = retrieveAttachedSpan(request);
+ private AuditSpanS3A extractAndActivateSpanFromRequest(
+ final SdkRequest request,
+ final ExecutionAttributes executionAttributes) {
+ AuditSpanS3A span = retrieveAttachedSpan(executionAttributes);
if (span == null) {
// no span is attached. Not unusual for the copy operations,
// or for calls to GetBucketLocation made by the AWS client
@@ -530,18 +548,16 @@ public void afterResponse(final Request request,
/**
* Forward to active span.
- * @param request request
- * @param response response.
- * @param exception exception raised.
+ * @param context execution context
+ * @param executionAttributes the execution attributes
+ * {@inheritDoc}
*/
@Override
- public void afterError(final Request request,
- final Response response,
- final Exception exception)
- throws AuditFailureException, SdkBaseException {
+ public void onExecutionFailure(Context.FailedExecution context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(request)
- .afterError(request, response, exception);
+ extractAndActivateSpanFromRequest(context.request(), executionAttributes).onExecutionFailure(
+ context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -549,11 +565,12 @@ public void afterError(final Request request,
}
@Override
- public AmazonWebServiceRequest beforeMarshalling(
- final AmazonWebServiceRequest request) {
+ public SdkRequest modifyRequest(Context.ModifyRequest context,
+ ExecutionAttributes executionAttributes) {
try {
- return extractAndActivateSpanFromRequest(request)
- .beforeMarshalling(request);
+ return extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .modifyRequest(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -561,10 +578,12 @@ public AmazonWebServiceRequest beforeMarshalling(
}
@Override
- public void beforeRequest(final Request request) {
+ public void beforeMarshalling(Context.BeforeMarshalling context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(request)
- .beforeRequest(request);
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .beforeMarshalling(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -572,10 +591,12 @@ public void beforeRequest(final Request request) {
}
@Override
- public void beforeAttempt(final HandlerBeforeAttemptContext context) {
+ public void afterMarshalling(Context.AfterMarshalling context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(context.getRequest())
- .beforeAttempt(context);
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .afterMarshalling(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -583,10 +604,12 @@ public void beforeAttempt(final HandlerBeforeAttemptContext context) {
}
@Override
- public void afterAttempt(final HandlerAfterAttemptContext context) {
+ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(context.getRequest())
- .afterAttempt(context);
+ return extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .modifyHttpRequest(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
@@ -594,73 +617,80 @@ public void afterAttempt(final HandlerAfterAttemptContext context) {
}
@Override
- public HttpResponse beforeUnmarshalling(final Request request,
- final HttpResponse httpResponse) {
+ public void beforeTransmission(Context.BeforeTransmission context,
+ ExecutionAttributes executionAttributes) {
try {
- extractAndActivateSpanFromRequest(request.getOriginalRequest())
- .beforeUnmarshalling(request, httpResponse);
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .beforeTransmission(context, executionAttributes);
} catch (AuditFailureException e) {
ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
throw e;
}
- return httpResponse;
}
- /**
- * Callbacks from the AWS SDK; all forward to the ActiveAuditManagerS3A.
- * We need a separate class because the SDK requires the handler list
- * to be list of {@code RequestHandler2} instances.
- */
- private class SdkRequestHandler extends RequestHandler2 {
-
- @Override
- public AmazonWebServiceRequest beforeExecution(
- final AmazonWebServiceRequest request) {
- return ActiveAuditManagerS3A.this.beforeExecution(request);
- }
-
- @Override
- public void afterResponse(final Request request,
- final Response response) {
- ActiveAuditManagerS3A.this.afterResponse(request, response);
- }
-
- @Override
- public void afterError(final Request request,
- final Response response,
- final Exception e) {
- ActiveAuditManagerS3A.this.afterError(request, response, e);
- }
-
- @Override
- public AmazonWebServiceRequest beforeMarshalling(
- final AmazonWebServiceRequest request) {
- return ActiveAuditManagerS3A.this.beforeMarshalling(request);
+ @Override
+ public void afterTransmission(Context.AfterTransmission context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .afterTransmission(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
+ }
- @Override
- public void beforeRequest(final Request request) {
- ActiveAuditManagerS3A.this.beforeRequest(request);
+ @Override
+ public SdkHttpResponse modifyHttpResponse(Context.ModifyHttpResponse context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ return extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .modifyHttpResponse(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
+ }
- @Override
- public void beforeAttempt(
- final HandlerBeforeAttemptContext context) {
- ActiveAuditManagerS3A.this.beforeAttempt(context);
+ @Override
+ public void beforeUnmarshalling(Context.BeforeUnmarshalling context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .beforeUnmarshalling(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
+ }
- @Override
- public HttpResponse beforeUnmarshalling(
- final Request request,
- final HttpResponse httpResponse) {
- return ActiveAuditManagerS3A.this.beforeUnmarshalling(request,
- httpResponse);
+ @Override
+ public void afterUnmarshalling(Context.AfterUnmarshalling context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .afterUnmarshalling(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
+ }
- @Override
- public void afterAttempt(
- final HandlerAfterAttemptContext context) {
- ActiveAuditManagerS3A.this.afterAttempt(context);
+ @Override
+ public SdkResponse modifyResponse(Context.ModifyResponse context,
+ ExecutionAttributes executionAttributes) {
+ try {
+ return extractAndActivateSpanFromRequest(context.request(),
+ executionAttributes)
+ .modifyResponse(context, executionAttributes);
+ } catch (AuditFailureException e) {
+ ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol());
+ throw e;
}
}
@@ -748,9 +778,8 @@ public void deactivate() {
* {@inheritDoc}
*/
@Override
- public T requestCreated(
- final T request) {
- return span.requestCreated(request);
+ public void requestCreated(final SdkRequest.Builder builder) {
+ span.requestCreated(builder);
}
/**
@@ -774,79 +803,132 @@ public void set(final String key, final String value) {
/**
* Forward to the inner span.
- * @param request request
- * @param type of request
- * @return an updated request.
+ * {@inheritDoc}
*/
@Override
- public T beforeExecution(
- final T request) {
- return span.beforeExecution(request);
+ public void beforeExecution(Context.BeforeExecution context,
+ ExecutionAttributes executionAttributes) {
+ span.beforeExecution(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param request request
- * @param response response.
+ * {@inheritDoc}
*/
@Override
- public void afterResponse(final Request request,
- final Response response) {
- span.afterResponse(request, response);
+ public void afterExecution(Context.AfterExecution context,
+ ExecutionAttributes executionAttributes) {
+ span.afterExecution(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param request request
- * @param response response.
- * @param exception exception raised.
+ * {@inheritDoc}
*/
@Override
- public void afterError(final Request request,
- final Response response,
- final Exception exception) {
- span.afterError(request, response, exception);
+ public void onExecutionFailure(Context.FailedExecution context,
+ ExecutionAttributes executionAttributes) {
+ span.onExecutionFailure(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param request request
- * @return request to marshall
+ * {@inheritDoc}
*/
@Override
- public AmazonWebServiceRequest beforeMarshalling(
- final AmazonWebServiceRequest request) {
- return span.beforeMarshalling(request);
+ public void beforeMarshalling(Context.BeforeMarshalling context,
+ ExecutionAttributes executionAttributes) {
+ span.beforeMarshalling(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param request request
+ * {@inheritDoc}
*/
@Override
- public void beforeRequest(final Request request) {
- span.beforeRequest(request);
+ public SdkRequest modifyRequest(Context.ModifyRequest context,
+ ExecutionAttributes executionAttributes) {
+ return span.modifyRequest(context, executionAttributes);
}
/**
* Forward to the inner span.
- * @param context full context, including the request.
+ * {@inheritDoc}
*/
@Override
- public void beforeAttempt(
- final HandlerBeforeAttemptContext context) {
- span.beforeAttempt(context);
+ public void afterMarshalling(Context.AfterMarshalling context,
+ ExecutionAttributes executionAttributes) {
+ span.afterMarshalling(context, executionAttributes);
}
/**
* Forward to the inner span.
- *
- * @param context full context, including the request.
+ * {@inheritDoc}
*/
@Override
- public void afterAttempt(
- final HandlerAfterAttemptContext context) {
- span.afterAttempt(context);
+ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
+ ExecutionAttributes executionAttributes) {
+ return span.modifyHttpRequest(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public void beforeTransmission(Context.BeforeTransmission context,
+ ExecutionAttributes executionAttributes) {
+ span.beforeTransmission(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public void afterTransmission(Context.AfterTransmission context,
+ ExecutionAttributes executionAttributes) {
+ span.afterTransmission(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public SdkHttpResponse modifyHttpResponse(Context.ModifyHttpResponse context,
+ ExecutionAttributes executionAttributes) {
+ return span.modifyHttpResponse(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public void beforeUnmarshalling(Context.BeforeUnmarshalling context,
+ ExecutionAttributes executionAttributes) {
+ span.beforeUnmarshalling(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public void afterUnmarshalling(Context.AfterUnmarshalling context,
+ ExecutionAttributes executionAttributes) {
+ span.afterUnmarshalling(context, executionAttributes);
+ }
+
+ /**
+ * Forward to the inner span.
+ * {@inheritDoc}
+ */
+ @Override
+ public SdkResponse modifyResponse(Context.ModifyResponse context,
+ ExecutionAttributes executionAttributes) {
+ return span.modifyResponse(context, executionAttributes);
}
@Override
@@ -859,5 +941,4 @@ public String toString() {
return sb.toString();
}
}
-
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
index fcf2015487c48..3a2d9d7f823ee 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
@@ -24,10 +24,14 @@
import java.util.HashMap;
import java.util.Map;
-import com.amazonaws.AmazonWebServiceRequest;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.awscore.AwsExecutionAttribute;
+import software.amazon.awssdk.core.SdkRequest;
+import software.amazon.awssdk.core.interceptor.Context;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.http.SdkHttpRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -258,21 +262,22 @@ private class LoggingAuditSpan extends AbstractAuditSpanImpl {
/**
* Attach Range of data for GetObject Request.
- * @param request given get object request
+ * @param request the sdk request to be modified
+ * @param executionAttributes execution attributes for this request
*/
- private void attachRangeFromRequest(AmazonWebServiceRequest request) {
- if (request instanceof GetObjectRequest) {
- long[] rangeValue = ((GetObjectRequest) request).getRange();
- if (rangeValue == null || rangeValue.length == 0) {
- return;
- }
- if (rangeValue.length != 2) {
- WARN_INCORRECT_RANGE.warn("Expected range to contain 0 or 2 elements."
- + " Got {} elements. Ignoring.", rangeValue.length);
- return;
+ private void attachRangeFromRequest(SdkHttpRequest request,
+ ExecutionAttributes executionAttributes) {
+
+ String operationName = executionAttributes.getAttribute(AwsExecutionAttribute.OPERATION_NAME);
+
+ if (operationName != null && operationName.equals("GetObject")) {
+ if (request.headers() != null && request.headers().get("Range") != null) {
+ String[] rangeHeader = request.headers().get("Range").get(0).split("=");
+ // only set header if range unit is bytes
+ if (rangeHeader[0].equals("bytes")) {
+ referrer.set(AuditConstants.PARAM_RANGE, rangeHeader[1]);
+ }
}
- String combinedRangeValue = String.format("%d-%d", rangeValue[0], rangeValue[1]);
- referrer.set(AuditConstants.PARAM_RANGE, combinedRangeValue);
}
}
@@ -346,64 +351,78 @@ public void set(final String key, final String value) {
referrer.set(key, value);
}
+
+
/**
- * Before execution, the logging auditor always builds
- * the referrer header, saves to the outer class
- * (where {@link #getLastHeader()} can retrieve it,
+ * Before transmitting a request, the logging auditor
+ * always builds the referrer header, saves to the outer
+ * class (where {@link #getLastHeader()} can retrieve it,
* and logs at debug.
* If configured to add the header to the S3 logs, it will
* be set as the HTTP referrer.
- * @param request request
- * @param type of request.
- * @return the request with any extra headers.
+ * @param context The current state of the execution,
+ * including the SDK and current HTTP request.
+ * @param executionAttributes A mutable set of attributes scoped
+ * to one specific request/response
+ * cycle that can be used to give data
+ * to future lifecycle methods.
+ * @return The potentially-modified HTTP request that should be
+ * sent to the service. Must not be null.
*/
@Override
- public T beforeExecution(
- final T request) {
+ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
+ ExecutionAttributes executionAttributes) {
+ SdkHttpRequest httpRequest = context.httpRequest();
+ SdkRequest sdkRequest = context.request();
+
// attach range for GetObject requests
- attachRangeFromRequest(request);
+ attachRangeFromRequest(httpRequest, executionAttributes);
+
// for delete op, attach the number of files to delete
- attachDeleteKeySizeAttribute(request);
+ attachDeleteKeySizeAttribute(sdkRequest);
+
// build the referrer header
final String header = referrer.buildHttpReferrer();
// update the outer class's field.
setLastHeader(header);
if (headerEnabled) {
// add the referrer header
- request.putCustomRequestHeader(HEADER_REFERRER,
- header);
+ httpRequest = httpRequest.toBuilder()
+ .appendHeader(HEADER_REFERRER, header)
+ .build();
}
if (LOG.isDebugEnabled()) {
LOG.debug("[{}] {} Executing {} with {}; {}",
currentThreadID(),
getSpanId(),
getOperationName(),
- analyzer.analyze(request),
+ analyzer.analyze(context.request()),
header);
}
+
// now see if the request is actually a blocked multipart request
- if (!isMultipartUploadEnabled && isRequestMultipartIO(request)) {
+ if (!isMultipartUploadEnabled && isRequestMultipartIO(sdkRequest)) {
throw new AuditOperationRejectedException("Multipart IO request "
- + request + " rejected " + header);
+ + sdkRequest + " rejected " + header);
}
- return request;
+ return httpRequest;
}
/**
* For delete requests, attach delete key size as a referrer attribute.
*
* @param request the request object.
- * @param type of the request.
*/
- private void attachDeleteKeySizeAttribute(T request) {
+ private void attachDeleteKeySizeAttribute(SdkRequest request) {
+
if (request instanceof DeleteObjectsRequest) {
- int keySize = ((DeleteObjectsRequest) request).getKeys().size();
- this.set(DELETE_KEYS_SIZE, String.valueOf(keySize));
+ int keySize = ((DeleteObjectsRequest) request).delete().objects().size();
+ referrer.set(DELETE_KEYS_SIZE, String.valueOf(keySize));
} else if (request instanceof DeleteObjectRequest) {
- String key = ((DeleteObjectRequest) request).getKey();
+ String key = ((DeleteObjectRequest) request).key();
if (key != null && key.length() > 0) {
- this.set(DELETE_KEYS_SIZE, "1");
+ referrer.set(DELETE_KEYS_SIZE, "1");
}
}
}
@@ -460,15 +479,13 @@ public boolean isValidSpan() {
}
@Override
- public T requestCreated(
- final T request) {
+ public void requestCreated(final SdkRequest.Builder builder) {
String error = "Creating a request outside an audit span "
- + analyzer.analyze(request);
+ + analyzer.analyze(builder.build());
LOG.info(error);
if (LOG.isDebugEnabled()) {
LOG.debug(error, new AuditFailureException("unaudited"));
}
- return request;
}
/**
@@ -476,20 +493,22 @@ public T requestCreated(
* increment the failure count.
* Some requests (e.g. copy part) are not expected in spans due
* to how they are executed; these do not trigger failures.
- * @param request request
- * @param type of request
- * @return an updated request.
- * @throws AuditFailureException if failure is enabled.
+ * @param context The current state of the execution, including
+ * the unmodified SDK request from the service
+ * client call.
+ * @param executionAttributes A mutable set of attributes scoped
+ * to one specific request/response
+ * cycle that can be used to give data
+ * to future lifecycle methods.
*/
@Override
- public T beforeExecution(
- final T request) {
-
+ public void beforeExecution(Context.BeforeExecution context,
+ ExecutionAttributes executionAttributes) {
String error = "executing a request outside an audit span "
- + analyzer.analyze(request);
+ + analyzer.analyze(context.request());
final String unaudited = getSpanId() + " "
+ UNAUDITED_OPERATION + " " + error;
- if (isRequestNotAlwaysInSpan(request)) {
+ if (isRequestNotAlwaysInSpan(context.request())) {
// can get by auditing during a copy, so don't overreact
LOG.debug(unaudited);
} else {
@@ -500,7 +519,7 @@ public T beforeExecution(
}
}
// now hand off to the superclass for its normal preparation
- return super.beforeExecution(request);
+ super.beforeExecution(context, executionAttributes);
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
index d1ebd922e073d..e58c906460daa 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
@@ -24,9 +24,8 @@
import java.util.List;
import java.util.UUID;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.services.s3.transfer.Transfer;
-import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.transfer.s3.progress.TransferListener;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@ -39,6 +38,7 @@
import org.apache.hadoop.fs.s3a.audit.OperationAuditorOptions;
import org.apache.hadoop.service.CompositeService;
+
import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore;
/**
@@ -121,17 +121,13 @@ public AuditSpanS3A createSpan(final String operation,
}
@Override
- public List createRequestHandlers() throws IOException {
+ public List createExecutionInterceptors() throws IOException {
return new ArrayList<>();
}
@Override
- public TransferStateChangeListener createStateChangeListener() {
- return new TransferStateChangeListener() {
- public void transferStateChanged(final Transfer transfer,
- final Transfer.TransferState state) {
- }
- };
+ public TransferListener createTransferListener() {
+ return new TransferListener() {};
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java
index f82e3d7f1e5e8..c170a2be6611d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java
@@ -18,10 +18,10 @@
package org.apache.hadoop.fs.s3a.audit.impl;
-import com.amazonaws.handlers.HandlerContextKey;
+import software.amazon.awssdk.core.interceptor.ExecutionAttribute;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.s3a.audit.AWSAuditEventCallbacks;
+import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A;
/**
* Internal constants; not intended for public use, or
@@ -34,11 +34,11 @@ private S3AInternalAuditConstants() {
}
/**
- * Handler key for audit span callbacks.
- * This is used to bind the handler in the AWS code.
+ * Exceution attribute for audit span callbacks.
+ * This is used to retrieve the span in the AWS code.
*/
- public static final HandlerContextKey
- AUDIT_SPAN_HANDLER_CONTEXT =
- new HandlerContextKey<>(
- "org.apache.hadoop.fs.s3a.audit.AWSAuditEventCallbacks");
+ public static final ExecutionAttribute
+ AUDIT_SPAN_EXECUTION_ATTRIBUTE =
+ new ExecutionAttribute<>(
+ "org.apache.hadoop.fs.s3a.audit.AuditSpanS3A");
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java
index 1815285738b0e..4754427a4b118 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java
@@ -21,7 +21,7 @@
import javax.annotation.Nullable;
import java.net.URI;
-import com.amazonaws.auth.AWSCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import org.apache.hadoop.conf.Configuration;
@@ -29,12 +29,9 @@
* Base class for AWS credential providers which
* take a URI and config in their constructor.
*
- * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
- * as part of upgrading S3A to SDK V2. See HADOOP-18073.
*/
-@Deprecated
public abstract class AbstractAWSCredentialProvider
- implements AWSCredentialsProvider {
+ implements AwsCredentialsProvider {
private final URI binding;
@@ -65,10 +62,4 @@ public URI getUri() {
return binding;
}
- /**
- * Refresh is a no-op by default.
- */
- @Override
- public void refresh() {
- }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java
index 2cdf0880affbb..4ef9d49df1aeb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java
@@ -23,8 +23,8 @@
import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.auth.AWSCredentials;
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -33,19 +33,17 @@
import org.apache.hadoop.fs.s3a.Invoker;
import org.apache.hadoop.fs.s3a.Retries;
+
/**
* Base class for session credential support.
*
- * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
- * as part of upgrading S3A to SDK V2. See HADOOP-18073.
*/
@InterfaceAudience.Private
-@Deprecated
public abstract class AbstractSessionCredentialsProvider
extends AbstractAWSCredentialProvider {
/** Credentials, created in {@link #init()}. */
- private volatile AWSCredentials awsCredentials;
+ private volatile AwsCredentials awsCredentials;
/** Atomic flag for on-demand initialization. */
private final AtomicBoolean initialized = new AtomicBoolean(false);
@@ -105,7 +103,7 @@ public boolean isInitialized() {
* @return the credentials
* @throws IOException on any failure.
*/
- protected abstract AWSCredentials createCredentials(Configuration config)
+ protected abstract AwsCredentials createCredentials(Configuration config)
throws IOException;
/**
@@ -115,10 +113,10 @@ protected abstract AWSCredentials createCredentials(Configuration config)
* is thrown here before any attempt to return the credentials
* is made.
* @return credentials, if set.
- * @throws SdkBaseException if one was raised during init
+ * @throws SdkException if one was raised during init
* @throws CredentialInitializationException on other failures.
*/
- public AWSCredentials getCredentials() throws SdkBaseException {
+ public AwsCredentials resolveCredentials() throws SdkException {
// do an on-demand init then raise an AWS SDK exception if
// there was a failure.
try {
@@ -126,8 +124,8 @@ public AWSCredentials getCredentials() throws SdkBaseException {
init();
}
} catch (IOException e) {
- if (e.getCause() instanceof SdkBaseException) {
- throw (SdkBaseException) e.getCause();
+ if (e.getCause() instanceof SdkException) {
+ throw (SdkException) e.getCause();
} else {
throw new CredentialInitializationException(e.getMessage(), e);
}
@@ -165,15 +163,16 @@ public IOException getInitializationException() {
* This will be interpreted as "this provider has no credentials to offer",
* rather than an explicit error or anonymous access.
*/
- protected static final class NoCredentials implements AWSCredentials {
+ protected static final class NoCredentials implements AwsCredentials {
@Override
- public String getAWSAccessKeyId() {
+ public String accessKeyId() {
return null;
}
@Override
- public String getAWSSecretKey() {
+ public String secretAccessKey() {
return null;
}
}
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
index fe816ecfcd90e..626fe4d44683f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
@@ -26,21 +26,23 @@
import java.util.Locale;
import java.util.concurrent.TimeUnit;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
-import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
-import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
-import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
+import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.services.sts.StsClient;
+import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
+import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
+import software.amazon.awssdk.services.sts.model.StsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
import org.apache.hadoop.fs.s3a.CredentialInitializationException;
import org.apache.hadoop.fs.s3a.Retries;
@@ -49,9 +51,10 @@
import org.apache.hadoop.fs.s3a.S3ARetryPolicy;
import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.S3AUtils.buildAWSProviderList;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.buildAWSProviderList;
/**
* Support IAM Assumed roles by instantiating an instance of
@@ -61,13 +64,10 @@
*
* Classname is used in configuration files; do not move.
*
- * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
- * as part of upgrading S3A to SDK V2. See HADOOP-18073.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-@Deprecated
-public class AssumedRoleCredentialProvider implements AWSCredentialsProvider,
+public final class AssumedRoleCredentialProvider implements AwsCredentialsProvider,
Closeable {
private static final Logger LOG =
@@ -78,7 +78,7 @@ public class AssumedRoleCredentialProvider implements AWSCredentialsProvider,
public static final String E_NO_ROLE = "Unset property "
+ ASSUMED_ROLE_ARN;
- private final STSAssumeRoleSessionCredentialsProvider stsProvider;
+ private final StsAssumeRoleCredentialsProvider stsProvider;
private final String sessionName;
@@ -90,22 +90,24 @@ public class AssumedRoleCredentialProvider implements AWSCredentialsProvider,
private final Invoker invoker;
+ private final StsClient stsClient;
+
/**
* Instantiate.
- * This calls {@link #getCredentials()} to fail fast on the inner
+ * This calls {@link #resolveCredentials()} to fail fast on the inner
* role credential retrieval.
* @param fsUri possibly null URI of the filesystem.
* @param conf configuration
* @throws IOException on IO problems and some parameter checking
* @throws IllegalArgumentException invalid parameters
- * @throws AWSSecurityTokenServiceException problems getting credentials
+ * @throws StsException problems getting credentials
*/
public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf)
throws IOException {
arn = conf.getTrimmed(ASSUMED_ROLE_ARN, "");
if (StringUtils.isEmpty(arn)) {
- throw new IOException(E_NO_ROLE);
+ throw new PathIOException(String.valueOf(fsUri), E_NO_ROLE);
}
// build up the base provider
@@ -114,8 +116,8 @@ public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf)
Arrays.asList(
SimpleAWSCredentialsProvider.class,
EnvironmentVariableCredentialsProvider.class),
- Sets.newHashSet(this.getClass()));
- LOG.debug("Credentials to obtain role credentials: {}", credentialsToSTS);
+ Sets.newHashSet(getClass()));
+ LOG.debug("Credentials used to obtain role credentials: {}", credentialsToSTS);
// then the STS binding
sessionName = conf.getTrimmed(ASSUMED_ROLE_SESSION_NAME,
@@ -125,29 +127,31 @@ public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf)
String policy = conf.getTrimmed(ASSUMED_ROLE_POLICY, "");
LOG.debug("{}", this);
- STSAssumeRoleSessionCredentialsProvider.Builder builder
- = new STSAssumeRoleSessionCredentialsProvider.Builder(arn, sessionName);
- builder.withRoleSessionDurationSeconds((int) duration);
+
+ AssumeRoleRequest.Builder requestBuilder =
+ AssumeRoleRequest.builder().roleArn(arn).roleSessionName(sessionName)
+ .durationSeconds((int) duration);
+
if (StringUtils.isNotEmpty(policy)) {
LOG.debug("Scope down policy {}", policy);
- builder.withScopeDownPolicy(policy);
+ requestBuilder.policy(policy);
}
+
String endpoint = conf.getTrimmed(ASSUMED_ROLE_STS_ENDPOINT, "");
String region = conf.getTrimmed(ASSUMED_ROLE_STS_ENDPOINT_REGION,
ASSUMED_ROLE_STS_ENDPOINT_REGION_DEFAULT);
- AWSSecurityTokenServiceClientBuilder stsbuilder =
+ stsClient =
STSClientFactory.builder(
conf,
fsUri != null ? fsUri.getHost() : "",
credentialsToSTS,
endpoint,
- region);
- // the STS client is not tracked for a shutdown in close(), because it
- // (currently) throws an UnsupportedOperationException in shutdown().
- builder.withStsClient(stsbuilder.build());
+ region).build();
//now build the provider
- stsProvider = builder.build();
+ stsProvider = StsAssumeRoleCredentialsProvider.builder()
+ .refreshRequest(requestBuilder.build())
+ .stsClient(stsClient).build();
// to handle STS throttling by the AWS account, we
// need to retry
@@ -155,21 +159,21 @@ public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf)
// and force in a fail-fast check just to keep the stack traces less
// convoluted
- getCredentials();
+ resolveCredentials();
}
/**
* Get credentials.
* @return the credentials
- * @throws AWSSecurityTokenServiceException if none could be obtained.
+ * @throws StsException if none could be obtained.
*/
@Override
@Retries.RetryRaw
- public AWSCredentials getCredentials() {
+ public AwsCredentials resolveCredentials() {
try {
- return invoker.retryUntranslated("getCredentials",
+ return invoker.retryUntranslated("resolveCredentials",
true,
- stsProvider::getCredentials);
+ stsProvider::resolveCredentials);
} catch (IOException e) {
// this is in the signature of retryUntranslated;
// its hard to see how this could be raised, but for
@@ -178,35 +182,28 @@ public AWSCredentials getCredentials() {
throw new CredentialInitializationException(
"getCredentials failed: " + e,
e);
- } catch (AWSSecurityTokenServiceException e) {
- LOG.error("Failed to get credentials for role {}",
+ } catch (SdkClientException e) {
+ LOG.error("Failed to resolve credentials for role {}",
arn, e);
throw e;
}
}
- @Override
- public void refresh() {
- stsProvider.refresh();
- }
-
/**
* Propagate the close() call to the inner stsProvider.
*/
@Override
public void close() {
- S3AUtils.closeAutocloseables(LOG, stsProvider, credentialsToSTS);
+ S3AUtils.closeAutocloseables(LOG, stsProvider, credentialsToSTS, stsClient);
}
@Override
public String toString() {
- final StringBuilder sb = new StringBuilder(
- "AssumedRoleCredentialProvider{");
- sb.append("role='").append(arn).append('\'');
- sb.append(", session'").append(sessionName).append('\'');
- sb.append(", duration=").append(duration);
- sb.append('}');
- return sb.toString();
+ String sb = "AssumedRoleCredentialProvider{" + "role='" + arn + '\''
+ + ", session'" + sessionName + '\''
+ + ", duration=" + duration
+ + '}';
+ return sb;
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java
new file mode 100644
index 0000000000000..b106777dd29cc
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.auth;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
+import org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.S3AUtils;
+import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider;
+import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider;
+import org.apache.hadoop.fs.s3a.adapter.AwsV1BindingSupport;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+import org.apache.hadoop.fs.s3native.S3xLoginHelper;
+import org.apache.hadoop.fs.store.LogExactlyOnce;
+
+import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
+import static org.apache.hadoop.fs.s3a.adapter.AwsV1BindingSupport.isAwsV1SdkAvailable;
+
+/**
+ * This class provides methods to create a {@link AWSCredentialProviderList}
+ * list of AWS credential providers.
+ */
+public final class CredentialProviderListFactory {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CredentialProviderListFactory.class);
+
+ /**
+ * A v1 entry has been remapped. warn once about this and then shut up.
+ */
+ private static final LogExactlyOnce LOG_REMAPPED_ENTRY = new LogExactlyOnce(LOG);
+
+ /**
+ * Error message when the AWS provider list built up contains a forbidden
+ * entry.
+ */
+ @VisibleForTesting
+ public static final String E_FORBIDDEN_AWS_PROVIDER
+ = "AWS provider class cannot be used";
+
+ /**
+ * The standard AWS provider list for AWS connections.
+ */
+ public static final List>
+ STANDARD_AWS_PROVIDERS = Collections.unmodifiableList(
+ Arrays.asList(
+ EnvironmentVariableCredentialsProvider.class,
+ IAMInstanceCredentialsProvider.class,
+ SimpleAWSCredentialsProvider.class,
+ TemporaryAWSCredentialsProvider.class));
+
+ /** V1 credential provider: {@value}. */
+ public static final String ANONYMOUS_CREDENTIALS_V1 =
+ "com.amazonaws.auth.AnonymousAWSCredentials";
+
+ /** V1 credential provider: {@value}. */
+ public static final String EC2_CONTAINER_CREDENTIALS_V1 =
+ "com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper";
+
+ /** V1 credential provider: {@value}. */
+ public static final String EC2_IAM_CREDENTIALS_V1 =
+ "com.amazonaws.auth.InstanceProfileCredentialsProvider";
+
+ /** V2 EC2 instance/container credential provider. */
+ public static final String EC2_IAM_CREDENTIALS_V2 =
+ IAMInstanceCredentialsProvider.class.getName();
+
+ /** V1 env var credential provider: {@value}. */
+ public static final String ENVIRONMENT_CREDENTIALS_V1 =
+ "com.amazonaws.auth.EnvironmentVariableCredentialsProvider";
+
+ /** V2 environment variables credential provider. */
+ public static final String ENVIRONMENT_CREDENTIALS_V2 =
+ EnvironmentVariableCredentialsProvider.class.getName();
+
+ /** V1 profile credential provider: {@value}. */
+ public static final String PROFILE_CREDENTIALS_V1 =
+ "com.amazonaws.auth.profile.ProfileCredentialsProvider";
+
+ /** V2 environment variables credential provider. */
+ public static final String PROFILE_CREDENTIALS_V2 =
+ ProfileCredentialsProvider.class.getName();
+
+ /**
+ * Private map of v1 to v2 credential provider name mapping.
+ */
+ private static final Map V1_V2_CREDENTIAL_PROVIDER_MAP =
+ initCredentialProvidersMap();
+
+ private CredentialProviderListFactory() {
+ }
+
+ /**
+ * Create the AWS credentials from the providers, the URI and
+ * the key {@link Constants#AWS_CREDENTIALS_PROVIDER} in the configuration.
+ * @param binding Binding URI -may be null
+ * @param conf filesystem configuration
+ * @return a credentials provider list
+ * @throws IOException Problems loading the providers (including reading
+ * secrets from credential files).
+ */
+ public static AWSCredentialProviderList createAWSCredentialProviderList(
+ @Nullable URI binding,
+ Configuration conf) throws IOException {
+ // this will reject any user:secret entries in the URI
+ S3xLoginHelper.rejectSecretsInURIs(binding);
+ AWSCredentialProviderList credentials =
+ buildAWSProviderList(binding,
+ conf,
+ AWS_CREDENTIALS_PROVIDER,
+ STANDARD_AWS_PROVIDERS,
+ new HashSet<>());
+ // make sure the logging message strips out any auth details
+ LOG.debug("For URI {}, using credentials {}",
+ binding, credentials);
+ return credentials;
+ }
+
+ /**
+ * Load list of AWS credential provider/credential provider factory classes.
+ * @param conf configuration
+ * @param key key
+ * @param defaultValue list of default values
+ * @return the list of classes, empty if the default list is empty and
+ * there was no match for the key in the configuration.
+ * @throws IOException on a failure to load the list.
+ */
+ private static Collection loadAWSProviderClasses(Configuration conf,
+ String key,
+ Class... defaultValue) throws IOException {
+ final Collection classnames = conf.getTrimmedStringCollection(key);
+ if (classnames.isEmpty()) {
+ // empty list; return the defaults
+ return Arrays.stream(defaultValue).map(c -> c.getName()).collect(Collectors.toList());
+ } else {
+ return classnames;
+ }
+ }
+
+ /**
+ * Maps V1 credential providers to either their equivalent SDK V2 class or hadoop provider.
+ */
+ private static Map initCredentialProvidersMap() {
+ Map v1v2CredentialProviderMap = new HashMap<>();
+
+ v1v2CredentialProviderMap.put(ANONYMOUS_CREDENTIALS_V1,
+ AnonymousAWSCredentialsProvider.NAME);
+ v1v2CredentialProviderMap.put(EC2_CONTAINER_CREDENTIALS_V1,
+ EC2_IAM_CREDENTIALS_V2);
+ v1v2CredentialProviderMap.put(EC2_IAM_CREDENTIALS_V1,
+ EC2_IAM_CREDENTIALS_V2);
+ v1v2CredentialProviderMap.put(ENVIRONMENT_CREDENTIALS_V1,
+ ENVIRONMENT_CREDENTIALS_V2);
+ v1v2CredentialProviderMap.put(PROFILE_CREDENTIALS_V1,
+ PROFILE_CREDENTIALS_V2);
+
+ return v1v2CredentialProviderMap;
+ }
+
+ /**
+ * Load list of AWS credential provider/credential provider factory classes;
+ * support a forbidden list to prevent loops, mandate full secrets, etc.
+ * @param binding Binding URI -may be null
+ * @param conf configuration
+ * @param key configuration key to use
+ * @param forbidden a possibly empty set of forbidden classes.
+ * @param defaultValues list of default providers.
+ * @return the list of classes, possibly empty
+ * @throws IOException on a failure to load the list.
+ */
+ public static AWSCredentialProviderList buildAWSProviderList(
+ @Nullable final URI binding,
+ final Configuration conf,
+ final String key,
+ final List> defaultValues,
+ final Set> forbidden) throws IOException {
+
+ // build up the base provider
+ Collection awsClasses = loadAWSProviderClasses(conf,
+ key,
+ defaultValues.toArray(new Class[defaultValues.size()]));
+
+ Map v1v2CredentialProviderMap = V1_V2_CREDENTIAL_PROVIDER_MAP;
+ final Set forbiddenClassnames =
+ forbidden.stream().map(c -> c.getName()).collect(Collectors.toSet());
+
+
+ // iterate through, checking for forbidden values and then instantiating
+ // each provider
+ AWSCredentialProviderList providers = new AWSCredentialProviderList();
+ for (String className : awsClasses) {
+ if (v1v2CredentialProviderMap.containsKey(className)) {
+ // mapping
+
+ final String mapped = v1v2CredentialProviderMap.get(className);
+ LOG_REMAPPED_ENTRY.warn("Credentials option {} contains AWS v1 SDK entry {}; mapping to {}",
+ key, className, mapped);
+ className = mapped;
+ }
+ // now scan the forbidden list. doing this after any mappings ensures the v1 names
+ // are also blocked
+ if (forbiddenClassnames.contains(className)) {
+ throw new InstantiationIOException(InstantiationIOException.Kind.Forbidden,
+ binding, className, key, E_FORBIDDEN_AWS_PROVIDER, null);
+ }
+
+ AwsCredentialsProvider provider;
+ try {
+ provider = createAWSV2CredentialProvider(conf, className, binding, key);
+ } catch (InstantiationIOException e) {
+ // failed to create a v2; try to see if it is a v1
+ if (e.getKind() == InstantiationIOException.Kind.IsNotImplementation) {
+ if (isAwsV1SdkAvailable()) {
+ // try to create v1
+ LOG.debug("Failed to create {} as v2 credentials, trying to instantiate as v1",
+ className);
+ try {
+ provider =
+ AwsV1BindingSupport.createAWSV1CredentialProvider(conf, className, binding, key);
+ LOG_REMAPPED_ENTRY.warn("Credentials option {} contains AWS v1 SDK entry {}",
+ key, className);
+ } catch (InstantiationIOException ex) {
+ // if it is something other than non-implementation, throw.
+ // that way, non-impl messages are about v2 not v1 in the error
+ if (ex.getKind() != InstantiationIOException.Kind.IsNotImplementation) {
+ throw ex;
+ } else {
+ throw e;
+ }
+ }
+ } else {
+ LOG.warn("Failed to instantiate {} as AWS v2 SDK credential provider;"
+ + " AWS V1 SDK is not on the classpth so unable to attempt to"
+ + " instantiate as a v1 provider", className, e);
+ throw e;
+ }
+ } else {
+ // any other problem
+ throw e;
+
+ }
+ LOG.debug("From provider class {} created Aws provider {}", className, provider);
+ }
+ providers.add(provider);
+ }
+ return providers;
+ }
+
+ /**
+ * Create an AWS v2 credential provider from its class by using reflection.
+ * @param conf configuration
+ * @param className credential class name
+ * @param uri URI of the FS
+ * @param key configuration key to use
+ * @return the instantiated class
+ * @throws IOException on any instantiation failure.
+ * @see S3AUtils#getInstanceFromReflection
+ */
+ private static AwsCredentialsProvider createAWSV2CredentialProvider(Configuration conf,
+ String className,
+ @Nullable URI uri, final String key) throws IOException {
+ LOG.debug("Credential provider class is {}", className);
+ return S3AUtils.getInstanceFromReflection(className, conf, uri, AwsCredentialsProvider.class,
+ "create", key);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
index ca9c518d30048..080b79e7f20d5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
@@ -21,39 +21,69 @@
import java.io.Closeable;
import java.io.IOException;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.HttpCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
+import software.amazon.awssdk.core.exception.SdkClientException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractIOException;
+
/**
* This is an IAM credential provider which wraps
- * an {@code EC2ContainerCredentialsProviderWrapper}
+ * an {@code ContainerCredentialsProvider}
* to provide credentials when the S3A connector is instantiated on AWS EC2
* or the AWS container services.
*
- * When it fails to authenticate, it raises a
- * {@link NoAwsCredentialsException} which can be recognized by retry handlers
+ * The provider is initialized with async credential refresh enabled to be less
+ * brittle against transient network issues.
+ *
+ * If the ContainerCredentialsProvider fails to authenticate, then an instance of
+ * {@link InstanceProfileCredentialsProvider} is created and attemped to
+ * be used instead, again with async credential refresh enabled.
+ *
+ * If both credential providers fail, a {@link NoAwsCredentialsException}
+ * is thrown, which can be recognized by retry handlers
* as a non-recoverable failure.
*
* It is implicitly public; marked evolving as we can change its semantics.
- *
- * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
- * as part of upgrading S3A to SDK V2. See HADOOP-18073.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-@Deprecated
public class IAMInstanceCredentialsProvider
- implements AWSCredentialsProvider, Closeable {
+ implements AwsCredentialsProvider, Closeable {
- private final AWSCredentialsProvider provider =
- new EC2ContainerCredentialsProviderWrapper();
+ private static final Logger LOG =
+ LoggerFactory.getLogger(IAMInstanceCredentialsProvider.class);
+
+ /**
+ * The credentials provider.
+ * Initially a container credentials provider, but if that fails
+ * fall back to the instance profile provider.
+ */
+ private HttpCredentialsProvider iamCredentialsProvider;
+ /**
+ * Is the container credentials provider in use?
+ */
+ private boolean isContainerCredentialsProvider;
+
+ /**
+ * Constructor.
+ * Build credentials provider with async refresh,
+ * mark {@link #isContainerCredentialsProvider} as true.
+ */
public IAMInstanceCredentialsProvider() {
+ isContainerCredentialsProvider = true;
+ iamCredentialsProvider = ContainerCredentialsProvider.builder()
+ .asyncCredentialUpdateEnabled(true)
+ .build();
}
/**
@@ -63,23 +93,76 @@ public IAMInstanceCredentialsProvider() {
* @throws NoAwsCredentialsException on auth failure to indicate non-recoverable.
*/
@Override
- public AWSCredentials getCredentials() {
+ public AwsCredentials resolveCredentials() {
try {
- return provider.getCredentials();
- } catch (AmazonClientException e) {
+ return getCredentials();
+ } catch (SdkClientException e) {
+
+ // if the exception contains an IOE, extract it
+ // so its type is the immediate cause of this new exception.
+ Throwable t = e;
+ final IOException ioe = maybeExtractIOException("IAM endpoint", e);
+ if (ioe != null) {
+ t = ioe;
+ }
throw new NoAwsCredentialsException("IAMInstanceCredentialsProvider",
- e.getMessage(),
- e);
+ e.getMessage(), t);
}
}
+ /**
+ * First try {@link ContainerCredentialsProvider}, which will throw an exception if credentials
+ * cannot be retrieved from the container. Then resolve credentials
+ * using {@link InstanceProfileCredentialsProvider}.
+ *
+ * @return credentials
+ */
+ private synchronized AwsCredentials getCredentials() {
+ try {
+ return iamCredentialsProvider.resolveCredentials();
+ } catch (SdkClientException e) {
+ LOG.debug("Failed to get credentials from container provider,", e);
+ if (isContainerCredentialsProvider) {
+ // create instance profile provider
+ LOG.debug("Switching to instance provider", e);
+
+ // close it to shut down any thread
+ iamCredentialsProvider.close();
+ isContainerCredentialsProvider = false;
+ iamCredentialsProvider = InstanceProfileCredentialsProvider.builder()
+ .asyncCredentialUpdateEnabled(true)
+ .build();
+ return iamCredentialsProvider.resolveCredentials();
+ } else {
+ // already using instance profile provider, so fail
+ throw e;
+ }
+
+ }
+ }
+
+ /**
+ * Is this a container credentials provider?
+ * @return true if the container credentials provider is in use;
+ * false for InstanceProfileCredentialsProvider
+ */
+ public boolean isContainerCredentialsProvider() {
+ return isContainerCredentialsProvider;
+ }
+
@Override
- public void refresh() {
- provider.refresh();
+ public synchronized void close() throws IOException {
+ // this be true but just for safety...
+ if (iamCredentialsProvider != null) {
+ iamCredentialsProvider.close();
+ }
}
@Override
- public void close() throws IOException {
- // no-op.
+ public String toString() {
+ return "IAMInstanceCredentialsProvider{" +
+ "credentialsProvider=" + iamCredentialsProvider +
+ ", isContainerCredentialsProvider=" + isContainerCredentialsProvider +
+ '}';
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java
index 72d29df3fe0e7..46d35e156d069 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java
@@ -24,19 +24,18 @@
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.SdkClientException;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSSessionCredentials;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.BasicSessionCredentials;
-import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
-import com.amazonaws.services.securitytoken.model.Credentials;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.services.sts.StsClient;
+import software.amazon.awssdk.services.sts.model.Credentials;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.Invoker;
import org.apache.hadoop.fs.s3a.Retries;
@@ -77,10 +76,10 @@ private MarshalledCredentialBinding() {
public static MarshalledCredentials fromSTSCredentials(
final Credentials credentials) {
MarshalledCredentials marshalled = new MarshalledCredentials(
- credentials.getAccessKeyId(),
- credentials.getSecretAccessKey(),
- credentials.getSessionToken());
- Date date = credentials.getExpiration();
+ credentials.accessKeyId(),
+ credentials.secretAccessKey(),
+ credentials.sessionToken());
+ Date date = Date.from(credentials.expiration());
marshalled.setExpiration(date != null ? date.getTime() : 0);
return marshalled;
}
@@ -91,11 +90,11 @@ public static MarshalledCredentials fromSTSCredentials(
* @return a set of marshalled credentials.
*/
public static MarshalledCredentials fromAWSCredentials(
- final AWSSessionCredentials credentials) {
+ final AwsSessionCredentials credentials) {
return new MarshalledCredentials(
- credentials.getAWSAccessKeyId(),
- credentials.getAWSSecretKey(),
- credentials.getSessionToken());
+ credentials.accessKeyId(),
+ credentials.secretAccessKey(),
+ credentials.sessionToken());
}
/**
@@ -156,7 +155,7 @@ public static MarshalledCredentials fromFileSystem(
* @throws NoAuthWithAWSException validation failure
* @throws NoAwsCredentialsException the credentials are actually empty.
*/
- public static AWSCredentials toAWSCredentials(
+ public static AwsCredentials toAWSCredentials(
final MarshalledCredentials marshalled,
final MarshalledCredentials.CredentialTypeRequired typeRequired,
final String component)
@@ -173,46 +172,49 @@ public static AWSCredentials toAWSCredentials(
final String secretKey = marshalled.getSecretKey();
if (marshalled.hasSessionToken()) {
// a session token was supplied, so return session credentials
- return new BasicSessionCredentials(accessKey, secretKey,
+ return AwsSessionCredentials.create(accessKey, secretKey,
marshalled.getSessionToken());
} else {
// these are full credentials
- return new BasicAWSCredentials(accessKey, secretKey);
+ return AwsBasicCredentials.create(accessKey, secretKey);
}
}
/**
* Request a set of credentials from an STS endpoint.
* @param parentCredentials the parent credentials needed to talk to STS
- * @param awsConf AWS client configuration
+ * @param configuration AWS client configuration
* @param stsEndpoint an endpoint, use "" for none
* @param stsRegion region; use if the endpoint isn't the AWS default.
* @param duration duration of the credentials in seconds. Minimum value: 900.
* @param invoker invoker to use for retrying the call.
+ * @param bucket bucket name.
* @return the credentials
* @throws IOException on a failure of the request
*/
@Retries.RetryTranslated
public static MarshalledCredentials requestSessionCredentials(
- final AWSCredentialsProvider parentCredentials,
- final ClientConfiguration awsConf,
+ final AwsCredentialsProvider parentCredentials,
+ final Configuration configuration,
final String stsEndpoint,
final String stsRegion,
final int duration,
- final Invoker invoker) throws IOException {
+ final Invoker invoker,
+ final String bucket) throws IOException {
try {
- final AWSSecurityTokenService tokenService =
+ final StsClient tokenService =
STSClientFactory.builder(parentCredentials,
- awsConf,
+ configuration,
stsEndpoint.isEmpty() ? null : stsEndpoint,
- stsRegion)
+ stsRegion,
+ bucket)
.build();
try (STSClientFactory.STSClient stsClient = STSClientFactory.createClientConnection(
tokenService, invoker)) {
return fromSTSCredentials(stsClient.requestSessionCredentials(duration,
TimeUnit.SECONDS));
}
- } catch (SdkClientException e) {
+ } catch (SdkException e) {
if (stsRegion.isEmpty()) {
LOG.error("Region must be provided when requesting session credentials.",
e);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java
index cde4cc94b7852..7dffa157c046c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java
@@ -21,7 +21,7 @@
import java.io.IOException;
import java.net.URI;
-import com.amazonaws.auth.AWSCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -40,7 +40,6 @@
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-@SuppressWarnings("deprecation")
public class MarshalledCredentialProvider extends
AbstractSessionCredentialsProvider {
@@ -85,7 +84,7 @@ public MarshalledCredentialProvider(
* @throws IOException on a failure
*/
@Override
- protected AWSCredentials createCredentials(final Configuration config)
+ protected AwsCredentials createCredentials(final Configuration config)
throws IOException {
return toAWSCredentials(credentials, typeRequired, component);
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/NoAuthWithAWSException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/NoAuthWithAWSException.java
index 7ec13b092c9bc..8f92153b2e1d6 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/NoAuthWithAWSException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/NoAuthWithAWSException.java
@@ -21,7 +21,7 @@
import org.apache.hadoop.fs.s3a.CredentialInitializationException;
/**
- * A specific subclass of {@code AmazonClientException} which is
+ * A specific subclass of {@code SdkException} which is
* used in the S3A retry policy to fail fast when there is any
* authentication problem.
*/
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java
index aa88a320ff2a8..d46c9f6f92152 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java
@@ -80,7 +80,7 @@ private RolePolicies() {
* Statement to allow KMS R/W access access, so full use of
* SSE-KMS.
*/
- public static final Statement STATEMENT_ALLOW_SSE_KMS_RW =
+ public static final Statement STATEMENT_ALLOW_KMS_RW =
statement(true, KMS_ALL_KEYS, KMS_ALL_OPERATIONS);
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java
index 4779f3c1cb0cd..6f1c7ef801dc3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java
@@ -20,31 +20,38 @@
import java.io.Closeable;
import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.concurrent.TimeUnit;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.client.builder.AwsClientBuilder;
-import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
-import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
-import com.amazonaws.services.securitytoken.model.AssumeRoleRequest;
-import com.amazonaws.services.securitytoken.model.Credentials;
-import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
+import software.amazon.awssdk.core.retry.RetryPolicy;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.apache.ProxyConfiguration;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.sts.StsClient;
+import software.amazon.awssdk.services.sts.StsClientBuilder;
+import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
+import software.amazon.awssdk.services.sts.model.Credentials;
+import software.amazon.awssdk.services.sts.model.GetSessionTokenRequest;
+import software.amazon.awssdk.thirdparty.org.apache.http.client.utils.URIBuilder;
+import org.apache.hadoop.fs.s3a.impl.AWSClientConfig;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.Invoker;
import org.apache.hadoop.fs.s3a.Retries;
-import org.apache.hadoop.fs.s3a.S3AUtils;
import static org.apache.commons.lang3.StringUtils.isEmpty;
import static org.apache.commons.lang3.StringUtils.isNotEmpty;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_STS;
import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*;
/**
@@ -71,17 +78,15 @@ public class STSClientFactory {
* @return the builder to call {@code build()}
* @throws IOException problem reading proxy secrets
*/
- public static AWSSecurityTokenServiceClientBuilder builder(
+ public static StsClientBuilder builder(
final Configuration conf,
final String bucket,
- final AWSCredentialsProvider credentials) throws IOException {
- final ClientConfiguration awsConf = S3AUtils.createAwsConf(conf, bucket,
- Constants.AWS_SERVICE_IDENTIFIER_STS);
+ final AwsCredentialsProvider credentials) throws IOException {
String endpoint = conf.getTrimmed(DELEGATION_TOKEN_ENDPOINT,
DEFAULT_DELEGATION_TOKEN_ENDPOINT);
String region = conf.getTrimmed(DELEGATION_TOKEN_REGION,
DEFAULT_DELEGATION_TOKEN_REGION);
- return builder(credentials, awsConf, endpoint, region);
+ return builder(credentials, conf, endpoint, region, bucket);
}
/**
@@ -96,64 +101,89 @@ public static AWSSecurityTokenServiceClientBuilder builder(
* @return the builder to call {@code build()}
* @throws IOException problem reading proxy secrets
*/
- public static AWSSecurityTokenServiceClientBuilder builder(
+ public static StsClientBuilder builder(
final Configuration conf,
final String bucket,
- final AWSCredentialsProvider credentials,
+ final AwsCredentialsProvider credentials,
final String stsEndpoint,
final String stsRegion) throws IOException {
- final ClientConfiguration awsConf = S3AUtils.createAwsConf(conf, bucket,
- Constants.AWS_SERVICE_IDENTIFIER_STS);
- return builder(credentials, awsConf, stsEndpoint, stsRegion);
+ return builder(credentials, conf, stsEndpoint, stsRegion, bucket);
}
/**
* Create the builder ready for any final configuration options.
* Picks up connection settings from the Hadoop configuration, including
* proxy secrets.
- * @param awsConf AWS configuration.
+ * @param conf AWS configuration.
* @param credentials AWS credential chain to use
* @param stsEndpoint optional endpoint "https://sns.us-west-1.amazonaws.com"
* @param stsRegion the region, e.g "us-west-1". Must be set if endpoint is.
+ * @param bucket bucket name
* @return the builder to call {@code build()}
+ * @throws IOException problem reading proxy secrets
*/
- public static AWSSecurityTokenServiceClientBuilder builder(
- final AWSCredentialsProvider credentials,
- final ClientConfiguration awsConf,
- final String stsEndpoint,
- final String stsRegion) {
- final AWSSecurityTokenServiceClientBuilder builder
- = AWSSecurityTokenServiceClientBuilder.standard();
+ public static StsClientBuilder builder(final AwsCredentialsProvider credentials,
+ final Configuration conf, final String stsEndpoint, final String stsRegion,
+ final String bucket) throws IOException {
+ final StsClientBuilder stsClientBuilder = StsClient.builder();
+
Preconditions.checkArgument(credentials != null, "No credentials");
- builder.withClientConfiguration(awsConf);
- builder.withCredentials(credentials);
+
+ final ClientOverrideConfiguration.Builder clientOverrideConfigBuilder =
+ AWSClientConfig.createClientConfigBuilder(conf, AWS_SERVICE_IDENTIFIER_STS);
+
+ final ApacheHttpClient.Builder httpClientBuilder =
+ AWSClientConfig.createHttpClientBuilder(conf);
+
+ final RetryPolicy.Builder retryPolicyBuilder = AWSClientConfig.createRetryPolicyBuilder(conf);
+
+ final ProxyConfiguration proxyConfig = AWSClientConfig.createProxyConfiguration(conf, bucket);
+
+ clientOverrideConfigBuilder.retryPolicy(retryPolicyBuilder.build());
+ httpClientBuilder.proxyConfiguration(proxyConfig);
+
+ stsClientBuilder.httpClientBuilder(httpClientBuilder)
+ .overrideConfiguration(clientOverrideConfigBuilder.build())
+ .credentialsProvider(credentials);
+
boolean destIsStandardEndpoint = STS_STANDARD.equals(stsEndpoint);
if (isNotEmpty(stsEndpoint) && !destIsStandardEndpoint) {
- Preconditions.checkArgument(
- isNotEmpty(stsRegion),
- "STS endpoint is set to %s but no signing region was provided",
- stsEndpoint);
+ Preconditions.checkArgument(isNotEmpty(stsRegion),
+ "STS endpoint is set to %s but no signing region was provided", stsEndpoint);
LOG.debug("STS Endpoint={}; region='{}'", stsEndpoint, stsRegion);
- builder.withEndpointConfiguration(
- new AwsClientBuilder.EndpointConfiguration(stsEndpoint, stsRegion));
+ stsClientBuilder.endpointOverride(getSTSEndpoint(stsEndpoint)).region(Region.of(stsRegion));
} else {
Preconditions.checkArgument(isEmpty(stsRegion),
- "STS signing region set set to %s but no STS endpoint specified",
- stsRegion);
+ "STS signing region set set to %s but no STS endpoint specified", stsRegion);
+ }
+ return stsClientBuilder;
+ }
+
+ /**
+ * Given a endpoint string, create the endpoint URI.
+ *
+ * @param endpoint possibly null endpoint.
+ * @return an endpoint uri
+ */
+ private static URI getSTSEndpoint(String endpoint) {
+ try {
+ return new URIBuilder().setScheme("https").setHost(endpoint).build();
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(e);
}
- return builder;
}
+
/**
* Create an STS Client instance.
- * @param tokenService STS instance
+ * @param stsClient STS instance
* @param invoker invoker to use
* @return an STS client bonded to that interface.
*/
public static STSClient createClientConnection(
- final AWSSecurityTokenService tokenService,
+ final StsClient stsClient,
final Invoker invoker) {
- return new STSClient(tokenService, invoker);
+ return new STSClient(stsClient, invoker);
}
/**
@@ -161,21 +191,19 @@ public static STSClient createClientConnection(
*/
public static final class STSClient implements Closeable {
- private final AWSSecurityTokenService tokenService;
+ private final StsClient stsClient;
private final Invoker invoker;
- private STSClient(final AWSSecurityTokenService tokenService,
+ private STSClient(final StsClient stsClient,
final Invoker invoker) {
- this.tokenService = tokenService;
+ this.stsClient = stsClient;
this.invoker = invoker;
}
@Override
public void close() throws IOException {
- // Since we are not using AbstractAWSSecurityTokenService, we
- // don't need to worry about catching UnsupportedOperationException.
- tokenService.shutdown();
+ stsClient.close();
}
/**
@@ -192,13 +220,13 @@ public Credentials requestSessionCredentials(
final TimeUnit timeUnit) throws IOException {
int durationSeconds = (int) timeUnit.toSeconds(duration);
LOG.debug("Requesting session token of duration {}", duration);
- final GetSessionTokenRequest request = new GetSessionTokenRequest();
- request.setDurationSeconds(durationSeconds);
+ final GetSessionTokenRequest request =
+ GetSessionTokenRequest.builder().durationSeconds(durationSeconds).build();
return invoker.retry("request session credentials", "",
true,
() ->{
LOG.info("Requesting Amazon STS Session credentials");
- return tokenService.getSessionToken(request).getCredentials();
+ return stsClient.getSessionToken(request).credentials();
});
}
@@ -222,15 +250,14 @@ public Credentials requestRole(
final TimeUnit timeUnit) throws IOException {
LOG.debug("Requesting role {} with duration {}; policy = {}",
roleARN, duration, policy);
- AssumeRoleRequest request = new AssumeRoleRequest();
- request.setDurationSeconds((int) timeUnit.toSeconds(duration));
- request.setRoleArn(roleARN);
- request.setRoleSessionName(sessionName);
+ AssumeRoleRequest.Builder requestBuilder =
+ AssumeRoleRequest.builder().durationSeconds((int) timeUnit.toSeconds(duration))
+ .roleArn(roleARN).roleSessionName(sessionName);
if (isNotEmpty(policy)) {
- request.setPolicy(policy);
+ requestBuilder.policy(policy);
}
return invoker.retry("request role credentials", "", true,
- () -> tokenService.assumeRole(request).getCredentials());
+ () -> stsClient.assumeRole(requestBuilder.build()).credentials());
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
new file mode 100644
index 0000000000000..21c390c07940b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.auth;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.signer.Aws4Signer;
+import software.amazon.awssdk.auth.signer.Aws4UnsignedPayloadSigner;
+import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
+import software.amazon.awssdk.core.signer.NoOpSigner;
+import software.amazon.awssdk.core.signer.Signer;
+
+import org.apache.hadoop.fs.s3a.S3AUtils;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unavailable;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
+
+/**
+ * Signer factory used to register and create signers.
+ */
+public final class SignerFactory {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SignerFactory.class);
+ public static final String VERSION_FOUR_SIGNER = "AWS4SignerType";
+ public static final String VERSION_FOUR_UNSIGNED_PAYLOAD_SIGNER = "AWS4UnsignedPayloadSignerType";
+ public static final String NO_OP_SIGNER = "NoOpSignerType";
+ private static final String S3_V4_SIGNER = "AWSS3V4SignerType";
+
+ /** The v2 signer is no longer available: {@value}. */
+ public static final String S3_V2_SIGNER = "S3SignerType";
+
+ private static final Map> SIGNERS
+ = new ConcurrentHashMap<>();
+
+ static {
+ // Register the standard signer types.
+ SIGNERS.put(VERSION_FOUR_SIGNER, Aws4Signer.class);
+ SIGNERS.put(VERSION_FOUR_UNSIGNED_PAYLOAD_SIGNER, Aws4UnsignedPayloadSigner.class);
+ SIGNERS.put(NO_OP_SIGNER, NoOpSigner.class);
+ SIGNERS.put(S3_V4_SIGNER, AwsS3V4Signer.class);
+ }
+
+
+ private SignerFactory() {
+ }
+
+ /**
+ * Register an implementation class for the given signer type.
+ *
+ * @param signerType The name of the signer type to register.
+ * @param signerClass The class implementing the given signature protocol.
+ */
+ public static void registerSigner(
+ final String signerType,
+ final Class signerClass) {
+
+ checkArgument(signerType != null, "signerType cannot be null");
+ checkArgument(signerClass != null, "signerClass cannot be null");
+
+ SIGNERS.put(signerType, signerClass);
+ }
+
+ /**
+ * Check if the signer has already been registered.
+ * @param signerType signer to get
+ * @return true if the signer is registered.
+ */
+ public static boolean isSignerRegistered(String signerType) {
+ return SIGNERS.containsKey(signerType);
+ }
+
+ /**
+ * Create an instance of the given signer.
+ *
+ * @param signerType The signer type.
+ * @param configKey Config key used to configure the signer.
+ * @return The new signer instance.
+ * @throws InstantiationIOException instantiation problems.
+ * @throws IOException on any other problem.
+ *
+ */
+ public static Signer createSigner(String signerType, String configKey) throws IOException {
+ if (S3_V2_SIGNER.equals(signerType)) {
+ throw unavailable(null, null, configKey, S3_V2_SIGNER + " is no longer supported");
+ }
+ if (!isSignerRegistered(signerType)) {
+ throw unavailable(null, null, configKey, "unknown signer type: " + signerType);
+ }
+ Class signerClass = SIGNERS.get(signerType);
+
+ String className = signerClass.getName();
+
+ LOG.debug("Signer class from {} and key {} is {}", signerType, configKey, className);
+
+ Signer signer =
+ S3AUtils.getInstanceFromReflection(className, null, null, Signer.class, "create",
+ configKey);
+
+ return signer;
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java
index e162428787cc4..f0a71007b4cf8 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java
@@ -22,14 +22,12 @@
import java.util.LinkedList;
import java.util.List;
-import com.amazonaws.auth.Signer;
-import com.amazonaws.auth.SignerFactory;
+import software.amazon.awssdk.core.signer.Signer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
-import org.apache.hadoop.fs.s3a.impl.V2Migration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
@@ -71,8 +69,6 @@ public void initCustomSigners() {
return;
}
- V2Migration.v1CustomSignerUsed();
-
for (String customSigner : customSigners) {
String[] parts = customSigner.split(":");
if (!(parts.length == 1 || parts.length == 2 || parts.length == 3)) {
@@ -113,17 +109,20 @@ public void initCustomSigners() {
}
}
- /*
- * Make sure the signer class is registered once with the AWS SDK
+ /**
+ * Make sure the signer class is registered once with the AWS SDK.
+ * @param signerName signer name
+ * @param signerClassName classname
+ * @param conf source configuration
+ * @throws RuntimeException if the class is not found
*/
private static void maybeRegisterSigner(String signerName,
String signerClassName, Configuration conf) {
- try {
- SignerFactory.getSignerByTypeAndService(signerName, null);
- } catch (IllegalArgumentException e) {
+
+ if (!SignerFactory.isSignerRegistered(signerName)) {
// Signer is not registered with the AWS SDK.
// Load the class and register the signer.
- Class clazz = null;
+ Class clazz;
try {
clazz = (Class) conf.getClassByName(signerClassName);
} catch (ClassNotFoundException cnfe) {
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java
index 6526f9a947815..8a55a970134f3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java
@@ -20,9 +20,6 @@
import java.util.Optional;
-import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
-import com.amazonaws.services.s3.model.SSECustomerKey;
-
import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
/**
@@ -35,37 +32,31 @@ public final class EncryptionSecretOperations {
private EncryptionSecretOperations() {
}
- /**
- * Create SSE-C client side key encryption options on demand.
- * @return an optional key to attach to a request.
+ /***
+ * Gets the SSE-C client side key if present.
+ *
* @param secrets source of the encryption secrets.
+ * @return an optional key to attach to a request.
*/
- public static Optional createSSECustomerKey(
- final EncryptionSecrets secrets) {
- if (secrets.hasEncryptionKey() &&
- secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_C) {
- return Optional.of(new SSECustomerKey(secrets.getEncryptionKey()));
+ public static Optional getSSECustomerKey(final EncryptionSecrets secrets) {
+ if (secrets.hasEncryptionKey() && secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_C) {
+ return Optional.of(secrets.getEncryptionKey());
} else {
return Optional.empty();
}
}
/**
- * Create SSE-KMS options for a request, iff the encryption is SSE-KMS.
- * @return an optional SSE-KMS param to attach to a request.
+ * Gets the SSE-KMS key if present, else let S3 use AWS managed key.
+ *
* @param secrets source of the encryption secrets.
+ * @return an optional key to attach to a request.
*/
- public static Optional createSSEAwsKeyManagementParams(
- final EncryptionSecrets secrets) {
-
- //Use specified key, otherwise default to default master aws/s3 key by AWS
- if (secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS) {
- if (secrets.hasEncryptionKey()) {
- return Optional.of(new SSEAwsKeyManagementParams(
- secrets.getEncryptionKey()));
- } else {
- return Optional.of(new SSEAwsKeyManagementParams());
- }
+ public static Optional getSSEAwsKMSKey(final EncryptionSecrets secrets) {
+ if ((secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS
+ || secrets.getEncryptionMethod() == S3AEncryptionMethods.DSSE_KMS)
+ && secrets.hasEncryptionKey()) {
+ return Optional.of(secrets.getEncryptionKey());
} else {
return Optional.empty();
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java
index 8a9ee30f64e5c..75d907aac81b0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java
@@ -23,7 +23,7 @@
import java.util.UUID;
import java.util.concurrent.TimeUnit;
-import com.amazonaws.services.securitytoken.model.Credentials;
+import software.amazon.awssdk.services.sts.model.Credentials;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.slf4j.Logger;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
index 2fe6be61e2583..f46c1f9405eb2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
@@ -120,11 +120,6 @@ public class S3ADelegationTokens extends AbstractDTService {
*/
private AbstractDelegationTokenBinding tokenBinding;
- /**
- * List of cred providers; unset until {@link #bindToDelegationToken(Token)}.
- */
- //private Optional credentialProviders = Optional.empty();
-
/**
* delegation binding information; unset until {@link #bindToDelegationToken(Token)}.
*/
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java
index 87fbdf53ccb11..2b21ea173ef01 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java
@@ -26,21 +26,18 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSSessionCredentials;
-import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+import software.amazon.awssdk.services.sts.StsClient;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
-import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.Invoker;
import org.apache.hadoop.fs.s3a.Retries;
import org.apache.hadoop.fs.s3a.S3ARetryPolicy;
-import org.apache.hadoop.fs.s3a.S3AUtils;
import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialProvider;
import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials;
import org.apache.hadoop.fs.s3a.auth.RoleModel;
@@ -50,8 +47,8 @@
import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
import static org.apache.hadoop.fs.s3a.Invoker.once;
-import static org.apache.hadoop.fs.s3a.S3AUtils.STANDARD_AWS_PROVIDERS;
-import static org.apache.hadoop.fs.s3a.S3AUtils.buildAWSProviderList;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.STANDARD_AWS_PROVIDERS;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.buildAWSProviderList;
import static org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding.fromAWSCredentials;
import static org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding.fromSTSCredentials;
import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*;
@@ -105,7 +102,8 @@ public class SessionTokenBinding extends AbstractDelegationTokenBinding {
private boolean hasSessionCreds;
/**
- * The auth chain for the parent options.
+ * The parent authentication chain: that used to request
+ * session/role credentials when deployed unbonded.
*/
private AWSCredentialProviderList parentAuthChain;
@@ -164,12 +162,14 @@ protected void serviceStart() throws Exception {
DEFAULT_DELEGATION_TOKEN_REGION);
// create the provider set for session credentials.
- parentAuthChain = buildAWSProviderList(
+ final AWSCredentialProviderList chain = buildAWSProviderList(
getCanonicalUri(),
conf,
AWS_CREDENTIALS_PROVIDER,
STANDARD_AWS_PROVIDERS,
new HashSet<>());
+ LOG.debug("Setting parent authentication chain to {}", chain);
+ setParentAuthChain(chain);
}
@Override
@@ -192,7 +192,7 @@ protected void serviceStop() throws Exception {
public AWSCredentialProviderList deployUnbonded()
throws IOException {
requireServiceStarted();
- return parentAuthChain;
+ return getParentAuthChain();
}
/**
@@ -292,23 +292,22 @@ private synchronized Optional maybeInitSTS()
// chain.
// As no codepath (session propagation, STS creation) will work,
// throw this.
- final AWSCredentials parentCredentials = once("get credentials",
+ final AwsCredentials parentCredentials = once("get credentials",
"",
- () -> parentAuthChain.getCredentials());
- hasSessionCreds = parentCredentials instanceof AWSSessionCredentials;
+ () -> getParentAuthChain().resolveCredentials());
+ hasSessionCreds = parentCredentials instanceof AwsSessionCredentials;
if (!hasSessionCreds) {
LOG.debug("Creating STS client for {}", getDescription());
invoker = new Invoker(new S3ARetryPolicy(conf), LOG_EVENT);
- ClientConfiguration awsConf =
- S3AUtils.createAwsConf(conf, uri.getHost(),
- Constants.AWS_SERVICE_IDENTIFIER_STS);
- AWSSecurityTokenService tokenService =
- STSClientFactory.builder(parentAuthChain,
- awsConf,
+
+ StsClient tokenService =
+ STSClientFactory.builder(getParentAuthChain(),
+ conf,
endpoint,
- region)
+ region,
+ uri.getHost())
.build();
stsClient = Optional.of(
STSClientFactory.createClientConnection(tokenService, invoker));
@@ -374,11 +373,11 @@ public SessionTokenIdentifier createTokenIdentifier(
+ " -duration unknown", getCanonicalUri());
}
origin += " " + CREDENTIALS_CONVERTED_TO_DELEGATION_TOKEN;
- final AWSCredentials awsCredentials
- = parentAuthChain.getCredentials();
- if (awsCredentials instanceof AWSSessionCredentials) {
+ final AwsCredentials awsCredentials
+ = getParentAuthChain().resolveCredentials();
+ if (awsCredentials instanceof AwsSessionCredentials) {
marshalledCredentials = fromAWSCredentials(
- (AWSSessionCredentials) awsCredentials);
+ (AwsSessionCredentials) awsCredentials);
} else {
throw new DelegationTokenIOException(
"AWS Authentication chain is no longer supplying session secrets");
@@ -425,4 +424,16 @@ protected void setTokenIdentifier(Optional
tokenIdentifier) {
this.tokenIdentifier = tokenIdentifier;
}
+
+ /**
+ * The auth chain for the parent options.
+ * @return the parent authentication chain.
+ */
+ protected AWSCredentialProviderList getParentAuthChain() {
+ return parentAuthChain;
+ }
+
+ protected void setParentAuthChain(AWSCredentialProviderList parentAuthChain) {
+ this.parentAuthChain = parentAuthChain;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java
index e53c690431ee0..09664a6dbdf63 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java
@@ -27,10 +27,11 @@
import java.util.List;
import java.util.UUID;
-import com.amazonaws.services.s3.model.MultipartUpload;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.VisibleForTesting;
@@ -975,7 +976,7 @@ protected void abortPendingUploadsInCleanup(
.executeWith(commitContext.getOuterSubmitter())
.suppressExceptions(suppressExceptions)
.run(u -> commitContext.abortMultipartCommit(
- u.getKey(), u.getUploadId()));
+ u.key(), u.uploadId()));
} else {
LOG.info("No pending uploads were found");
}
@@ -1300,8 +1301,8 @@ protected void warnOnActiveUploads(final Path path) {
DateFormat df = DateFormat.getDateTimeInstance();
pending.forEach(u ->
LOG.info("[{}] {}",
- df.format(u.getInitiated()),
- u.getKey()));
+ df.format(Date.from(u.initiated())),
+ u.key()));
if (shouldAbortUploadsInCleanup()) {
LOG.warn("This committer will abort these uploads in job cleanup");
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PutTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PutTracker.java
index 10440f77e7277..6c3cf3942d527 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PutTracker.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PutTracker.java
@@ -21,7 +21,7 @@
import java.io.IOException;
import java.util.List;
-import com.amazonaws.services.s3.model.PartETag;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.statistics.IOStatistics;
@@ -76,7 +76,7 @@ public boolean outputImmediatelyVisible() {
* @throws IOException I/O problem or validation failure.
*/
public boolean aboutToComplete(String uploadId,
- List parts,
+ List parts,
long bytesWritten,
final IOStatistics iostatistics)
throws IOException {
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
index 77c3fed11fb24..e4541ba4da370 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
@@ -31,9 +31,10 @@
import java.util.List;
import java.util.Map;
-import com.amazonaws.services.s3.model.PartETag;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
import com.fasterxml.jackson.annotation.JsonProperty;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -215,13 +216,13 @@ public void touch(long millis) {
* @param parts ordered list of etags.
* @throws ValidationFailure if the data is invalid
*/
- public void bindCommitData(List parts) throws ValidationFailure {
+ public void bindCommitData(List parts) throws ValidationFailure {
etags = new ArrayList<>(parts.size());
int counter = 1;
- for (PartETag part : parts) {
- verify(part.getPartNumber() == counter,
- "Expected part number %s but got %s", counter, part.getPartNumber());
- etags.add(part.getETag());
+ for (CompletedPart part : parts) {
+ verify(part.partNumber() == counter,
+ "Expected part number %s but got %s", counter, part.partNumber());
+ etags.add(part.eTag());
counter++;
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java
index ef56d82978158..d1943fa47773f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java
@@ -21,6 +21,8 @@
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -32,13 +34,15 @@
import javax.annotation.Nullable;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+import software.amazon.awssdk.services.s3.model.UploadPartResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
@@ -155,9 +159,9 @@ public CommitOperations(S3AFileSystem fs,
* @param tagIds list of tags
* @return same list, now in numbered tuples
*/
- public static List toPartEtags(List tagIds) {
+ public static List toPartEtags(List tagIds) {
return IntStream.range(0, tagIds.size())
- .mapToObj(i -> new PartETag(i + 1, tagIds.get(i)))
+ .mapToObj(i -> CompletedPart.builder().partNumber(i + 1).eTag(tagIds.get(i)).build())
.collect(Collectors.toList());
}
@@ -566,26 +570,30 @@ public SinglePendingCommit uploadFileToPendingCommit(File localFile,
numParts, length));
}
- List parts = new ArrayList<>((int) numParts);
+ List parts = new ArrayList<>((int) numParts);
LOG.debug("File size is {}, number of parts to upload = {}",
length, numParts);
- for (int partNumber = 1; partNumber <= numParts; partNumber += 1) {
- progress.progress();
- long size = Math.min(length - offset, uploadPartSize);
- UploadPartRequest part;
- part = writeOperations.newUploadPartRequest(
- destKey,
- uploadId,
- partNumber,
- (int) size,
- null,
- localFile,
- offset);
- part.setLastPart(partNumber == numParts);
- UploadPartResult partResult = writeOperations.uploadPart(part, statistics);
- offset += uploadPartSize;
- parts.add(partResult.getPartETag());
+
+ // Open the file to upload.
+ try (InputStream fileStream = Files.newInputStream(localFile.toPath())) {
+ for (int partNumber = 1; partNumber <= numParts; partNumber += 1) {
+ progress.progress();
+ long size = Math.min(length - offset, uploadPartSize);
+ UploadPartRequest part = writeOperations.newUploadPartRequestBuilder(
+ destKey,
+ uploadId,
+ partNumber,
+ size).build();
+ // Read from the file input stream at current position.
+ RequestBody body = RequestBody.fromInputStream(fileStream, size);
+ UploadPartResponse response = writeOperations.uploadPart(part, body, statistics);
+ offset += uploadPartSize;
+ parts.add(CompletedPart.builder()
+ .partNumber(partNumber)
+ .eTag(response.eTag())
+ .build());
+ }
}
commitData.bindCommitData(parts);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
index 1a5451df801dd..b2e703e1b088d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
@@ -20,19 +20,22 @@
import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.CompletedPart;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.Retries;
+import org.apache.hadoop.fs.s3a.S3ADataBlocks;
import org.apache.hadoop.fs.s3a.WriteOperationHelper;
import org.apache.hadoop.fs.s3a.commit.PutTracker;
import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
@@ -125,7 +128,7 @@ public boolean outputImmediatelyVisible() {
*/
@Override
public boolean aboutToComplete(String uploadId,
- List parts,
+ List parts,
long bytesWritten,
final IOStatistics iostatistics)
throws IOException {
@@ -144,10 +147,9 @@ public boolean aboutToComplete(String uploadId,
headers.put(X_HEADER_MAGIC_MARKER, Long.toString(bytesWritten));
PutObjectRequest originalDestPut = writer.createPutObjectRequest(
originalDestKey,
- new ByteArrayInputStream(EMPTY),
0,
- new PutObjectOptions(true, null, headers));
- upload(originalDestPut);
+ new PutObjectOptions(true, null, headers), false);
+ upload(originalDestPut, new ByteArrayInputStream(EMPTY));
// build the commit summary
SinglePendingCommit commitData = new SinglePendingCommit();
@@ -170,22 +172,22 @@ public boolean aboutToComplete(String uploadId,
path, pendingPartKey, commitData);
PutObjectRequest put = writer.createPutObjectRequest(
pendingPartKey,
- new ByteArrayInputStream(bytes),
- bytes.length, null);
- upload(put);
+ bytes.length, null, false);
+ upload(put, new ByteArrayInputStream(bytes));
return false;
}
/**
* PUT an object.
* @param request the request
+ * @param inputStream input stream of data to be uploaded
* @throws IOException on problems
*/
@Retries.RetryTranslated
- private void upload(PutObjectRequest request) throws IOException {
- trackDurationOfInvocation(trackerStatistics,
- COMMITTER_MAGIC_MARKER_PUT.getSymbol(), () ->
- writer.putObject(request, PutObjectOptions.keepingDirs(), null));
+ private void upload(PutObjectRequest request, InputStream inputStream) throws IOException {
+ trackDurationOfInvocation(trackerStatistics, COMMITTER_MAGIC_MARKER_PUT.getSymbol(),
+ () -> writer.putObject(request, PutObjectOptions.keepingDirs(),
+ new S3ADataBlocks.BlockUploadData(inputStream), false, null));
}
@Override
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSCannedACL.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSCannedACL.java
new file mode 100644
index 0000000000000..12a89d50f6b0d
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSCannedACL.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+/**
+ * Enum to map AWS SDK V1 Acl values to SDK V2.
+ */
+public enum AWSCannedACL {
+ Private("private"),
+ PublicRead("public-read"),
+ PublicReadWrite("public-read-write"),
+ AuthenticatedRead("authenticated-read"),
+ AwsExecRead("aws-exec-read"),
+ BucketOwnerRead("bucket-owner-read"),
+ BucketOwnerFullControl("bucket-owner-full-control"),
+ LogDeliveryWrite("log-delivery-write");
+
+ private final String value;
+
+ AWSCannedACL(String value){
+ this.value = value;
+ }
+
+ public String toString() {
+ return this.value;
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
new file mode 100644
index 0000000000000..3400daff509b2
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
+import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
+import software.amazon.awssdk.core.retry.RetryPolicy;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.apache.ProxyConfiguration;
+import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.S3AUtils;
+import org.apache.hadoop.fs.s3a.auth.SignerFactory;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.http.client.utils.URIBuilder;
+
+import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_S3;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_STS;
+import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_TTL;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_TTL;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ESTABLISH_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAXIMUM_CONNECTIONS;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAX_ERROR_RETRIES;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_REQUEST_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SOCKET_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.ESTABLISH_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS;
+import static org.apache.hadoop.fs.s3a.Constants.MAX_ERROR_RETRIES;
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_DOMAIN;
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_HOST;
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_PASSWORD;
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_PORT;
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_SECURED;
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_USERNAME;
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_WORKSTATION;
+import static org.apache.hadoop.fs.s3a.Constants.REQUEST_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.SIGNING_ALGORITHM;
+import static org.apache.hadoop.fs.s3a.Constants.SIGNING_ALGORITHM_S3;
+import static org.apache.hadoop.fs.s3a.Constants.SIGNING_ALGORITHM_STS;
+import static org.apache.hadoop.fs.s3a.Constants.SOCKET_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.USER_AGENT_PREFIX;
+import static org.apache.hadoop.fs.s3a.S3AUtils.longOption;
+
+/**
+ * Methods for configuring the S3 client.
+ * These methods are used when creating and configuring
+ * {@link software.amazon.awssdk.services.s3.S3Client} which communicates with the S3 service.
+ */
+public final class AWSClientConfig {
+ private static final Logger LOG = LoggerFactory.getLogger(AWSClientConfig.class);
+
+ private AWSClientConfig() {
+ }
+
+ public static ClientOverrideConfiguration.Builder createClientConfigBuilder(Configuration conf,
+ String awsServiceIdentifier) throws IOException {
+ ClientOverrideConfiguration.Builder overrideConfigBuilder =
+ ClientOverrideConfiguration.builder();
+
+ initRequestTimeout(conf, overrideConfigBuilder);
+
+ initUserAgent(conf, overrideConfigBuilder);
+
+ String signer = conf.getTrimmed(SIGNING_ALGORITHM, "");
+ if (!signer.isEmpty()) {
+ LOG.debug("Signer override = {}", signer);
+ overrideConfigBuilder.putAdvancedOption(SdkAdvancedClientOption.SIGNER,
+ SignerFactory.createSigner(signer, SIGNING_ALGORITHM));
+ }
+
+ initSigner(conf, overrideConfigBuilder, awsServiceIdentifier);
+
+ return overrideConfigBuilder;
+ }
+
+ /**
+ * Configures the http client.
+ *
+ * @param conf The Hadoop configuration
+ * @return Http client builder
+ * @throws IOException on any problem
+ */
+ public static ApacheHttpClient.Builder createHttpClientBuilder(Configuration conf)
+ throws IOException {
+ ApacheHttpClient.Builder httpClientBuilder =
+ ApacheHttpClient.builder();
+
+ httpClientBuilder.maxConnections(S3AUtils.intOption(conf, MAXIMUM_CONNECTIONS,
+ DEFAULT_MAXIMUM_CONNECTIONS, 1));
+
+ int connectionEstablishTimeout =
+ S3AUtils.intOption(conf, ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT, 0);
+ int socketTimeout = S3AUtils.intOption(conf, SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT, 0);
+
+ httpClientBuilder.connectionTimeout(Duration.ofMillis(connectionEstablishTimeout));
+ httpClientBuilder.socketTimeout(Duration.ofMillis(socketTimeout));
+
+ // set the connection TTL irrespective of whether the connection is in use or not.
+ // this can balance requests over different S3 servers, and avoid failed
+ // connections. See HADOOP-18845.
+ long ttl = longOption(conf, CONNECTION_TTL, DEFAULT_CONNECTION_TTL, -1);
+ httpClientBuilder.connectionTimeToLive(Duration.ofMillis(ttl));
+
+ NetworkBinding.bindSSLChannelMode(conf, httpClientBuilder);
+
+ return httpClientBuilder;
+ }
+
+ /**
+ * Configures the async http client.
+ *
+ * @param conf The Hadoop configuration
+ * @return Http client builder
+ */
+ public static NettyNioAsyncHttpClient.Builder createAsyncHttpClientBuilder(Configuration conf) {
+ NettyNioAsyncHttpClient.Builder httpClientBuilder =
+ NettyNioAsyncHttpClient.builder();
+
+ httpClientBuilder.maxConcurrency(S3AUtils.intOption(conf, MAXIMUM_CONNECTIONS,
+ DEFAULT_MAXIMUM_CONNECTIONS, 1));
+
+ int connectionEstablishTimeout =
+ S3AUtils.intOption(conf, ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT, 0);
+ int socketTimeout = S3AUtils.intOption(conf, SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT, 0);
+
+ httpClientBuilder.connectionTimeout(Duration.ofMillis(connectionEstablishTimeout));
+ httpClientBuilder.readTimeout(Duration.ofMillis(socketTimeout));
+ httpClientBuilder.writeTimeout(Duration.ofMillis(socketTimeout));
+
+ // set the connection TTL irrespective of whether the connection is in use or not.
+ // this can balance requests over different S3 servers, and avoid failed
+ // connections. See HADOOP-18845.
+ long ttl = longOption(conf, CONNECTION_TTL, DEFAULT_CONNECTION_TTL, -1);
+ httpClientBuilder.connectionTimeToLive(Duration.ofMillis(ttl));
+
+ // TODO: Don't think you can set a socket factory for the netty client.
+ // NetworkBinding.bindSSLChannelMode(conf, awsConf);
+
+ return httpClientBuilder;
+ }
+
+ /**
+ * Configures the retry policy.
+ *
+ * @param conf The Hadoop configuration
+ * @return Retry policy builder
+ */
+ public static RetryPolicy.Builder createRetryPolicyBuilder(Configuration conf) {
+
+ RetryPolicy.Builder retryPolicyBuilder = RetryPolicy.builder();
+
+ retryPolicyBuilder.numRetries(S3AUtils.intOption(conf, MAX_ERROR_RETRIES,
+ DEFAULT_MAX_ERROR_RETRIES, 0));
+
+ return retryPolicyBuilder;
+ }
+
+ /**
+ * Configures the proxy.
+ *
+ * @param conf The Hadoop configuration
+ * @param bucket Optional bucket to use to look up per-bucket proxy secrets
+ * @return Proxy configuration
+ * @throws IOException on any IO problem
+ */
+ public static ProxyConfiguration createProxyConfiguration(Configuration conf,
+ String bucket) throws IOException {
+
+ ProxyConfiguration.Builder proxyConfigBuilder = ProxyConfiguration.builder();
+
+ String proxyHost = conf.getTrimmed(PROXY_HOST, "");
+ int proxyPort = conf.getInt(PROXY_PORT, -1);
+
+ if (!proxyHost.isEmpty()) {
+ if (proxyPort >= 0) {
+ String scheme = conf.getBoolean(PROXY_SECURED, false) ? "https" : "http";
+ proxyConfigBuilder.endpoint(buildURI(scheme, proxyHost, proxyPort));
+ } else {
+ if (conf.getBoolean(PROXY_SECURED, false)) {
+ LOG.warn("Proxy host set without port. Using HTTPS default 443");
+ proxyConfigBuilder.endpoint(buildURI("https", proxyHost, 443));
+ } else {
+ LOG.warn("Proxy host set without port. Using HTTP default 80");
+ proxyConfigBuilder.endpoint(buildURI("http", proxyHost, 80));
+ }
+ }
+ final String proxyUsername = S3AUtils.lookupPassword(bucket, conf, PROXY_USERNAME,
+ null, null);
+ final String proxyPassword = S3AUtils.lookupPassword(bucket, conf, PROXY_PASSWORD,
+ null, null);
+ if ((proxyUsername == null) != (proxyPassword == null)) {
+ String msg = "Proxy error: " + PROXY_USERNAME + " or " +
+ PROXY_PASSWORD + " set without the other.";
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+ proxyConfigBuilder.username(proxyUsername);
+ proxyConfigBuilder.password(proxyPassword);
+ proxyConfigBuilder.ntlmDomain(conf.getTrimmed(PROXY_DOMAIN));
+ proxyConfigBuilder.ntlmWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using proxy server {}:{} as user {} with password {} on "
+ + "domain {} as workstation {}", proxyHost, proxyPort, proxyUsername, proxyPassword,
+ PROXY_DOMAIN, PROXY_WORKSTATION);
+ }
+ } else if (proxyPort >= 0) {
+ String msg =
+ "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+
+ return proxyConfigBuilder.build();
+ }
+
+ /**
+ * Configures the proxy for the async http client.
+ *
+ * @param conf The Hadoop configuration
+ * @param bucket Optional bucket to use to look up per-bucket proxy secrets
+ * @return Proxy configuration
+ * @throws IOException on any IO problem
+ */
+ public static software.amazon.awssdk.http.nio.netty.ProxyConfiguration
+ createAsyncProxyConfiguration(Configuration conf,
+ String bucket) throws IOException {
+
+ software.amazon.awssdk.http.nio.netty.ProxyConfiguration.Builder proxyConfigBuilder =
+ software.amazon.awssdk.http.nio.netty.ProxyConfiguration.builder();
+
+ String proxyHost = conf.getTrimmed(PROXY_HOST, "");
+ int proxyPort = conf.getInt(PROXY_PORT, -1);
+
+ if (!proxyHost.isEmpty()) {
+ if (proxyPort >= 0) {
+ String scheme = conf.getBoolean(PROXY_SECURED, false) ? "https" : "http";
+ proxyConfigBuilder.host(proxyHost);
+ proxyConfigBuilder.port(proxyPort);
+ proxyConfigBuilder.scheme(scheme);
+ } else {
+ if (conf.getBoolean(PROXY_SECURED, false)) {
+ LOG.warn("Proxy host set without port. Using HTTPS default 443");
+ proxyConfigBuilder.host(proxyHost);
+ proxyConfigBuilder.port(443);
+ proxyConfigBuilder.scheme("https");
+ } else {
+ LOG.warn("Proxy host set without port. Using HTTP default 80");
+ proxyConfigBuilder.host(proxyHost);
+ proxyConfigBuilder.port(80);
+ proxyConfigBuilder.scheme("http");
+ }
+ }
+ final String proxyUsername = S3AUtils.lookupPassword(bucket, conf, PROXY_USERNAME,
+ null, null);
+ final String proxyPassword = S3AUtils.lookupPassword(bucket, conf, PROXY_PASSWORD,
+ null, null);
+ if ((proxyUsername == null) != (proxyPassword == null)) {
+ String msg = "Proxy error: " + PROXY_USERNAME + " or " +
+ PROXY_PASSWORD + " set without the other.";
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+ proxyConfigBuilder.username(proxyUsername);
+ proxyConfigBuilder.password(proxyPassword);
+ // TODO: check NTLM support
+ // proxyConfigBuilder.ntlmDomain(conf.getTrimmed(PROXY_DOMAIN));
+ // proxyConfigBuilder.ntlmWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using proxy server {}:{} as user {} with password {} on "
+ + "domain {} as workstation {}", proxyHost, proxyPort, proxyUsername, proxyPassword,
+ PROXY_DOMAIN, PROXY_WORKSTATION);
+ }
+ } else if (proxyPort >= 0) {
+ String msg =
+ "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ } else {
+ return null;
+ }
+
+ return proxyConfigBuilder.build();
+ }
+
+ /***
+ * Builds a URI, throws an IllegalArgumentException in case of errors.
+ *
+ * @param host proxy host
+ * @param port proxy port
+ * @return uri with host and port
+ */
+ private static URI buildURI(String scheme, String host, int port) {
+ try {
+ return new URIBuilder().setScheme(scheme).setHost(host).setPort(port).build();
+ } catch (URISyntaxException e) {
+ String msg =
+ "Proxy error: incorrect " + PROXY_HOST + " or " + PROXY_PORT;
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+ }
+
+ /**
+ * Initializes the User-Agent header to send in HTTP requests to AWS
+ * services. We always include the Hadoop version number. The user also
+ * may set an optional custom prefix to put in front of the Hadoop version
+ * number. The AWS SDK internally appends its own information, which seems
+ * to include the AWS SDK version, OS and JVM version.
+ *
+ * @param conf Hadoop configuration
+ * @param clientConfig AWS SDK configuration to update
+ */
+ private static void initUserAgent(Configuration conf,
+ ClientOverrideConfiguration.Builder clientConfig) {
+ String userAgent = "Hadoop " + VersionInfo.getVersion();
+ String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, "");
+ if (!userAgentPrefix.isEmpty()) {
+ userAgent = userAgentPrefix + ", " + userAgent;
+ }
+ LOG.debug("Using User-Agent: {}", userAgent);
+ clientConfig.putAdvancedOption(SdkAdvancedClientOption.USER_AGENT_PREFIX, userAgent);
+ }
+
+ private static void initSigner(Configuration conf,
+ ClientOverrideConfiguration.Builder clientConfig, String awsServiceIdentifier)
+ throws IOException {
+ String configKey = null;
+ switch (awsServiceIdentifier) {
+ case AWS_SERVICE_IDENTIFIER_S3:
+ configKey = SIGNING_ALGORITHM_S3;
+ break;
+ case AWS_SERVICE_IDENTIFIER_STS:
+ configKey = SIGNING_ALGORITHM_STS;
+ break;
+ default:
+ // Nothing to do. The original signer override is already setup
+ }
+ if (configKey != null) {
+ String signerOverride = conf.getTrimmed(configKey, "");
+ if (!signerOverride.isEmpty()) {
+ LOG.debug("Signer override for {} = {}", awsServiceIdentifier, signerOverride);
+ clientConfig.putAdvancedOption(SdkAdvancedClientOption.SIGNER,
+ SignerFactory.createSigner(signerOverride, configKey));
+ }
+ }
+ }
+
+ /**
+ * Configures request timeout.
+ *
+ * @param conf Hadoop configuration
+ * @param clientConfig AWS SDK configuration to update
+ */
+ private static void initRequestTimeout(Configuration conf,
+ ClientOverrideConfiguration.Builder clientConfig) {
+ long requestTimeoutMillis = conf.getTimeDuration(REQUEST_TIMEOUT,
+ DEFAULT_REQUEST_TIMEOUT, TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
+
+ if (requestTimeoutMillis > Integer.MAX_VALUE) {
+ LOG.debug("Request timeout is too high({} ms). Setting to {} ms instead",
+ requestTimeoutMillis, Integer.MAX_VALUE);
+ requestTimeoutMillis = Integer.MAX_VALUE;
+ }
+
+ if(requestTimeoutMillis > 0) {
+ clientConfig.apiCallAttemptTimeout(Duration.ofMillis(requestTimeoutMillis));
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java
new file mode 100644
index 0000000000000..e0d6fa5aecc0b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+/**
+ * Common S3 HTTP header values used throughout the Amazon Web Services S3 Java client.
+ */
+public interface AWSHeaders {
+
+ /*
+ * Standard HTTP Headers.
+ */
+
+ String CACHE_CONTROL = "Cache-Control";
+ String CONTENT_DISPOSITION = "Content-Disposition";
+ String CONTENT_ENCODING = "Content-Encoding";
+ String CONTENT_LENGTH = "Content-Length";
+ String CONTENT_RANGE = "Content-Range";
+ String CONTENT_MD5 = "Content-MD5";
+ String CONTENT_TYPE = "Content-Type";
+ String CONTENT_LANGUAGE = "Content-Language";
+ String DATE = "Date";
+ String ETAG = "ETag";
+ String LAST_MODIFIED = "Last-Modified";
+
+ /*
+ * Amazon HTTP Headers used by S3A.
+ */
+
+ /** S3's version ID header. */
+ String S3_VERSION_ID = "x-amz-version-id";
+
+ /** Header describing what class of storage a user wants. */
+ String STORAGE_CLASS = "x-amz-storage-class";
+
+ /** Header describing what archive tier the object is in, if any. */
+ String ARCHIVE_STATUS = "x-amz-archive-status";
+
+ /** Header for optional server-side encryption algorithm. */
+ String SERVER_SIDE_ENCRYPTION = "x-amz-server-side-encryption";
+
+ /** Range header for the get object request. */
+ String RANGE = "Range";
+
+ /**
+ * Encrypted symmetric key header that is used in the Encryption Only (EO) envelope
+ * encryption mechanism.
+ */
+ @Deprecated
+ String CRYPTO_KEY = "x-amz-key";
+
+ /** JSON-encoded description of encryption materials used during encryption. */
+ String MATERIALS_DESCRIPTION = "x-amz-matdesc";
+
+ /** Header for the optional restore information of an object. */
+ String RESTORE = "x-amz-restore";
+
+ /**
+ * Key wrapping algorithm such as "AESWrap" and "RSA/ECB/OAEPWithSHA-256AndMGF1Padding".
+ */
+ String CRYPTO_KEYWRAP_ALGORITHM = "x-amz-wrap-alg";
+ /**
+ * Content encryption algorithm, such as "AES/GCM/NoPadding".
+ */
+ String CRYPTO_CEK_ALGORITHM = "x-amz-cek-alg";
+
+ /**
+ * Headers in request indicating that the requester must be charged for data
+ * transfer.
+ */
+ String REQUESTER_PAYS_HEADER = "x-amz-request-payer";
+
+ /** Header for the replication status of an Amazon S3 Object.*/
+ String OBJECT_REPLICATION_STATUS = "x-amz-replication-status";
+
+ String OBJECT_LOCK_MODE = "x-amz-object-lock-mode";
+
+ String OBJECT_LOCK_RETAIN_UNTIL_DATE = "x-amz-object-lock-retain-until-date";
+
+ String OBJECT_LOCK_LEGAL_HOLD_STATUS = "x-amz-object-lock-legal-hold";
+
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BulkDeleteRetryHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BulkDeleteRetryHandler.java
index 4169a9899cb15..5808607762ea6 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BulkDeleteRetryHandler.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BulkDeleteRetryHandler.java
@@ -20,8 +20,9 @@
import java.util.List;
-import com.amazonaws.SdkClientException;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,6 +31,7 @@
import org.apache.hadoop.fs.s3a.Statistic;
import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext;
+
import static org.apache.hadoop.fs.s3a.S3AUtils.isThrottleException;
import static org.apache.hadoop.fs.s3a.Statistic.IGNORED_ERRORS;
import static org.apache.hadoop.fs.s3a.Statistic.STORE_IO_THROTTLED;
@@ -113,15 +115,15 @@ public void bulkDeleteRetried(
* @param deleteRequest request which failed.
*/
private void onDeleteThrottled(final DeleteObjectsRequest deleteRequest) {
- final List keys = deleteRequest.getKeys();
+ final List keys = deleteRequest.delete().objects();
final int size = keys.size();
incrementStatistic(STORE_IO_THROTTLED, size);
instrumentation.addValueToQuantiles(STORE_IO_THROTTLE_RATE, size);
THROTTLE_LOG.info(
"Bulk delete {} keys throttled -first key = {}; last = {}",
size,
- keys.get(0).getKey(),
- keys.get(size - 1).getKey());
+ keys.get(0).key(),
+ keys.get(size - 1).key());
}
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java
index 0156207419210..e0580df08a76d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java
@@ -22,6 +22,7 @@
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
@@ -155,19 +156,21 @@ public static void waitForCompletion(
* Wait for a single of future to complete, extracting IOEs afterwards.
* @param future future to wait for.
* @param type
+ * @return the result
* @throws IOException if one of the called futures raised an IOE.
* @throws RuntimeException if one of the futures raised one.
*/
- public static void waitForCompletion(
+ public static T waitForCompletion(
final CompletableFuture future)
throws IOException {
try (DurationInfo ignore =
new DurationInfo(LOG, false, "Waiting for task completion")) {
- future.join();
+ return future.join();
} catch (CancellationException e) {
throw new IOException(e);
} catch (CompletionException e) {
raiseInnerCause(e);
+ return null;
}
}
@@ -175,31 +178,35 @@ public static void waitForCompletion(
* Wait for a single of future to complete, ignoring exceptions raised.
* @param future future to wait for.
* @param type
+ * @return the outcome if successfully retrieved.
*/
- public static void waitForCompletionIgnoringExceptions(
+ public static Optional waitForCompletionIgnoringExceptions(
@Nullable final CompletableFuture future) {
- if (future != null) {
- try (DurationInfo ignore =
- new DurationInfo(LOG, false, "Waiting for task completion")) {
- future.join();
- } catch (Exception e) {
- LOG.debug("Ignoring exception raised in task completion: ");
- }
+
+ try {
+ return maybeAwaitCompletion(future);
+ } catch (Exception e) {
+ LOG.debug("Ignoring exception raised in task completion: ", e);
+ return Optional.empty();
}
}
/**
* Block awaiting completion for any non-null future passed in;
* No-op if a null arg was supplied.
+ * @param return type
* @param future future
+ * @return the outcome; is empty if the future was null/had no return value
* @throws IOException if one of the called futures raised an IOE.
* @throws RuntimeException if one of the futures raised one.
*/
- public static void maybeAwaitCompletion(
- @Nullable final CompletableFuture future)
+ public static Optional maybeAwaitCompletion(
+ @Nullable final CompletableFuture future)
throws IOException {
if (future != null) {
- waitForCompletion(future);
+ return Optional.ofNullable(waitForCompletion(future));
+ } else {
+ return Optional.empty();
}
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeDetectionPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeDetectionPolicy.java
index 205826dadae6f..65291aa7b2659 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeDetectionPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeDetectionPolicy.java
@@ -20,11 +20,12 @@
import java.util.Locale;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.transfer.model.CopyResult;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -211,11 +212,24 @@ public String toString() {
* null if the attribute is unavailable (such as when the policy says to use
* versionId but object versioning is not enabled for the bucket).
*/
- public abstract String getRevisionId(ObjectMetadata objectMetadata,
+ public abstract String getRevisionId(HeadObjectResponse objectMetadata,
String uri);
/**
- * Like {{@link #getRevisionId(ObjectMetadata, String)}}, but retrieves the
+ * Like {{@link #getRevisionId(HeadObjectResponse, String)}}, but retrieves the
+ * * revision identifier from {@link GetObjectResponse}.
+ *
+ * @param getObjectResponse the response instance
+ * @param uri the URI of the object
+ * @return the revisionId string as interpreted by this policy, or potentially
+ * null if the attribute is unavailable (such as when the policy says to use
+ * versionId but object versioning is not enabled for the bucket).
+ */
+ public abstract String getRevisionId(GetObjectResponse getObjectResponse,
+ String uri);
+
+ /**
+ * Like {{@link #getRevisionId(HeadObjectResponse, String)}}, but retrieves the
* revision identifier from {@link S3ObjectAttributes}.
*
* @param s3Attributes the object attributes
@@ -226,44 +240,44 @@ public abstract String getRevisionId(ObjectMetadata objectMetadata,
public abstract String getRevisionId(S3ObjectAttributes s3Attributes);
/**
- * Like {{@link #getRevisionId(ObjectMetadata, String)}}, but retrieves the
- * revision identifier from {@link CopyResult}.
+ * Like {{@link #getRevisionId(HeadObjectResponse, String)}}, but retrieves the
+ * revision identifier from {@link CopyObjectResponse}.
*
- * @param copyResult the copy result
+ * @param copyObjectResponse the copy response
* @return the revisionId string as interpreted by this policy, or potentially
* null if the attribute is unavailable (such as when the policy says to use
* versionId but object versioning is not enabled for the bucket).
*/
- public abstract String getRevisionId(CopyResult copyResult);
+ public abstract String getRevisionId(CopyObjectResponse copyObjectResponse);
/**
- * Applies the given {@link #getRevisionId(ObjectMetadata, String) revisionId}
- * as a server-side qualification on the {@code GetObjectRequest}.
+ * Applies the given {@link #getRevisionId(HeadObjectResponse, String) revisionId}
+ * as a server-side qualification on the {@code GetObjectRequest.Builder}.
*
* @param request the request
* @param revisionId the revision id
*/
- public abstract void applyRevisionConstraint(GetObjectRequest request,
+ public abstract void applyRevisionConstraint(GetObjectRequest.Builder request,
String revisionId);
/**
- * Applies the given {@link #getRevisionId(ObjectMetadata, String) revisionId}
+ * Applies the given {@link #getRevisionId(HeadObjectResponse, String) revisionId}
* as a server-side qualification on the {@code CopyObjectRequest}.
*
- * @param request the request
+ * @param requestBuilder the copy object request builder
* @param revisionId the revision id
*/
- public abstract void applyRevisionConstraint(CopyObjectRequest request,
+ public abstract void applyRevisionConstraint(CopyObjectRequest.Builder requestBuilder,
String revisionId);
/**
- * Applies the given {@link #getRevisionId(ObjectMetadata, String) revisionId}
+ * Applies the given {@link #getRevisionId(HeadObjectResponse, String) revisionId}
* as a server-side qualification on the {@code GetObjectMetadataRequest}.
*
- * @param request the request
+ * @param requestBuilder the head object request builder
* @param revisionId the revision id
*/
- public abstract void applyRevisionConstraint(GetObjectMetadataRequest request,
+ public abstract void applyRevisionConstraint(HeadObjectRequest.Builder requestBuilder,
String revisionId);
/**
@@ -323,7 +337,7 @@ public ImmutablePair onChangeDetected(
}
/**
- * Change detection policy based on {@link ObjectMetadata#getETag() eTag}.
+ * Change detection policy based on {@link HeadObjectResponse#eTag()} () eTag}.
*/
static class ETagChangeDetectionPolicy extends ChangeDetectionPolicy {
@@ -332,8 +346,13 @@ static class ETagChangeDetectionPolicy extends ChangeDetectionPolicy {
}
@Override
- public String getRevisionId(ObjectMetadata objectMetadata, String uri) {
- return objectMetadata.getETag();
+ public String getRevisionId(GetObjectResponse objectMetadata, String uri) {
+ return objectMetadata.eTag();
+ }
+
+ @Override
+ public String getRevisionId(HeadObjectResponse objectMetadata, String uri) {
+ return objectMetadata.eTag();
}
@Override
@@ -342,34 +361,34 @@ public String getRevisionId(S3ObjectAttributes s3Attributes) {
}
@Override
- public String getRevisionId(CopyResult copyResult) {
- return copyResult.getETag();
+ public String getRevisionId(CopyObjectResponse copyObjectResponse) {
+ return copyObjectResponse.copyObjectResult().eTag();
}
@Override
- public void applyRevisionConstraint(GetObjectRequest request,
+ public void applyRevisionConstraint(GetObjectRequest.Builder builder,
String revisionId) {
if (revisionId != null) {
LOG.debug("Restricting get request to etag {}", revisionId);
- request.withMatchingETagConstraint(revisionId);
+ builder.ifMatch(revisionId);
} else {
LOG.debug("No etag revision ID to use as a constraint");
}
}
@Override
- public void applyRevisionConstraint(CopyObjectRequest request,
+ public void applyRevisionConstraint(CopyObjectRequest.Builder requestBuilder,
String revisionId) {
if (revisionId != null) {
LOG.debug("Restricting copy request to etag {}", revisionId);
- request.withMatchingETagConstraint(revisionId);
+ requestBuilder.copySourceIfMatch(revisionId);
} else {
LOG.debug("No etag revision ID to use as a constraint");
}
}
@Override
- public void applyRevisionConstraint(GetObjectMetadataRequest request,
+ public void applyRevisionConstraint(HeadObjectRequest.Builder requestBuilder,
String revisionId) {
LOG.debug("Unable to restrict HEAD request to etag; will check later");
}
@@ -388,7 +407,7 @@ public String toString() {
/**
* Change detection policy based on
- * {@link ObjectMetadata#getVersionId() versionId}.
+ * {@link HeadObjectResponse#versionId()} () versionId}.
*/
static class VersionIdChangeDetectionPolicy extends
ChangeDetectionPolicy {
@@ -398,8 +417,16 @@ static class VersionIdChangeDetectionPolicy extends
}
@Override
- public String getRevisionId(ObjectMetadata objectMetadata, String uri) {
- String versionId = objectMetadata.getVersionId();
+ public String getRevisionId(HeadObjectResponse objectMetadata, String uri) {
+ return logIfNull(objectMetadata.versionId(), uri);
+ }
+
+ @Override
+ public String getRevisionId(GetObjectResponse getObjectResponse, String uri) {
+ return logIfNull(getObjectResponse.versionId(), uri);
+ }
+
+ private String logIfNull(String versionId, String uri) {
if (versionId == null) {
// this policy doesn't work if the bucket doesn't have object versioning
// enabled (which isn't by default)
@@ -419,38 +446,38 @@ public String getRevisionId(S3ObjectAttributes s3Attributes) {
}
@Override
- public String getRevisionId(CopyResult copyResult) {
- return copyResult.getVersionId();
+ public String getRevisionId(CopyObjectResponse copyObjectResponse) {
+ return copyObjectResponse.versionId();
}
@Override
- public void applyRevisionConstraint(GetObjectRequest request,
+ public void applyRevisionConstraint(GetObjectRequest.Builder builder,
String revisionId) {
if (revisionId != null) {
LOG.debug("Restricting get request to version {}", revisionId);
- request.withVersionId(revisionId);
+ builder.versionId(revisionId);
} else {
LOG.debug("No version ID to use as a constraint");
}
}
@Override
- public void applyRevisionConstraint(CopyObjectRequest request,
+ public void applyRevisionConstraint(CopyObjectRequest.Builder requestBuilder,
String revisionId) {
if (revisionId != null) {
LOG.debug("Restricting copy request to version {}", revisionId);
- request.withSourceVersionId(revisionId);
+ requestBuilder.sourceVersionId(revisionId);
} else {
LOG.debug("No version ID to use as a constraint");
}
}
@Override
- public void applyRevisionConstraint(GetObjectMetadataRequest request,
+ public void applyRevisionConstraint(HeadObjectRequest.Builder requestBuilder,
String revisionId) {
if (revisionId != null) {
LOG.debug("Restricting metadata request to version {}", revisionId);
- request.withVersionId(revisionId);
+ requestBuilder.versionId(revisionId);
} else {
LOG.debug("No version ID to use as a constraint");
}
@@ -482,7 +509,13 @@ public Source getSource() {
}
@Override
- public String getRevisionId(final ObjectMetadata objectMetadata,
+ public String getRevisionId(final GetObjectResponse objectMetadata,
+ final String uri) {
+ return null;
+ }
+
+ @Override
+ public String getRevisionId(final HeadObjectResponse objectMetadata,
final String uri) {
return null;
}
@@ -493,24 +526,24 @@ public String getRevisionId(final S3ObjectAttributes s3ObjectAttributes) {
}
@Override
- public String getRevisionId(CopyResult copyResult) {
+ public String getRevisionId(CopyObjectResponse copyObjectResponse) {
return null;
}
@Override
- public void applyRevisionConstraint(final GetObjectRequest request,
+ public void applyRevisionConstraint(final GetObjectRequest.Builder builder,
final String revisionId) {
}
@Override
- public void applyRevisionConstraint(CopyObjectRequest request,
+ public void applyRevisionConstraint(CopyObjectRequest.Builder requestBuilder,
String revisionId) {
}
@Override
- public void applyRevisionConstraint(GetObjectMetadataRequest request,
+ public void applyRevisionConstraint(HeadObjectRequest.Builder requestBuilder,
String revisionId) {
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeTracker.java
index 2e144bbfcd04f..c7e9139f32d96 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeTracker.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeTracker.java
@@ -18,14 +18,14 @@
package org.apache.hadoop.fs.s3a.impl;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.transfer.model.CopyResult;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,6 +39,7 @@
import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
import org.apache.hadoop.fs.s3a.statistics.ChangeTrackerStatistics;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_412_PRECONDITION_FAILED;
import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
/**
@@ -55,8 +56,6 @@ public class ChangeTracker {
private static final Logger LOG =
LoggerFactory.getLogger(ChangeTracker.class);
- /** {@code 412 Precondition Failed} (HTTP/1.1 - RFC 2616) */
- public static final int SC_PRECONDITION_FAILED = 412;
public static final String CHANGE_REPORTED_BY_S3 = "Change reported by S3";
/** Policy to use. */
@@ -117,15 +116,15 @@ public long getVersionMismatches() {
/**
* Apply any revision control set by the policy if it is to be
* enforced on the server.
- * @param request request to modify
+ * @param builder request builder to modify
* @return true iff a constraint was added.
*/
public boolean maybeApplyConstraint(
- final GetObjectRequest request) {
+ final GetObjectRequest.Builder builder) {
if (policy.getMode() == ChangeDetectionPolicy.Mode.Server
&& revisionId != null) {
- policy.applyRevisionConstraint(request, revisionId);
+ policy.applyRevisionConstraint(builder, revisionId);
return true;
}
return false;
@@ -134,26 +133,26 @@ public boolean maybeApplyConstraint(
/**
* Apply any revision control set by the policy if it is to be
* enforced on the server.
- * @param request request to modify
+ * @param requestBuilder copy object request builder to modify
* @return true iff a constraint was added.
*/
public boolean maybeApplyConstraint(
- final CopyObjectRequest request) {
+ final CopyObjectRequest.Builder requestBuilder) {
if (policy.getMode() == ChangeDetectionPolicy.Mode.Server
&& revisionId != null) {
- policy.applyRevisionConstraint(request, revisionId);
+ policy.applyRevisionConstraint(requestBuilder, revisionId);
return true;
}
return false;
}
public boolean maybeApplyConstraint(
- final GetObjectMetadataRequest request) {
+ final HeadObjectRequest.Builder requestBuilder) {
if (policy.getMode() == ChangeDetectionPolicy.Mode.Server
&& revisionId != null) {
- policy.applyRevisionConstraint(request, revisionId);
+ policy.applyRevisionConstraint(requestBuilder, revisionId);
return true;
}
return false;
@@ -168,7 +167,7 @@ public boolean maybeApplyConstraint(
* @throws PathIOException raised on failure
* @throws RemoteFileChangedException if the remote file has changed.
*/
- public void processResponse(final S3Object object,
+ public void processResponse(final GetObjectResponse object,
final String operation,
final long pos) throws PathIOException {
if (object == null) {
@@ -191,24 +190,24 @@ public void processResponse(final S3Object object,
}
}
- processMetadata(object.getObjectMetadata(), operation);
+ processMetadata(object, operation);
}
/**
* Process the response from the server for validation against the
* change policy.
- * @param copyResult result of a copy operation
+ * @param copyObjectResponse response of a copy operation
* @throws PathIOException raised on failure
* @throws RemoteFileChangedException if the remote file has changed.
*/
- public void processResponse(final CopyResult copyResult)
+ public void processResponse(final CopyObjectResponse copyObjectResponse)
throws PathIOException {
// ETag (sometimes, depending on encryption and/or multipart) is not the
// same on the copied object as the original. Version Id seems to never
// be the same on the copy. As such, there isn't really anything that
// can be verified on the response, except that a revision ID is present
// if required.
- String newRevisionId = policy.getRevisionId(copyResult);
+ String newRevisionId = policy.getRevisionId(copyObjectResponse);
LOG.debug("Copy result {}: {}", policy.getSource(), newRevisionId);
if (newRevisionId == null && policy.isRequireVersion()) {
throw new NoVersionAttributeException(uri, String.format(
@@ -227,13 +226,11 @@ public void processResponse(final CopyResult copyResult)
* generated (e.g. "copy", "read", "select").
* @throws RemoteFileChangedException if the remote file has changed.
*/
- public void processException(SdkBaseException e, String operation) throws
+ public void processException(SdkException e, String operation) throws
RemoteFileChangedException {
- if (e instanceof AmazonServiceException) {
- AmazonServiceException serviceException = (AmazonServiceException) e;
- // This isn't really going to be hit due to
- // https://github.com/aws/aws-sdk-java/issues/1644
- if (serviceException.getStatusCode() == SC_PRECONDITION_FAILED) {
+ if (e instanceof AwsServiceException) {
+ AwsServiceException serviceException = (AwsServiceException)e;
+ if (serviceException.statusCode() == SC_412_PRECONDITION_FAILED) {
versionMismatches.versionMismatchError();
throw new RemoteFileChangedException(uri, operation, String.format(
RemoteFileChangedException.PRECONDITIONS_FAILED
@@ -254,12 +251,26 @@ public void processException(SdkBaseException e, String operation) throws
* @throws PathIOException raised on failure
* @throws RemoteFileChangedException if the remote file has changed.
*/
- public void processMetadata(final ObjectMetadata metadata,
+ public void processMetadata(final HeadObjectResponse metadata,
final String operation) throws PathIOException {
final String newRevisionId = policy.getRevisionId(metadata, uri);
processNewRevision(newRevisionId, operation, -1);
}
+ /**
+ * Process the response from server for validation against the change
+ * policy.
+ * @param getObjectResponse response returned from server
+ * @param operation operation in progress
+ * @throws PathIOException raised on failure
+ * @throws RemoteFileChangedException if the remote file has changed.
+ */
+ public void processMetadata(final GetObjectResponse getObjectResponse,
+ final String operation) throws PathIOException {
+ final String newRevisionId = policy.getRevisionId(getObjectResponse, uri);
+ processNewRevision(newRevisionId, operation, -1);
+ }
+
/**
* Validate a revision from the server against our expectations.
* @param newRevisionId new revision.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java
index 8205668e8f354..ba21f6028a5f8 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java
@@ -21,8 +21,8 @@
import javax.net.ssl.HostnameVerifier;
import java.io.IOException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.thirdparty.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.thirdparty.org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
@@ -35,13 +35,12 @@ public class ConfigureShadedAWSSocketFactory implements
NetworkBinding.ConfigureAWSSocketFactory {
@Override
- public void configureSocketFactory(final ClientConfiguration awsConf,
+ public void configureSocketFactory(final ApacheHttpClient.Builder httpClientBuilder,
final DelegatingSSLSocketFactory.SSLChannelMode channelMode)
throws IOException {
DelegatingSSLSocketFactory.initializeDefaultFactory(channelMode);
- awsConf.getApacheHttpClientConfig().setSslSocketFactory(
- new SSLConnectionSocketFactory(
- DelegatingSSLSocketFactory.getDefaultFactory(),
- (HostnameVerifier) null));
+ httpClientBuilder.socketFactory(new SSLConnectionSocketFactory(
+ DelegatingSSLSocketFactory.getDefaultFactory(),
+ (HostnameVerifier) null));
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CopyOutcome.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CopyOutcome.java
deleted file mode 100644
index 16459ac45b850..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CopyOutcome.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.impl;
-
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.services.s3.transfer.Copy;
-import com.amazonaws.services.s3.transfer.model.CopyResult;
-
-/**
- * Extracts the outcome of a TransferManager-executed copy operation.
- */
-public final class CopyOutcome {
-
- /**
- * Result of a successful copy.
- */
- private final CopyResult copyResult;
-
- /** the copy was interrupted. */
- private final InterruptedException interruptedException;
-
- /**
- * The copy raised an AWS Exception of some form.
- */
- private final SdkBaseException awsException;
-
- public CopyOutcome(CopyResult copyResult,
- InterruptedException interruptedException,
- SdkBaseException awsException) {
- this.copyResult = copyResult;
- this.interruptedException = interruptedException;
- this.awsException = awsException;
- }
-
- public CopyResult getCopyResult() {
- return copyResult;
- }
-
- public InterruptedException getInterruptedException() {
- return interruptedException;
- }
-
- public SdkBaseException getAwsException() {
- return awsException;
- }
-
- /**
- * Calls {@code Copy.waitForCopyResult()} to await the result, converts
- * it to a copy outcome.
- * Exceptions caught and
- * @param copy the copy operation.
- * @return the outcome.
- */
- public static CopyOutcome waitForCopy(Copy copy) {
- try {
- CopyResult result = copy.waitForCopyResult();
- return new CopyOutcome(result, null, null);
- } catch (SdkBaseException e) {
- return new CopyOutcome(null, null, e);
- } catch (InterruptedException e) {
- return new CopyOutcome(null, e, null);
- }
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java
index 0797c36c52910..7cc94cf89203f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java
@@ -21,10 +21,11 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService;
import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;
import org.slf4j.Logger;
@@ -40,6 +41,8 @@
import org.apache.hadoop.fs.s3a.Tristate;
import org.apache.hadoop.util.DurationInfo;
+
+import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletionIgnoringExceptions;
import static org.apache.hadoop.fs.store.audit.AuditingFunctions.callableWithinAuditSpan;
import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.maybeAwaitCompletion;
@@ -109,6 +112,16 @@ public class DeleteOperation extends ExecutingStoreOperation {
*/
private long filesDeleted;
+ /**
+ * Do directory operations purge pending uploads?
+ */
+ private final boolean dirOperationsPurgeUploads;
+
+ /**
+ * Count of uploads aborted.
+ */
+ private Optional uploadsAborted = Optional.empty();
+
/**
* Constructor.
* @param context store context
@@ -116,12 +129,14 @@ public class DeleteOperation extends ExecutingStoreOperation {
* @param recursive recursive delete?
* @param callbacks callback provider
* @param pageSize size of delete pages
+ * @param dirOperationsPurgeUploads Do directory operations purge pending uploads?
*/
public DeleteOperation(final StoreContext context,
final S3AFileStatus status,
final boolean recursive,
final OperationCallbacks callbacks,
- final int pageSize) {
+ final int pageSize,
+ final boolean dirOperationsPurgeUploads) {
super(context);
this.status = status;
@@ -133,12 +148,22 @@ public DeleteOperation(final StoreContext context,
this.pageSize = pageSize;
executor = MoreExecutors.listeningDecorator(
context.createThrottledExecutor(1));
+ this.dirOperationsPurgeUploads = dirOperationsPurgeUploads;
}
public long getFilesDeleted() {
return filesDeleted;
}
+ /**
+ * Get the count of uploads aborted.
+ * Non-empty iff enabled, and the operations completed without errors.
+ * @return count of aborted uploads.
+ */
+ public Optional getUploadsAborted() {
+ return uploadsAborted;
+ }
+
/**
* Delete a file or directory tree.
*
@@ -235,6 +260,17 @@ protected void deleteDirectoryTree(final Path path,
try (DurationInfo ignored =
new DurationInfo(LOG, false, "deleting %s", dirKey)) {
+ final CompletableFuture abortUploads;
+ if (dirOperationsPurgeUploads) {
+ final StoreContext sc = getStoreContext();
+ final String key = sc.pathToKey(path) + "/";
+ LOG.debug("All uploads under {} will be deleted", key);
+ abortUploads = submit(sc.getExecutor(), sc.getActiveAuditSpan(), () ->
+ callbacks.abortMultipartUploadsUnderPrefix(key));
+ } else {
+ abortUploads = null;
+ }
+
// init the lists of keys and paths to delete
resetDeleteList();
deleteFuture = null;
@@ -256,10 +292,10 @@ protected void deleteDirectoryTree(final Path path,
LOG.debug("Deleting final batch of listed files");
submitNextBatch();
maybeAwaitCompletion(deleteFuture);
-
+ uploadsAborted = waitForCompletionIgnoringExceptions(abortUploads);
}
- LOG.debug("Delete \"{}\" completed; deleted {} objects", path,
- filesDeleted);
+ LOG.debug("Delete \"{}\" completed; deleted {} objects and aborted {} uploads", path,
+ filesDeleted, uploadsAborted.orElse(0L));
}
/**
@@ -312,7 +348,8 @@ private void submitNextBatch()
throws IOException {
// delete a single page of keys and the metadata.
// block for any previous batch.
- maybeAwaitCompletion(deleteFuture);
+ maybeAwaitCompletion(deleteFuture).ifPresent(count ->
+ LOG.debug("Deleted {} uploads", count));
// delete the current page of keys and paths
deleteFuture = submitDelete(keys);
@@ -386,9 +423,9 @@ private void asyncDeleteAction(
"Delete page of %d keys", keyList.size())) {
if (!keyList.isEmpty()) {
// first delete the files.
- List files = keyList.stream()
+ List files = keyList.stream()
.filter(e -> !e.isDirMarker)
- .map(e -> e.keyVersion)
+ .map(e -> e.objectIdentifier)
.collect(Collectors.toList());
LOG.debug("Deleting of {} file objects", files.size());
Invoker.once("Remove S3 Files",
@@ -398,9 +435,9 @@ private void asyncDeleteAction(
false
));
// now the dirs
- List