forbiddenClassnames =
+ forbidden.stream().map(c -> c.getName()).collect(Collectors.toSet());
+
+
+ // iterate through, checking for forbidden values and then instantiating
+ // each provider
+ AWSCredentialProviderList providers = new AWSCredentialProviderList();
+ for (String className : awsClasses) {
+ if (v1v2CredentialProviderMap.containsKey(className)) {
+ // mapping
+
+ final String mapped = v1v2CredentialProviderMap.get(className);
+ LOG_REMAPPED_ENTRY.warn("Credentials option {} contains AWS v1 SDK entry {}; mapping to {}",
+ key, className, mapped);
+ className = mapped;
+ }
+ // now scan the forbidden list. doing this after any mappings ensures the v1 names
+ // are also blocked
+ if (forbiddenClassnames.contains(className)) {
+ throw new InstantiationIOException(InstantiationIOException.Kind.Forbidden,
+ binding, className, key, E_FORBIDDEN_AWS_PROVIDER, null);
+ }
+
+ AwsCredentialsProvider provider;
+ try {
+ provider = createAWSV2CredentialProvider(conf, className, binding, key);
+ } catch (InstantiationIOException e) {
+ // failed to create a v2; try to see if it is a v1
+ if (e.getKind() == InstantiationIOException.Kind.IsNotImplementation) {
+ if (isAwsV1SdkAvailable()) {
+ // try to create v1
+ LOG.debug("Failed to create {} as v2 credentials, trying to instantiate as v1",
+ className);
+ try {
+ provider =
+ AwsV1BindingSupport.createAWSV1CredentialProvider(conf, className, binding, key);
+ LOG_REMAPPED_ENTRY.warn("Credentials option {} contains AWS v1 SDK entry {}",
+ key, className);
+ } catch (InstantiationIOException ex) {
+ // if it is something other than non-implementation, throw.
+ // that way, non-impl messages are about v2 not v1 in the error
+ if (ex.getKind() != InstantiationIOException.Kind.IsNotImplementation) {
+ throw ex;
+ } else {
+ throw e;
+ }
+ }
+ } else {
+ LOG.warn("Failed to instantiate {} as AWS v2 SDK credential provider;"
+ + " AWS V1 SDK is not on the classpth so unable to attempt to"
+ + " instantiate as a v1 provider", className, e);
+ throw e;
+ }
+ } else {
+ // any other problem
+ throw e;
+
+ }
+ LOG.debug("From provider class {} created Aws provider {}", className, provider);
+ }
+ providers.add(provider);
+ }
+ return providers;
+ }
+
+ /**
+ * Create an AWS v2 credential provider from its class by using reflection.
+ * @param conf configuration
+ * @param className credential class name
+ * @param uri URI of the FS
+ * @param key configuration key to use
+ * @return the instantiated class
+ * @throws IOException on any instantiation failure.
+ * @see S3AUtils#getInstanceFromReflection
+ */
+ private static AwsCredentialsProvider createAWSV2CredentialProvider(Configuration conf,
+ String className,
+ @Nullable URI uri, final String key) throws IOException {
+ LOG.debug("Credential provider class is {}", className);
+ return S3AUtils.getInstanceFromReflection(className, conf, uri, AwsCredentialsProvider.class,
+ "create", key);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
index f505cfcab5d4f..2e39b275b4a4d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
@@ -32,7 +32,7 @@
/**
* This is an IAM credential provider which wraps
- * an {@code EC2ContainerCredentialsProviderWrapper}
+ * an {@code ContainerCredentialsProvider}
* to provide credentials when the S3A connector is instantiated on AWS EC2
* or the AWS container services.
*
@@ -90,4 +90,11 @@ private AwsCredentials getCredentials() {
public void close() throws IOException {
// no-op.
}
+
+ @Override
+ public String toString() {
+ return "IAMInstanceCredentialsProvider{" +
+ "containerCredentialsProvider=" + containerCredentialsProvider +
+ '}';
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
index 7beabb9fa3c84..c786086947fac 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java
@@ -106,7 +106,7 @@ public static Signer createSigner(String signerType, String configKey) throws IO
LOG.debug("Signer class is {}", className);
Signer signer =
- S3AUtils.getInstanceFromReflection(signerClass, null, null, Signer.class, "create",
+ S3AUtils.getInstanceFromReflection(className, null, null, Signer.class, "create",
configKey);
return signer;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
index 2fe6be61e2583..f46c1f9405eb2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
@@ -120,11 +120,6 @@ public class S3ADelegationTokens extends AbstractDTService {
*/
private AbstractDelegationTokenBinding tokenBinding;
- /**
- * List of cred providers; unset until {@link #bindToDelegationToken(Token)}.
- */
- //private Optional credentialProviders = Optional.empty();
-
/**
* delegation binding information; unset until {@link #bindToDelegationToken(Token)}.
*/
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java
index 780c13177880b..2b21ea173ef01 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java
@@ -47,8 +47,8 @@
import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
import static org.apache.hadoop.fs.s3a.Invoker.once;
-import static org.apache.hadoop.fs.s3a.auth.AwsCredentialListProvider.STANDARD_AWS_PROVIDERS;
-import static org.apache.hadoop.fs.s3a.auth.AwsCredentialListProvider.buildAWSProviderList;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.STANDARD_AWS_PROVIDERS;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.buildAWSProviderList;
import static org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding.fromAWSCredentials;
import static org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding.fromSTSCredentials;
import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*;
@@ -102,7 +102,8 @@ public class SessionTokenBinding extends AbstractDelegationTokenBinding {
private boolean hasSessionCreds;
/**
- * The auth chain for the parent options.
+ * The parent authentication chain: that used to request
+ * session/role credentials when deployed unbonded.
*/
private AWSCredentialProviderList parentAuthChain;
@@ -161,12 +162,14 @@ protected void serviceStart() throws Exception {
DEFAULT_DELEGATION_TOKEN_REGION);
// create the provider set for session credentials.
- parentAuthChain = buildAWSProviderList(
+ final AWSCredentialProviderList chain = buildAWSProviderList(
getCanonicalUri(),
conf,
AWS_CREDENTIALS_PROVIDER,
STANDARD_AWS_PROVIDERS,
new HashSet<>());
+ LOG.debug("Setting parent authentication chain to {}", chain);
+ setParentAuthChain(chain);
}
@Override
@@ -189,7 +192,7 @@ protected void serviceStop() throws Exception {
public AWSCredentialProviderList deployUnbonded()
throws IOException {
requireServiceStarted();
- return parentAuthChain;
+ return getParentAuthChain();
}
/**
@@ -291,7 +294,7 @@ private synchronized Optional maybeInitSTS()
// throw this.
final AwsCredentials parentCredentials = once("get credentials",
"",
- () -> parentAuthChain.resolveCredentials());
+ () -> getParentAuthChain().resolveCredentials());
hasSessionCreds = parentCredentials instanceof AwsSessionCredentials;
if (!hasSessionCreds) {
@@ -300,7 +303,7 @@ private synchronized Optional maybeInitSTS()
invoker = new Invoker(new S3ARetryPolicy(conf), LOG_EVENT);
StsClient tokenService =
- STSClientFactory.builder(parentAuthChain,
+ STSClientFactory.builder(getParentAuthChain(),
conf,
endpoint,
region,
@@ -371,7 +374,7 @@ public SessionTokenIdentifier createTokenIdentifier(
}
origin += " " + CREDENTIALS_CONVERTED_TO_DELEGATION_TOKEN;
final AwsCredentials awsCredentials
- = parentAuthChain.resolveCredentials();
+ = getParentAuthChain().resolveCredentials();
if (awsCredentials instanceof AwsSessionCredentials) {
marshalledCredentials = fromAWSCredentials(
(AwsSessionCredentials) awsCredentials);
@@ -421,4 +424,16 @@ protected void setTokenIdentifier(Optional
tokenIdentifier) {
this.tokenIdentifier = tokenIdentifier;
}
+
+ /**
+ * The auth chain for the parent options.
+ * @return the parent authentication chain.
+ */
+ protected AWSCredentialProviderList getParentAuthChain() {
+ return parentAuthChain;
+ }
+
+ protected void setParentAuthChain(AWSCredentialProviderList parentAuthChain) {
+ this.parentAuthChain = parentAuthChain;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
index a69907755eeac..0faa30efc68ca 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java
@@ -348,7 +348,7 @@ private static void initSigner(Configuration conf,
if (configKey != null) {
String signerOverride = conf.getTrimmed(configKey, "");
if (!signerOverride.isEmpty()) {
- LOG.debug("Signer override for {}} = {}", awsServiceIdentifier, signerOverride);
+ LOG.debug("Signer override for {} = {}", awsServiceIdentifier, signerOverride);
clientConfig.putAdvancedOption(SdkAdvancedClientOption.SIGNER,
SignerFactory.createSigner(signerOverride, configKey));
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java
index 3cb714588bd39..e0d6fa5aecc0b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java
@@ -24,7 +24,7 @@
public interface AWSHeaders {
/*
- * Standard HTTP Headers
+ * Standard HTTP Headers.
*/
String CACHE_CONTROL = "Cache-Control";
@@ -40,22 +40,22 @@ public interface AWSHeaders {
String LAST_MODIFIED = "Last-Modified";
/*
- * Amazon HTTP Headers used by S3A
+ * Amazon HTTP Headers used by S3A.
*/
- /** S3's version ID header */
+ /** S3's version ID header. */
String S3_VERSION_ID = "x-amz-version-id";
- /** Header describing what class of storage a user wants */
+ /** Header describing what class of storage a user wants. */
String STORAGE_CLASS = "x-amz-storage-class";
- /** Header describing what archive tier the object is in, if any */
+ /** Header describing what archive tier the object is in, if any. */
String ARCHIVE_STATUS = "x-amz-archive-status";
- /** Header for optional server-side encryption algorithm */
+ /** Header for optional server-side encryption algorithm. */
String SERVER_SIDE_ENCRYPTION = "x-amz-server-side-encryption";
- /** Range header for the get object request */
+ /** Range header for the get object request. */
String RANGE = "Range";
/**
@@ -65,10 +65,10 @@ public interface AWSHeaders {
@Deprecated
String CRYPTO_KEY = "x-amz-key";
- /** JSON-encoded description of encryption materials used during encryption */
+ /** JSON-encoded description of encryption materials used during encryption. */
String MATERIALS_DESCRIPTION = "x-amz-matdesc";
- /** Header for the optional restore information of an object */
+ /** Header for the optional restore information of an object. */
String RESTORE = "x-amz-restore";
/**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java
new file mode 100644
index 0000000000000..435db879fabf8
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.net.URI;
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.fs.PathIOException;
+
+/**
+ * An instantiation exception raised during reflection-based creation
+ * of classes.
+ * Uses an enum of kind so tests/code can examine it, without
+ * creating a full hierarchy of exception classes.
+ */
+public class InstantiationIOException extends PathIOException {
+
+ public static final String ABSTRACT_PROVIDER =
+ "is abstract and therefore cannot be created";
+
+ public static final String CONSTRUCTOR_EXCEPTION = "constructor exception";
+
+ public static final String INSTANTIATION_EXCEPTION
+ = "instantiation exception";
+
+ public static final String DOES_NOT_IMPLEMENT
+ = "does not implement";
+
+ /**
+ * Exception kind.
+ */
+ private final Kind kind;
+
+ /**
+ * Class being instantiated.
+ */
+ private final String classname;
+
+ /**
+ * key used.
+ */
+ private final String key;
+
+ /**
+ * An (extensible) enum of kinds of instantiation failure.
+ */
+ public enum Kind {
+ Forbidden,
+ InstantiationFailure,
+ IsAbstract,
+ IsNotImplementation,
+ Other,
+ Unavailable,
+ UnsupportedConstructor,
+ }
+
+ public InstantiationIOException(
+ Kind kind,
+ @Nullable URI uri, String classname,
+ @Nullable String key,
+ String message,
+ Throwable cause) {
+ super(uri!= null ? uri.toString() : "",
+ "Class " + classname + " " + message
+ + (key != null ? (" (configuration key " + key + ")") : ""),
+ cause);
+ this.kind = kind;
+ this.classname = classname;
+ this.key = key;
+ }
+
+ public String getClassname() {
+ return classname;
+ }
+
+ public Kind getKind() {
+ return kind;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ /**
+ * Class is abstract.
+ * @param uri URI of filesystem
+ * @param classname classname.
+ * @param key configuration key
+ * @return an exception.
+ */
+ public static InstantiationIOException isAbstract(URI uri, String classname, String key) {
+ return new InstantiationIOException(Kind.IsAbstract,
+ uri, classname, key, ABSTRACT_PROVIDER, null);
+ }
+
+ /**
+ * Class does not implement the desired interface.
+ * @param uri URI of filesystem
+ * @param classname classname.
+ * @param interfaceName required interface
+ * @param key configuration key
+ * @return an exception.
+ */
+ public static InstantiationIOException isNotInstanceOf(
+ @Nullable URI uri,
+ String classname,
+ String interfaceName,
+ String key) {
+ return new InstantiationIOException(Kind.IsNotImplementation, uri, classname,
+ key, DOES_NOT_IMPLEMENT + " " + interfaceName, null);
+ }
+
+ /**
+ * Class is unavailable for some reason, probably a missing dependency.
+ * @param uri URI of filesystem
+ * @param classname classname.
+ * @param key configuration key
+ * @param text text to include
+ * @return an exception.
+ */
+ public static InstantiationIOException unavailable(
+ @Nullable URI uri,
+ String classname,
+ String key,
+ String text) {
+ return new InstantiationIOException(Kind.Unavailable,
+ uri, classname, key, text, null);
+ }
+
+ /**
+ * Failure to find a valid constructor (signature, visibility) or
+ * factory method.
+ * @param uri URI of filesystem
+ * @param classname classname.
+ * @param key configuration key
+ * @return an exception.
+ */
+ public static InstantiationIOException unsupportedConstructor(
+ @Nullable URI uri,
+ String classname,
+ String key) {
+ return new InstantiationIOException(Kind.UnsupportedConstructor,
+ uri, classname, key, CONSTRUCTOR_EXCEPTION, null);
+ }
+
+ /**
+ * General instantiation failure.
+ * @param uri URI of filesystem
+ * @param classname classname.
+ * @param key configuration key
+ * @param t thrown
+ * @return an exception.
+ */
+ public static InstantiationIOException instantiationException(
+ @Nullable URI uri,
+ String classname,
+ String key,
+ Throwable t) {
+ return new InstantiationIOException(Kind.InstantiationFailure,
+ uri, classname, key, INSTANTIATION_EXCEPTION + " " + t, t);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java
index c9156f42047b2..bc9b0e49a37b0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java
@@ -23,11 +23,22 @@
import org.apache.hadoop.fs.store.LogExactlyOnce;
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_REQUEST_HANDLERS;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SDK_V2_UPGRADE_LOG_NAME;
/**
* This class provides utility methods required for migrating S3A to AWS Java SDK V2.
* For more information on the upgrade, see HADOOP-18073.
+ *
+ * in HADOOP-18382. Upgrade AWS SDK to V2 - Prerequisites,
+ * this class contained a series of `LogExactlyOnce` loggers to warn on
+ * the first use of a feature which would change incompatibly; this shipped in Hadoop 3.3.5.
+ *
+ * With the move to v2 completed, attempts to use the v1 classes, will fail
+ * -except for the special case of support for v1 credential providers.
+ *
+ * The warning methods are still present, where appropriate, but downgraded to debug
+ * and only retained for debugging migration issues.
*/
public final class V2Migration {
@@ -35,76 +46,17 @@ private V2Migration() { }
public static final Logger SDK_V2_UPGRADE_LOG = LoggerFactory.getLogger(SDK_V2_UPGRADE_LOG_NAME);
- private static final LogExactlyOnce WARN_ON_DELEGATION_TOKENS =
- new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
-
- private static final LogExactlyOnce WARN_ON_GET_S3_CLIENT =
- new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
-
- private static final LogExactlyOnce WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER =
- new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
-
- private static final LogExactlyOnce WARN_OF_CUSTOM_SIGNER =
- new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
-
private static final LogExactlyOnce WARN_OF_REQUEST_HANDLERS =
new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
- private static final LogExactlyOnce WARN_ON_GET_OBJECT_METADATA =
- new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
-
- /**
- * Warns on an AWS V1 credential provider being referenced directly.
- * @param name name of the credential provider
- */
- public static void v1ProviderReferenced(String name) {
- WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER.warn(
- "Directly referencing AWS SDK V1 credential provider {}. AWS SDK V1 credential "
- + "providers will be removed once S3A is upgraded to SDK V2", name);
- }
-
/**
- * Warns on the v1 s3 client being requested.
+ * Notes use of request handlers.
+ * @param handlers handlers declared
*/
- public static void v1S3ClientRequested() {
- WARN_ON_GET_S3_CLIENT.warn(
- "getAmazonS3ClientForTesting() will be removed as part of upgrading S3A to AWS SDK V2");
- }
-
- /**
- * Warns when v1 credential providers are used with delegation tokens.
- */
- public static void v1DelegationTokenCredentialProvidersUsed() {
- WARN_ON_DELEGATION_TOKENS.warn(
- "The credential provider interface has changed in AWS SDK V2, custom credential "
- + "providers used in delegation tokens binding classes will need to be updated once "
- + "S3A is upgraded to SDK V2");
- }
-
- /**
- * Warns on use of custom signers.
- */
- public static void v1CustomSignerUsed() {
- WARN_OF_CUSTOM_SIGNER.warn(
- "The signer interface has changed in AWS SDK V2, custom signers will need to be updated "
- + "once S3A is upgraded to SDK V2");
- }
-
- /**
- * Warns on use of request handlers.
- */
- public static void v1RequestHandlersUsed() {
+ public static void v1RequestHandlersUsed(final String handlers) {
WARN_OF_REQUEST_HANDLERS.warn(
- "The request handler interface has changed in AWS SDK V2, use exception interceptors "
- + "once S3A is upgraded to SDK V2");
- }
-
- /**
- * Warns on use of getObjectMetadata.
- */
- public static void v1GetObjectMetadataCalled() {
- WARN_ON_GET_OBJECT_METADATA.warn("getObjectMetadata() called. This operation and it's response "
- + "will be changed as part of upgrading S3A to AWS SDK V2");
+ "Ignoring V1 SDK request handlers set in {}: {}",
+ AUDIT_REQUEST_HANDLERS, handlers);
}
}
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md
index 094ea5668c05e..ea53b2e1fa9e3 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md
@@ -195,7 +195,7 @@ Here are the full set of configuration options.
fs.s3a.assumed.role.credentials.provider
org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
- com.amazonaws.auth.EnvironmentVariableCredentialsProvider
+ software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider
List of credential providers to authenticate with the STS endpoint and
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md
index 7a95907217789..dbb9c0b3d97c9 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md
@@ -22,7 +22,7 @@ and inside the AWS S3 SDK, immediately before the request is executed.
The full architecture is covered in [Auditing Architecture](auditing_architecture.html);
this document covers its use.
-## Important: Auditing is disabled by default
+## Important: Auditing is currently enabled
Due to a memory leak from the use of `ThreadLocal` fields, this auditing feature
leaked memory as S3A filesystem instances were created and deleted.
@@ -32,7 +32,7 @@ See [HADOOP-18091](https://issues.apache.org/jira/browse/HADOOP-18091) _S3A audi
To avoid these memory leaks, auditing was disabled by default in the hadoop 3.3.2 release.
-As these memory leaks have now been fixed, auditing has been re-enabled.
+As these memory leaks have now been fixed, auditing has been re-enabled in Hadoop 3.3.5+
To disable it, set `fs.s3a.audit.enabled` to `false`.
@@ -77,7 +77,7 @@ ideally even identifying the process/job generating load.
## Using Auditing
-Auditing is disabled by default.
+Auditing is enabled by default.
When auditing enabled, a Logging Auditor will annotate the S3 logs through a custom
HTTP Referrer header in requests made to S3.
Other auditor classes may be used instead.
@@ -88,7 +88,7 @@ Other auditor classes may be used instead.
|--------|---------|---------------|
| `fs.s3a.audit.enabled` | Is auditing enabled? | `true` |
| `fs.s3a.audit.service.classname` | Auditor classname | `org.apache.hadoop.fs.s3a.audit.impl.LoggingAuditor` |
-| `fs.s3a.audit.request.handlers` | List of extra subclasses of AWS SDK RequestHandler2 to include in handler chain | `""` |
+| `fs.s3a.audit.execution.interceptors` | Implementations of AWS v2 SDK `ExecutionInterceptor` to include in handler chain | `""` |
| `fs.s3a.audit.referrer.enabled` | Logging auditor to publish the audit information in the HTTP Referrer header | `true` |
| `fs.s3a.audit.referrer.filter` | List of audit fields to filter | `""` |
| `fs.s3a.audit.reject.out.of.span.operations` | Auditor to reject operations "outside of a span" | `false` |
@@ -96,14 +96,14 @@ Other auditor classes may be used instead.
### Disabling Auditing.
-In this release of Hadoop, auditing is disabled.
+In this release of Hadoop, auditing is enabled by default.
This can be explicitly set globally or for specific buckets
```xml
fs.s3a.audit.enabled
- false
+ true
```
@@ -162,6 +162,26 @@ correlate access by S3 clients to the actual operations taking place.
Note: this logging is described as "Best Effort". There's no guarantee as to
when logs arrive.
+### Integration with AWS SDK request processing
+
+The auditing component inserts itself into the AWS SDK request processing
+code, so it can attach the referrer header.
+
+It is possible to declare extra classes to add to the processing chain,
+all of which must implement the interface `software.amazon.awssdk.core.interceptor.ExecutionInterceptor`.
+
+The list of classes is set in the configuration option `fs.s3a.audit.execution.interceptors`.
+
+Any class in the list which implements `org.apache.hadoop.conf.Configurable` will have
+`Configurable.setConf()` called with the filesystem configuration passed down.
+
+Before the upgrade to the V2 SDK, a list of extra subclasses of the AWS SDK `com.amazonaws.handlers.RequestHandler2`
+class could be declared in the option `fs.s3a.audit.request.handlers`;
+these would be wired up into the V1 request processing pipeline.
+
+This option is now ignored completely, other than printing a warning message the first time a filesystem is created with a non-empty value.
+
+
### Rejecting out-of-span operations
The logging auditor can be configured to raise an exception whenever
@@ -201,7 +221,7 @@ The HTTP referrer header is attached by the logging auditor.
If the S3 Bucket is configured to log requests to another bucket, then these logs
entries will include the audit information _as the referrer_.
-This can be parsed (consult AWS documentation for a regular expression)
+The S3 Server log entries can be parsed (consult AWS documentation for a regular expression)
and the http referrer header extracted.
```
@@ -242,13 +262,15 @@ If any of the field values were `null`, the field is omitted.
_Notes_
-* Thread IDs are from the current thread in the JVM, so can be compared to those in`````````
+* Thread IDs are from the current thread in the JVM, so can be compared to those in
Log4J logs. They are never unique.
* Task Attempt/Job IDs are only ever set during operations involving the S3A committers, specifically
- all operations excecuted by the committer.
+ all operations executed by the committer.
Operations executed in the same thread as the committer's instantiation _may_ also report the
IDs, even if they are unrelated to the actual task. Consider them "best effort".
+Thread IDs are generated as follows:
+
```java
Long.toString(Thread.currentThread().getId())
```
@@ -269,6 +291,8 @@ This is why the span ID is always passed in as part of the URL,
rather than just an HTTP query parameter: even if
the header is chopped, the span ID will always be present.
+As of August 2023, this header is not collected in AWS CloudTrail -only S3 Server logs.
+
## Privacy Implications of HTTP Referrer auditing
When the S3A client makes requests of an S3 bucket, the auditor
@@ -423,6 +447,12 @@ log4j.logger.org.apache.hadoop.fs.s3a.audit=TRACE
This is very noisy and not recommended in normal operation.
+If logging of HTTP IO is enabled then the "referer" header is printed as part of every request:
+```
+log4j.logger.org.apache.http=DEBUG
+log4j.logger.software.amazon.awssdk.thirdparty.org.apache.http.client.HttpClient=DEBUG
+```
+
## Integration with S3A Committers
Work submitted through the S3A committer will have the job (query) ID associated
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md
index e649a8d76d539..9ac2a780ca233 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md
@@ -20,28 +20,42 @@ This work is tracked in [HADOOP-18073](https://issues.apache.org/jira/browse/HAD
## Why the upgrade?
- Moving to SDK V2 will provide performance benefits.
-For example, the [transfer manager for SDKV2](https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/)
+For example, the [transfer manager for SDK V2](https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/)
is built using java bindings of the AWS Common Runtime S3
client (https://github.com/awslabs/aws-crt-java) (CRT).
CRT is a set of packages written in C, designed for maximising performance when interacting with AWS
services such as S3.
+- The V1 SDK is essentially in maintenance mode.
- New features such as [additional checksum algorithms](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/)
-which S3A will benefit from are not available in SDKV1.
+which S3A will benefit from are not available in SDK V1.
## What's changing?
The [SDK V2](https://github.com/aws/aws-sdk-java-v2) for S3 is very different from
[SDK V1](https://github.com/aws/aws-sdk-java), and brings breaking changes for S3A.
-A complete list of the changes can be found in the [Changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/docs/LaunchChangelog.md#41-s3-changes).
+A complete list of the changes can be found in the
+[Changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/docs/LaunchChangelog.md#41-s3-changes).
-The major changes and how this affects S3A are listed below.
+## Packaging: `aws-java-sdk-bundle-1.12.x.jar` becomes `bundle-2.x.y.jar`
-### Package Change
+As the module name is lost, in hadoop releases a large JAR file with
+the name "bundle" is now part of the distribution.
+This is the AWS V2 SDK shaded artifact.
-Package names have changed, all classes in SDK V2 are under `software.amazon.awssdk`, SDK V1 classes
-were under `com.amazonaws`.
+The new and old SDKs can co-exist; the only place that the hadoop code
+may still use the original SDK is when a non-standard V1 AWS credential
+provider is declared.
+
+Any deployment of the S3A connector must include this JAR or
+the subset of non-shaded aws- JARs needed for communication
+with S3 and any other services used.
+As before: the exact set of dependencies used by the S3A connector
+is neither defined nor comes with any commitments of stability
+or compatibility of dependent libraries.
-### Credential Providers
+
+
+## Credential Provider changes and migration
- Interface change: [com.amazonaws.auth.AWSCredentialsProvider](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/AWSCredentialsProvider.java)
has been replaced by [software.amazon.awssdk.auth.credentials.AwsCredentialsProvider](https://github.com/aws/aws-sdk-java-v2/blob/master/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.java).
@@ -49,23 +63,296 @@ has been replaced by [software.amazon.awssdk.auth.credentials.AwsCredentialsProv
changed.
The change in interface will mean that custom credential providers will need to be updated to now
-implement `AwsCredentialsProvider` instead of `AWSCredentialProvider`.
+implement `software.amazon.awssdk.auth.credentials.AwsCredentialsProvider` instead of
+`com.amazonaws.auth.AWSCredentialsProvider`.
-Due to change in class names, references to SDK V1 credential providers
-in `fs.s3a.aws.credentials.provider` will need to be updated to reference V2 providers.
+### Original V1 `AWSCredentialsProvider` interface
-### Delegation Tokens
+Note how the interface begins with the capitalized "AWS" acronym.
+The V2 interface starts with "Aws". This is a very subtle change
+for developers to spot.
+Compilers _will_ detect and report the type mismatch.
+
+
+```java
+package com.amazonaws.auth;
+
+public interface AWSCredentialsProvider {
+
+ public AWSCredentials getCredentials();
+
+ public void refresh();
+
+}
+
+```
+The interface binding also supported a factory method, `AWSCredentialsProvider instance()` which,
+if available, would be invoked in preference to using any constructor.
+
+If the interface implemented `Closeable` or `AutoCloseable`, these would
+be invoked when the provider chain was being shut down.
+
+### New V2 `AwsCredentialsProvider` interface
+
+```java
+package software.amazon.awssdk.auth.credentials;
+
+public interface AwsCredentialsProvider {
+
+ AwsCredentials resolveCredentials();
+
+}
+```
+
+1. There is no `refresh()` method any more.
+2. `getCredentials()` has become `resolveCredentials()`.
+3. There is now the expectation in the SDK that credential resolution/lookup etc will be
+ performed in `resolveCredentials()`.
+4. If the interface implements `Closeable` or `AutoCloseable`, these will
+ be invoked when the provider chain is being shut down.
+5. A static method `create()` which returns an `AwsCredentialsProvider` or subclass; this will be used
+ in preference to a constructor
+
+### S3A `AWSCredentialProviderList` is now a V2 credential provider
+
+The class `org.apache.hadoop.fs.s3a.AWSCredentialProviderList` has moved from
+being a V1 to a V2 credential provider; even if an instance can be created with
+existing code, the V1 methods will not resolve:
+
+```
+java.lang.NoSuchMethodError: org.apache.hadoop.fs.s3a.AWSCredentialProviderList.getCredentials()Lcom/amazonaws/auth/AWSCredentials;
+ at org.apache.hadoop.fs.store.diag.S3ADiagnosticsInfo.validateFilesystem(S3ADiagnosticsInfo.java:903)
+```
+
+### Migration of Credential Providers listed in `fs.s3a.aws.credentials.provider`
+
+
+Before: `fs.s3a.aws.credentials.provider` took a list of v1 credential providers,
+This took a list containing
+1. V1 credential providers implemented in the `hadoop-aws` module.
+2. V1 credential providers implemented in the `aws-sdk-bundle` library.
+3. Custom V1 credential providers placed onto the classpath.
+4. Custom subclasses of hadoop-aws credential providers.
+
+And here is how they change
+1. All `hadoop-aws` credential providers migrated to V2.
+2. Well-known `aws-sdk-bundle` credential providers _automatically remapped_ to their V2 equivalents.
+3. Custom v1 providers supported if the original `aws-sdk-bundle` JAR is on the classpath.
+4. Custom subclasses of hadoop-aws credential providers need manual migration.
+
+Because of (1) and (2), As result, standard `fs.s3a.aws.credentials.provider` configurations
+should seamlessly upgrade. This also means that the same provider list, if restricted to
+those classes, will work across versions.
+
+
+
+### `hadoop-aws` credential providers migration to V2
+
+All the fs.s3a credential providers have the same name and functionality as before.
+
+| Hadoop module credential provider | Authentication Mechanism |
+|----------------------------------------------------------------|--------------------------------------------------|
+| `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider` | Session Credentials in configuration |
+| `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider` | Simple name/secret credentials in configuration |
+| `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider` | Anonymous Login |
+| `org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider` | [Assumed Role credentials](./assumed_roles.html) |
+| `org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` | EC2/k8s instance credentials |
+
+### Automatic `aws-sdk-bundle` credential provider remapping
+
+The commonly-used set of V1 credential providers are automatically remapped to V2 equivalents.
+
+
+
+| V1 Credential Provider | Remapped V2 substitute |
+|-------------------------------------------------------------|----------------------------------------------------------------------------------|
+| `com.amazonaws.auth.AnonymousAWSCredentials` | `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider` |
+| `com.amazonaws.auth.EnvironmentVariableCredentialsProvider` | `software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider` |
+| `com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper` | `org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` |
+| `com.amazonaws.auth.InstanceProfileCredentialsProvider` | `org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` |
+| `com.amazonaws.auth.profile.ProfileCredentialsProvider` | `software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider` |
+
+There are still a number of troublespots here:
+
+#### Less widely used`com.amazonaws.auth.` AWS providers
+
+There should be equivalents in the new SDK, but as well as being renamed
+they are likely to have moved different factory/builder mechanisms.
+Identify the changed classes and use their
+names in the `fs.s3a.aws.credentials.provider` option.
+
+If a V2 equivalent is not found; provided the V1 SDK is added to the classpath,
+it should still be possible to use the existing classes.
+
+
+#### Private/third-party credential providers
+
+Provided the V1 SDK is added to the classpath,
+it should still be possible to use the existing classes.
+
+Adding a V2 equivalent is the recommended long-term solution.
+
+#### Custom subclasses of the Hadoop credential providers
-Custom credential providers used in delegation token binding classes will also need to be updated.
+Because all the standard hadoop credential providers have been upgraded,
+any subclasses of these are not going to link or work.
-### AmazonS3 replaced by S3Client
+These will need to be manually migrated to being V2 Credential providers.
-The s3 client is an instance of `S3Client` in V2 rather than `AmazonS3`.
-For this reason, the `S3ClientFactory` will be deprecated and replaced by one that creates a V2
-`S3Client`.
+## Source code/binary integration changes
+
+The major changes and how this affects S3A are listed below.
+
+### SDK API Package Change
+
+* Package names have changed, all classes in SDK V2 are under `software.amazon.awssdk`, SDK V1 classes
+were under `com.amazonaws`.
+* There is no interoperability between the old and new classes.
+* All classnames are different, often in very subtle ways. It is possible to use both in the same
+ class, as is done in the package `org.apache.hadoop.fs.s3a.adapter`.
+* All the core message classes are now automatically generated from a JSON protocol description.
+* All getter methods have been renamed.
+* All classes are constructed via builder methods
+* Message classes are no longer Java `Serializable`.
+
+Most of these changes simply create what will feel to be gratuitous migration effort;
+the removable of the `Serializable` nature from all message response classes can
+potentially break applications -such as anything passing them between Spark workers.
+See AWS SDK V2 issue [Simplify Modeled Message Marshalling #82](https://github.com/aws/aws-sdk-java-v2/issues/82),
+note that it was filed in 2017, then implement your own workaround pending that issue
+being resolved.
+
+### Compilation/Linkage Errors
+
+Any code making use of V1 sdk classes will fail if they
+* Expect the V1 sdk classes to be on the classpath when `hadoop-aws` is declared as a dependency
+* Use V1-SDK-compatible methods previously exported by the `S3AFileSystem` class and associated classes.
+* Try to pass s3a classes to V1 SDK classes (e.g. credential providers).
+
+The sole solution to these problems is "move to the V2 SDK".
+
+Some `S3AUtils` methods are deleted
+```
+cannot find symbol
+[ERROR] symbol: method createAwsConf(org.apache.hadoop.conf.Configuration,java.lang.String)
+[ERROR] location: class org.apache.hadoop.fs.s3a.S3AUtils
+```
+
+The signature and superclass of `AWSCredentialProviderList` has changed, which can surface in different
+ways
+
+Signature mismatch
+```
+ cannot find symbol
+[ERROR] symbol: method getCredentials()
+[ERROR] location: variable credentials of type org.apache.hadoop.fs.s3a.AWSCredentialProviderList
+```
+
+It is no longer a V1 credential provider, cannot be used to pass credentials to a V1 SDK class
+```
+incompatible types: org.apache.hadoop.fs.s3a.AWSCredentialProviderList cannot be converted to com.amazonaws.auth.AWSCredentialsProvider
+```
+
+### `AmazonS3` replaced by `S3Client`; factory and accessor changed.
+
+The V1 s3 client class `com.amazonaws.services.s3.AmazonS3` has been superseded by
+`software.amazon.awssdk.services.s3.S3Client`
+
+The `S3ClientFactory` interface has been replaced by one that creates a V2 `S3Client`.
+* Custom implementations will need to be updated.
+* The `InconsistentS3ClientFactory` class has been deleted.
+
+#### `S3AFileSystem` method changes: `S3AInternals`.
+
+The low-level s3 operations/client accessors have been moved into a new interface,
+`org.apache.hadoop.fs.s3a.S3AInternals`, which must be accessed via the
+`S3AFileSystem.getS3AInternals()` method.
+They have also been updated to return V2 SDK classes.
+
+```java
+@InterfaceStability.Unstable
+@InterfaceAudience.LimitedPrivate("testing/diagnostics")
+public interface S3AInternals {
+ S3Client getAmazonS3V2ClientForTesting(String reason);
+
+ @Retries.RetryTranslated
+ @AuditEntryPoint
+ String getBucketLocation() throws IOException;
+
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ String getBucketLocation(String bucketName) throws IOException;
+
+ @AuditEntryPoint
+ @Retries.RetryTranslated
+ HeadObjectResponse getObjectMetadata(Path path) throws IOException;
+
+ AWSCredentialProviderList shareCredentials(final String purpose);
+}
+```
+
+
+##### `S3AFileSystem.getAmazonS3ClientForTesting(String)` moved and return type changed
+
+The `S3AFileSystem.getAmazonS3ClientForTesting()` method has been been deleted.
+
+Compilation
+```
+cannot find symbol
+[ERROR] symbol: method getAmazonS3ClientForTesting(java.lang.String)
+[ERROR] location: variable fs of type org.apache.hadoop.fs.s3a.S3AFileSystem
+```
+
+It has been replaced by an `S3AInternals` equivalent which returns the V2 `S3Client`
+of the filesystem instance.
+
+```java
+((S3AFilesystem)fs).getAmazonS3ClientForTesting("testing")
+```
+
+```java
+((S3AFilesystem)fs).getS3AInternals().getAmazonS3ClientForTesting("testing")
+```
+
+##### `S3AFileSystem.getObjectMetadata(Path path)` moved to `S3AInternals`; return type changed
+
+The `getObjectMetadata(Path)` call has been moved to the `S3AInternals` interface
+and an instance of the `software.amazon.awssdk.services.s3.model.HeadObjectResponse` class
+returned.
+The original `S3AFileSystem` method has been deleted
+
+Before:
+```java
+((S3AFilesystem)fs).getObjectMetadata(path)
+```
+
+After:
+```java
+((S3AFilesystem)fs).getS3AInternals().getObjectMetadata(path)
+```
+
+##### `AWSCredentialProviderList shareCredentials(String)` moved to `S3AInternals`
+
+The operation to share a reference-counted access to the AWS credentials used
+by the S3A FS has been moved to `S3AInternals`.
+
+This is very much an implementation method, used to allow extension modules to share
+an authentication chain into other AWS SDK client services (dynamoDB, etc.).
+
+### Delegation Tokens
+
+1. Custom credential providers used in delegation token binding classes will need to be updated
+2. The return type from delegation token binding has changed to support more class
+ instances being returned in the future.
+
+`AWSCredentialProviderList` has been upgraded to the V2 API.
+* It still retains a `refresh()` method but this is now a deprecated no-op.
+* It is still `Closeable`; its `close()` method iterates through all entries in
+the list; if they are `Closeable` or `AutoCloseable` then their `close()` method is invoked.
+* Accordingly, providers may still perform background refreshes in separate threads;
+ the S3A client will close its provider list when the filesystem itself is closed.
-The `getAmazonS3ClientForTesting()` method will also be updated to return the `S3Client`.
### Signers
@@ -74,3 +361,21 @@ has been replaced by [software.amazon.awssdk.core.signer.Signer](https://github.
The change in signers will mean the custom signers will need to be updated to implement the new
interface.
+
+There is no support to assist in this migration.
+
+### S3A Auditing Extensions.
+
+The callbacks from the SDK have all changed, as has
+the interface `org.apache.hadoop.fs.s3a.audit.AWSAuditEventCallbacks`
+
+Examine the interface and associated implementations to
+see how to migrate.
+
+The option `fs.s3a.audit.request.handlers` to declare a list of v1 SDK
+`com.amazonaws.handlers.RequestHandler2` implementations to include
+in the AWS request chain is no longer supported: a warning is printed
+and the value ignored.
+
+The V2 SDK equivalent, classes implementing `software.amazon.awssdk.core.interceptor.ExecutionInterceptor`
+can be declared in the configuration option `fs.s3a.audit.execution.interceptors`.
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md
index 7aaa1b8b5ce79..6ed6365a9a9ae 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md
@@ -339,7 +339,7 @@ Here is the effective list of providers if none are declared:
org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,
org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
- com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
+ software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider,
org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 5d41f8c742747..4e5ead1be171c 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -249,56 +249,39 @@ a warning has been printed since Hadoop 2.8 whenever such a URL was used.
```xml
fs.s3a.access.key
- AWS access key ID.
- Omit for IAM role-based or provider-based authentication.
+ AWS access key ID used by S3A file system. Omit for IAM role-based or provider-based authentication.
fs.s3a.secret.key
- AWS secret key.
- Omit for IAM role-based or provider-based authentication.
+ AWS secret key used by S3A file system. Omit for IAM role-based or provider-based authentication.
- fs.s3a.aws.credentials.provider
-
- Comma-separated class names of credential provider classes which implement
- com.amazonaws.auth.AWSCredentialsProvider.
-
- These are loaded and queried in sequence for a valid set of credentials.
- Each listed class must implement one of the following means of
- construction, which are attempted in order:
- 1. a public constructor accepting java.net.URI and
- org.apache.hadoop.conf.Configuration,
- 2. a public static method named getInstance that accepts no
- arguments and returns an instance of
- com.amazonaws.auth.AWSCredentialsProvider, or
- 3. a public default constructor.
-
- Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
- anonymous access to a publicly accessible S3 bucket without any credentials.
- Please note that allowing anonymous access to an S3 bucket compromises
- security and therefore is unsuitable for most use cases. It can be useful
- for accessing public data sets without requiring AWS credentials.
-
- If unspecified, then the default list of credential provider classes,
- queried in sequence, is:
- 1. org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider:
- Uses the values of fs.s3a.access.key and fs.s3a.secret.key.
- 2. com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports
- configuration of AWS access key ID and secret access key in
- environment variables named AWS_ACCESS_KEY_ID and
- AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
- 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
- of instance profile credentials if running in an EC2 VM.
+ fs.s3a.session.token
+ Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
+ as one of the providers.
- fs.s3a.session.token
+ fs.s3a.aws.credentials.provider
+
+ org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,
+ org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
+ software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider,
+ org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider
+
- Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
- as one of the providers.
+ Comma-separated class names of credential provider classes which implement
+ software.amazon.awssdk.auth.credentials.AwsCredentialsProvider.
+
+ When S3A delegation tokens are not enabled, this list will be used
+ to directly authenticate with S3 and other AWS services.
+ When S3A Delegation tokens are enabled, depending upon the delegation
+ token binding it may be used
+ to communicate wih the STS endpoint to request session/role
+ credentials.
```
@@ -350,13 +333,19 @@ credentials if they are defined.
1. The [AWS environment variables](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment),
are then looked for: these will return session or full credentials depending
on which values are set.
-1. An attempt is made to query the Amazon EC2 Instance Metadata Service to
+1. An attempt is made to query the Amazon EC2 Instance/k8s container Metadata Service to
retrieve credentials published to EC2 VMs.
S3A can be configured to obtain client authentication providers from classes
-which integrate with the AWS SDK by implementing the `com.amazonaws.auth.AWSCredentialsProvider`
-Interface. This is done by listing the implementation classes, in order of
+which integrate with the AWS SDK by implementing the
+`software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`
+interface.
+This is done by listing the implementation classes, in order of
preference, in the configuration option `fs.s3a.aws.credentials.provider`.
+In previous hadoop releases, providers were required to
+implement the AWS V1 SDK interface `com.amazonaws.auth.AWSCredentialsProvider`.
+Consult the [Upgrading S3A to AWS SDK V2](./aws_sdk_upgrade.html) documentation
+to see how to migrate credential providers.
*Important*: AWS Credential Providers are distinct from _Hadoop Credential Providers_.
As will be covered later, Hadoop Credential Providers allow passwords and other secrets
@@ -371,21 +360,23 @@ this is advised as a more secure way to store valuable secrets.
There are a number of AWS Credential Providers inside the `hadoop-aws` JAR:
-| classname | description |
-|-----------|-------------|
-| `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider`| Session Credentials |
-| `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider`| Simple name/secret credentials |
-| `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider`| Anonymous Login |
-| `org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider<`| [Assumed Role credentials](assumed_roles.html) |
+| Hadoop module credential provider | Authentication Mechanism |
+|----------------------------------------------------------------|--------------------------------------------------|
+| `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider` | Session Credentials in configuration |
+| `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider` | Simple name/secret credentials in configuration |
+| `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider` | Anonymous Login |
+| `org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider` | [Assumed Role credentials](./assumed_roles.html) |
+| `org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` | EC2/k8s instance credentials |
-There are also many in the Amazon SDKs, in particular two which are automatically
-set up in the authentication chain:
+There are also many in the Amazon SDKs, with the common ones being.
| classname | description |
|-----------|-------------|
-| `com.amazonaws.auth.InstanceProfileCredentialsProvider`| EC2 Metadata Credentials |
-| `com.amazonaws.auth.EnvironmentVariableCredentialsProvider`| AWS Environment Variables |
+| `software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider` | AWS Environment Variables |
+| `software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider`| EC2 Metadata Credentials |
+| `software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider`| EC2/k8s Metadata Credentials |
+
### EC2 IAM Metadata Authentication with `InstanceProfileCredentialsProvider`
@@ -402,7 +393,7 @@ You can configure Hadoop to authenticate to AWS using a [named profile](https://
To authenticate with a named profile:
-1. Declare `com.amazonaws.auth.profile.ProfileCredentialsProvider` as the provider.
+1. Declare `software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider` as the provider.
1. Set your profile via the `AWS_PROFILE` environment variable.
1. Due to a [bug in version 1 of the AWS Java SDK](https://github.com/aws/aws-sdk-java/issues/803),
you'll need to remove the `profile` prefix from the AWS configuration section heading.
@@ -525,50 +516,9 @@ This means that the default S3A authentication chain can be defined as
org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,
org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
- com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
+ software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider
org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider
-
- Comma-separated class names of credential provider classes which implement
- com.amazonaws.auth.AWSCredentialsProvider.
-
- When S3A delegation tokens are not enabled, this list will be used
- to directly authenticate with S3 and other AWS services.
- When S3A Delegation tokens are enabled, depending upon the delegation
- token binding it may be used
- to communicate with the STS endpoint to request session/role
- credentials.
-
- These are loaded and queried in sequence for a valid set of credentials.
- Each listed class must implement one of the following means of
- construction, which are attempted in order:
- * a public constructor accepting java.net.URI and
- org.apache.hadoop.conf.Configuration,
- * a public constructor accepting org.apache.hadoop.conf.Configuration,
- * a public static method named getInstance that accepts no
- arguments and returns an instance of
- com.amazonaws.auth.AWSCredentialsProvider, or
- * a public default constructor.
-
- Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
- anonymous access to a publicly accessible S3 bucket without any credentials.
- Please note that allowing anonymous access to an S3 bucket compromises
- security and therefore is unsuitable for most use cases. It can be useful
- for accessing public data sets without requiring AWS credentials.
-
- If unspecified, then the default list of credential provider classes,
- queried in sequence, is:
- * org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider: looks
- for session login secrets in the Hadoop configuration.
- * org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider:
- Uses the values of fs.s3a.access.key and fs.s3a.secret.key.
- * com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports
- configuration of AWS access key ID and secret access key in
- environment variables named AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
- and AWS_SESSION_TOKEN as documented in the AWS SDK.
- * org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider: picks up
- IAM credentials of any EC2 VM or AWS container in which the process is running.
-
```
@@ -1414,7 +1364,7 @@ role information available when deployed in Amazon EC2.
```xml
fs.s3a.aws.credentials.provider
- com.amazonaws.auth.InstanceProfileCredentialsProvider
+ org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider
```
@@ -2136,7 +2086,7 @@ If no custom signers are being used - this value does not need to be set.
`SignerName:SignerClassName` - register a new signer with the specified name,
and the class for this signer.
-The Signer Class must implement `com.amazonaws.auth.Signer`.
+The Signer Class must implement `software.amazon.awssdk.core.signer.Signer`.
`SignerName:SignerClassName:SignerInitializerClassName` - similar time above
except also allows for a custom SignerInitializer
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md
index 649c80a22d4f7..1361b8d9f63f3 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md
@@ -14,7 +14,7 @@
# S3 Select
-**Experimental Feature**
+**Deprecated Feature**
@@ -60,6 +60,20 @@ Record Readers.
It's better here to directly use the Apache Spark, Hive, Impala, Flink or
similar, which all use the latest ASF-supported libraries.
+## Dependencies: eventstream JAR
+
+To use S3 Select through the S3A connector, an extra JAR MUST be added to the classpath of your application,
+`eventstream-1.0.1.jar`.a
+For command line tool use, this should be done by adding it to `share/hadoop/common/lib/`
+
+```xml
+
+ software.amazon.eventstream
+ eventstream
+ 1.0.1
+
+```
+
## Enabling/Disabling S3 Select
S3 Select is enabled by default:
@@ -288,10 +302,12 @@ hadoop s3guard \
```
-## Use in MR/Analytics queries: Work in Progress
+## Use in MR/Analytics queries: Partially Supported
-S3 Select support in analytics queries is a work in progress. It does
-not work reliably with large source files where the work is split up.
+S3 Select support in analytics queries is only partially supported.
+It does not work reliably with large source files where the work is split up,
+and as the various query engines all assume that .csv and .json formats are splittable,
+things go very wrong, fast.
As a proof of concept *only*, S3 Select queries can be made through
MapReduce jobs which use any Hadoop `RecordReader`
@@ -663,6 +679,24 @@ to the `get()` call: do it.
## Troubleshooting
+### `NoClassDefFoundError: software/amazon/eventstream/MessageDecoder`
+
+Select operation failing with a missing eventstream class.
+
+```
+java.io.IOException: java.lang.NoClassDefFoundError: software/amazon/eventstream/MessageDecoder
+at org.apache.hadoop.fs.s3a.select.SelectObjectContentHelper.select(SelectObjectContentHelper.java:75)
+at org.apache.hadoop.fs.s3a.WriteOperationHelper.lambda$select$10(WriteOperationHelper.java:660)
+at org.apache.hadoop.fs.store.audit.AuditingFunctions.lambda$withinAuditSpan$0(AuditingFunctions.java:62)
+at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:122)
+```
+
+The eventstream JAR is not on the classpath/not in sync with the version of the full "bundle.jar" JDK
+
+Fix: get a compatible version of the JAR on the classpath.
+
+### SQL errors
+
Getting S3 Select code to work is hard, though those knowledgeable in SQL
will find it easier.
@@ -673,7 +707,6 @@ Problems can be split into:
1. Datatype casting issues
1. Bad records/data in source files.
1. Failure to configure MR jobs to work correctly.
-1. Failure of MR jobs due to
The exceptions here are all based on the experience during writing tests;
more may surface with broader use.
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
index 846c4a3ef2bf7..b3863879a320c 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
@@ -1002,9 +1002,6 @@ using an absolute XInclude reference to it.
## Failure Injection
-**Warning do not enable any type of failure injection in production. The
-following settings are for testing only.**
-
S3A provides an "Inconsistent S3 Client Factory" that can be used to
simulate throttling by injecting random failures on S3 client requests.
@@ -1016,55 +1013,8 @@ inconsistencies during testing of S3Guard. Now that S3 is consistent,
injecting inconsistency is no longer needed during testing.
-### Enabling the InconsistentS3CClientFactory
-
-To enable the fault-injecting client via configuration, switch the
-S3A client to use the "Inconsistent S3 Client Factory" when connecting to
-S3:
-
-```xml
-
- fs.s3a.s3.client.factory.impl
- org.apache.hadoop.fs.s3a.InconsistentS3ClientFactory
-
-```
-
-The inconsistent client will, on every AWS SDK request,
-generate a random number, and if less than the probability,
-raise a 503 exception.
-
-```xml
-
-
- fs.s3a.failinject.throttle.probability
- 0.05
-
-```
-
-These exceptions are returned to S3; they do not test the
-AWS SDK retry logic.
-
-
-### Using the `InconsistentS3CClientFactory` in downstream integration tests
-
-The inconsistent client is shipped in the `hadoop-aws` JAR, so it can
-be used in integration tests.
-
-## Testing S3Guard
-
-As part of the removal of S3Guard from the production code, the tests have been updated
-so that
-
-* All S3Guard-specific tests have been deleted.
-* All tests parameterized on S3Guard settings have had those test configurations removed.
-* The maven profiles option to run tests with S3Guard have been removed.
-
-There is no need to test S3Guard -and so tests are lot faster.
-(We developers are all happy)
-
-
-## Testing Assumed Roles
+## Testing Assumed Roles
Tests for the AWS Assumed Role credential provider require an assumed
role to request.
@@ -1285,10 +1235,13 @@ time bin/hadoop fs -copyToLocal -t 10 $BUCKET/\*aws\* tmp
# ---------------------------------------------------
# S3 Select on Landsat
+# this will fail with a ClassNotFoundException unless
+# eventstore JAR is added to the classpath
# ---------------------------------------------------
export LANDSATGZ=s3a://landsat-pds/scene_list.gz
+
bin/hadoop s3guard select -header use -compression gzip $LANDSATGZ \
"SELECT s.entityId,s.cloudCover FROM S3OBJECT s WHERE s.cloudCover < '0.0' LIMIT 100"
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
index 78c92c43cc7ea..68d40e56b8c53 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
@@ -70,14 +70,45 @@ These are Hadoop filesystem client classes, found in the `hadoop-aws` JAR.
An exception reporting this class as missing means that this JAR is not on
the classpath.
-### `ClassNotFoundException: com.amazonaws.services.s3.AmazonS3Client`
-(or other `com.amazonaws` class.)
+### `NoClassDefFoundError: software/amazon/awssdk/crt/s3/S3MetaRequest`
+
+The library `aws-crt.jar` is not on the classpath. Its classes
+are not in the AWS `bundle.jar` file, yet may be needed by some uses made
+of the SDK.
+
+Fix: add.
+
+```
+java.lang.BootstrapMethodError: java.lang.NoClassDefFoundError: software/amazon/awssdk/crt/s3/S3MetaRequest
+at software.amazon.awssdk.services.s3.internal.crt.S3MetaRequestPauseObservable.(S3MetaRequestPauseObservable.java:33)
+at software.amazon.awssdk.transfer.s3.internal.DefaultS3TransferManager.uploadFile(DefaultS3TransferManager.java:205)
+at org.apache.hadoop.fs.s3a.S3AFileSystem.putObject(S3AFileSystem.java:3064)
+at org.apache.hadoop.fs.s3a.S3AFileSystem.executePut(S3AFileSystem.java:4054)
+
+```
+### `ClassNotFoundException: software.amazon.awssdk.services.s3.S3Client`
+
+(or other `software.amazon` class.)
-This means that the `aws-java-sdk-bundle.jar` JAR is not on the classpath:
+This means that the AWS V2 SDK `bundle.jar` JAR is not on the classpath:
add it.
-### `java.lang.NoSuchMethodError` referencing a `com.amazonaws` class
+### `ClassNotFoundException: com.amazonaws.auth.AWSCredentials`
+
+(or other `com.amazonaws` class.)
+
+With the move to the [V2 AWS SDK](../aws_sdk_upgrade.html),
+the v1 SDK classes are no longer on the classpath.
+
+If this happens when trying to use a custom credential provider defined
+in `fs.s3a.aws.credentials.provider`, then add the `aws-sdk-bundle.jar`
+JAR to the classpath.
+
+If this happens in your own/third-party code, then again, add the JAR,
+and/or consider moving to the v2 sdk yourself.
+
+### `java.lang.NoSuchMethodError` referencing a `software.amazon` class
This can be triggered by incompatibilities between the AWS SDK on the classpath
and the version which Hadoop was compiled with.
@@ -86,12 +117,14 @@ The AWS SDK JARs change their signature enough between releases that the only
way to safely update the AWS SDK version is to recompile Hadoop against the later
version.
-The sole fix is to use the same version of the AWS SDK with which Hadoop
+The fix is to use the same version of the AWS SDK with which Hadoop
was built.
This can also be caused by having more than one version of an AWS SDK
-JAR on the classpath. If the full `aws-java-sdk-bundle<` JAR is on the
-classpath, do not add any of the `aws-sdk-` JARs.
+JAR on the classpath. If the full `bundle.jar` JAR is on the
+classpath, do not add any of the `aws-sdk-` JARs *except* for
+`aws-crt.jar` (which is required) and
+`eventstream.jar` which is required when using S3 Select.
### `java.lang.NoSuchMethodError` referencing an `org.apache.hadoop` class
@@ -1990,51 +2023,3 @@ com.amazonaws.SdkClientException: Unable to execute HTTP request:
When this happens, try to set `fs.s3a.connection.request.timeout` to a larger value or disable it
completely by setting it to `0`.
-
-## SDK Upgrade Warnings
-
-S3A will soon be upgraded to [AWS's Java SDK V2](https://github.com/aws/aws-sdk-java-v2).
-For more information on the upgrade and what's changing, see
-[Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html).
-
-S3A logs the following warnings for things that will be changing in the upgrade. To disable these
-logs, comment out `log4j.logger.org.apache.hadoop.fs.s3a.SDKV2Upgrade` in log4j.properties.
-
-### `Directly referencing AWS SDK V1 credential provider`
-
-This will be logged when an AWS credential provider is referenced directly in
-`fs.s3a.aws.credentials.provider`.
-For example, `com.amazonaws.auth.AWSSessionCredentialsProvider`
-
-To stop this warning, remove any AWS credential providers from `fs.s3a.aws.credentials.provider`.
-Instead, use S3A's credential providers.
-
-### `getAmazonS3ClientForTesting() will be removed`
-
-This will be logged when `getAmazonS3ClientForTesting()` is called to get the S3 Client. With V2,
-the S3 client will change from type `com.amazonaws.services.s3.AmazonS3` to
-`software.amazon.awssdk.services.s3.S3Client`, and so this method will be removed.
-
-###
-### `Custom credential providers used in delegation tokens binding classes will need to be updated`
-
-This will be logged when delegation tokens are used.
-Delegation tokens allow the use of custom binding classes which can implement custom credential
-providers.
-These credential providers will currently be implementing
-`com.amazonaws.auth.AWSCredentialsProvider` and will need to be updated to implement
-`software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`.
-
-###
-### `The signer interface has changed in AWS SDK V2, custom signers will need to be updated`
-
-This will be logged when a custom signer is used.
-Custom signers will currently be implementing `com.amazonaws.auth.Signer` and will need to be
-updated to implement `software.amazon.awssdk.core.signer.Signer`.
-
-###
-### `getObjectMetadata() called. This operation and it's response will be changed`
-
-This will be logged when `getObjectMetadata` is called. In SDK V2, this operation has changed to
-`headObject()` and will return a response of the type `HeadObjectResponse`.
-
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
index d233081ee6851..b6da282e390a7 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
@@ -64,7 +64,7 @@ public void setup() throws Exception {
// unset S3CSE property from config to avoid pathIOE.
conf.unset(Constants.S3_ENCRYPTION_ALGORITHM);
fs.initialize(uri, conf);
- s3 = fs.getAmazonS3ClientForTesting("mocking");
+ s3 = fs.getS3AInternals().getAmazonS3V2ClientForTesting("mocking");
}
public Configuration createConfiguration() {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
index e90ad8b73efae..93f41cfaa81bb 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
@@ -210,6 +210,14 @@ public S3AFileSystem getFileSystem() {
return (S3AFileSystem) super.getFileSystem();
}
+ /**
+ * Get the {@link S3AInternals} internal access for the
+ * test filesystem.
+ * @return internals.
+ */
+ public S3AInternals getS3AInternals() {
+ return getFileSystem().getS3AInternals();
+ }
/**
* Describe a test in the logs.
* @param text text to print
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java
index 794480d4409fb..8d927dc957b16 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java
@@ -69,7 +69,7 @@ public static void assertEncrypted(S3AFileSystem fs,
final S3AEncryptionMethods algorithm,
final String kmsKeyArn)
throws IOException {
- HeadObjectResponse md = fs.getObjectMetadata(path);
+ HeadObjectResponse md = fs.getS3AInternals().getObjectMetadata(path);
String details = String.format(
"file %s with encryption algorithm %s and key %s",
path,
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
index 6d1b10954e7c5..bccbe79c2a48b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
@@ -26,8 +26,9 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+import org.assertj.core.api.Assertions;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
@@ -41,12 +42,14 @@
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getCSVTestPath;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
-import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_BINDING;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.CONSTRUCTOR_EXCEPTION;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assert.*;
/**
- * Integration tests for {@link Constants#AWS_CREDENTIALS_PROVIDER} logic.
+ * Integration tests for {@link Constants#AWS_CREDENTIALS_PROVIDER} logic
+ * through the S3A Filesystem instantiation process.
*/
public class ITestS3AAWSCredentialsProvider {
private static final Logger LOG =
@@ -55,17 +58,21 @@ public class ITestS3AAWSCredentialsProvider {
@Rule
public Timeout testTimeout = new Timeout(60_1000, TimeUnit.MILLISECONDS);
+ /**
+ * Expecting a wrapped ClassNotFoundException.
+ */
@Test
- public void testBadConfiguration() throws IOException {
- Configuration conf = createConf();
- conf.set(AWS_CREDENTIALS_PROVIDER, "no.such.class");
- try {
- createFailingFS(conf);
- } catch (IOException e) {
- if (!(e.getCause() instanceof ClassNotFoundException)) {
- LOG.error("Unexpected nested cause: {} in {}", e.getCause(), e, e);
- throw e;
- }
+ public void testProviderClassNotFound() throws Exception {
+ Configuration conf = createConf("no.such.class");
+ final InstantiationIOException e =
+ intercept(InstantiationIOException.class, "java.lang.ClassNotFoundException", () ->
+ createFailingFS(conf));
+ if (InstantiationIOException.Kind.InstantiationFailure != e.getKind()) {
+ throw e;
+ }
+ if (!(e.getCause() instanceof ClassNotFoundException)) {
+ LOG.error("Unexpected nested cause: {} in {}", e.getCause(), e, e);
+ throw e;
}
}
@@ -73,10 +80,10 @@ public void testBadConfiguration() throws IOException {
* A bad CredentialsProvider which has no suitable constructor.
*
* This class does not provide a public constructor accepting Configuration,
- * or a public factory method named getInstance that accepts no arguments,
+ * or a public factory method named create() that accepts no arguments,
* or a public default constructor.
*/
- static class BadCredentialsProviderConstructor
+ public static class BadCredentialsProviderConstructor
implements AwsCredentialsProvider {
@SuppressWarnings("unused")
@@ -92,37 +99,57 @@ public AwsCredentials resolveCredentials() {
@Test
public void testBadCredentialsConstructor() throws Exception {
- Configuration conf = createConf();
- conf.set(AWS_CREDENTIALS_PROVIDER,
- BadCredentialsProviderConstructor.class.getName());
- try {
- createFailingFS(conf);
- } catch (IOException e) {
- GenericTestUtils.assertExceptionContains(CONSTRUCTOR_EXCEPTION, e);
+ Configuration conf = createConf(BadCredentialsProviderConstructor.class);
+ final InstantiationIOException ex =
+ intercept(InstantiationIOException.class, CONSTRUCTOR_EXCEPTION, () ->
+ createFailingFS(conf));
+ if (InstantiationIOException.Kind.UnsupportedConstructor != ex.getKind()) {
+ throw ex;
}
}
- protected Configuration createConf() {
+ /**
+ * Create a configuration bonded to the given provider classname.
+ * @param provider provider to bond to
+ * @return a configuration
+ */
+ protected Configuration createConf(String provider) {
Configuration conf = new Configuration();
removeBaseAndBucketOverrides(conf,
DELEGATION_TOKEN_BINDING,
AWS_CREDENTIALS_PROVIDER);
+ conf.set(AWS_CREDENTIALS_PROVIDER, provider);
+ conf.set(DELEGATION_TOKEN_BINDING, "");
return conf;
}
+ /**
+ * Create a configuration bonded to the given provider class.
+ * @param provider provider to bond to
+ * @return a configuration
+ */
+ protected Configuration createConf(Class provider) {
+ return createConf(provider.getName());
+ }
+
/**
* Create a filesystem, expect it to fail by raising an IOException.
* Raises an assertion exception if in fact the FS does get instantiated.
+ * The FS is always deleted.
* @param conf configuration
* @throws IOException an expected exception.
*/
private void createFailingFS(Configuration conf) throws IOException {
- S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf);
- fs.listStatus(new Path("/"));
- fail("Expected exception - got " + fs);
+ try(S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
+ fs.listStatus(new Path("/"));
+ fail("Expected exception - got " + fs);
+ }
}
- static class BadCredentialsProvider implements AwsCredentialsProvider {
+ /**
+ * Returns an invalid set of credentials.
+ */
+ public static class BadCredentialsProvider implements AwsCredentialsProvider {
@SuppressWarnings("unused")
public BadCredentialsProvider(Configuration conf) {
@@ -137,34 +164,60 @@ public AwsCredentials resolveCredentials() {
@Test
public void testBadCredentials() throws Exception {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER, BadCredentialsProvider.class.getName());
- try {
- createFailingFS(conf);
- } catch (AccessDeniedException e) {
- // expected
- } catch (AWSServiceIOException e) {
- GenericTestUtils.assertExceptionContains(
- "UnrecognizedClientException", e);
- // expected
- }
+ Configuration conf = createConf(BadCredentialsProvider.class);
+ intercept(AccessDeniedException.class, "", () ->
+ createFailingFS(conf));
}
+ /**
+ * Test using the anonymous credential provider with the public csv
+ * test file; if the test file path is unset then it will be skipped.
+ */
@Test
public void testAnonymousProvider() throws Exception {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER,
- AnonymousAWSCredentialsProvider.class.getName());
+ Configuration conf = createConf(AnonymousAWSCredentialsProvider.class);
Path testFile = getCSVTestPath(conf);
try (FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf)) {
- assertNotNull("S3AFileSystem instance must not be null", fs);
- assertTrue("FileSystem must be the instance of S3AFileSystem", fs instanceof S3AFileSystem);
+ Assertions.assertThat(fs)
+ .describedAs("Filesystem")
+ .isNotNull();
FileStatus stat = fs.getFileStatus(testFile);
- assertNotNull("FileStatus with qualified path must not be null", stat);
assertEquals(
"The qualified path returned by getFileStatus should be same as the original file",
testFile, stat.getPath());
}
}
+ /**
+ * Create credentials via the create() method.
+ * They are invalid credentials, so IO will fail as access denied.
+ */
+ @Test
+ public void testCredentialsWithCreateMethod() throws Exception {
+ Configuration conf = createConf(CredentialsProviderWithCreateMethod.class);
+ intercept(AccessDeniedException.class, "", () ->
+ createFailingFS(conf));
+ }
+
+ /**
+ * Credentials via the create() method.
+ */
+ public static final class CredentialsProviderWithCreateMethod implements AwsCredentialsProvider {
+
+ public static AwsCredentialsProvider create() {
+ LOG.info("creating CredentialsProviderWithCreateMethod");
+ return new CredentialsProviderWithCreateMethod();
+ }
+
+ /** Private: cannot be created directly. */
+ private CredentialsProviderWithCreateMethod() {
+ }
+
+ @Override
+ public AwsCredentials resolveCredentials() {
+ return AwsBasicCredentials.create("bad_key", "bad_secret");
+ }
+
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java
index 2507ae2f5104f..38c4685eb137e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java
@@ -36,8 +36,10 @@
import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_ACCESSPOINT_REQUIRED;
+import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A;
import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
@@ -124,8 +126,12 @@ public static void expectUnknownStore(
private Configuration createConfigurationWithProbe(final int probe) {
Configuration conf = new Configuration(getFileSystem().getConf());
S3ATestUtils.disableFilesystemCaching(conf);
+ removeBaseAndBucketOverrides(conf,
+ S3A_BUCKET_PROBE,
+ ENDPOINT,
+ AWS_REGION);
conf.setInt(S3A_BUCKET_PROBE, probe);
- conf.set(AWS_REGION, "eu-west-1");
+ conf.set(AWS_REGION, EU_WEST_1);
return conf;
}
@@ -204,7 +210,7 @@ public void testAccessPointRequired() throws Exception {
*/
private Configuration createArnConfiguration() {
Configuration configuration = createConfigurationWithProbe(2);
- configuration.set(AWS_REGION, "eu-west-1");
+ configuration.set(AWS_REGION, EU_WEST_1);
return configuration;
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java
index 7ffb16833e044..6924065dbeeb4 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java
@@ -90,7 +90,7 @@ private void assertObjectHasLoggingGrant(Path path, boolean isFile) {
S3AFileSystem fs = getFileSystem();
StoreContext storeContext = fs.createStoreContext();
- S3Client s3 = fs.getAmazonS3ClientForTesting("acls");
+ S3Client s3 = getS3AInternals().getAmazonS3V2ClientForTesting("acls");
String key = storeContext.pathToKey(path);
if (!isFile) {
key = key + "/";
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index f7bdaa62422ed..d970d8a3ca7d0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,6 +18,18 @@
package org.apache.hadoop.fs.s3a;
+import java.io.File;
+import java.net.URI;
+import java.nio.file.AccessDeniedException;
+import java.security.PrivilegedExceptionAction;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import software.amazon.awssdk.core.client.config.SdkClientConfiguration;
import software.amazon.awssdk.core.client.config.SdkClientOption;
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
@@ -26,7 +38,6 @@
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
-import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.sts.StsClient;
import software.amazon.awssdk.services.sts.model.StsException;
@@ -38,29 +49,17 @@
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.auth.STSClientFactory;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
-import org.apache.hadoop.test.GenericTestUtils;
-
-import org.assertj.core.api.Assertions;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.File;
-import java.net.URI;
-import java.security.PrivilegedExceptionAction;
-
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.VersionInfo;
import org.apache.http.HttpStatus;
-import org.junit.rules.TemporaryFolder;
+import static java.util.Objects.requireNonNull;
import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.EU_WEST_1;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -91,6 +90,23 @@ public class ITestS3AConfiguration {
@Rule
public final TemporaryFolder tempDir = new TemporaryFolder();
+ /**
+ * Get the S3 client of the active filesystem.
+ * @param reason why?
+ * @return the client
+ */
+ private S3Client getS3Client(String reason) {
+ return requireNonNull(getS3AInternals().getAmazonS3V2ClientForTesting(reason));
+ }
+
+ /**
+ * Get the internals of the active filesystem.
+ * @return the internals
+ */
+ private S3AInternals getS3AInternals() {
+ return fs.getS3AInternals();
+ }
+
/**
* Test if custom endpoint is picked up.
*
@@ -118,7 +134,6 @@ public void testEndpoint() throws Exception {
} else {
conf.set(Constants.ENDPOINT, endpoint);
fs = S3ATestUtils.createTestFileSystem(conf);
- S3Client s3 = fs.getAmazonS3ClientForTesting("test endpoint");
String endPointRegion = "";
// Differentiate handling of "s3-" and "s3." based endpoint identifiers
String[] endpointParts = StringUtils.split(endpoint, '.');
@@ -129,9 +144,7 @@ public void testEndpoint() throws Exception {
} else {
fail("Unexpected endpoint");
}
- // TODO: review way to get the bucket region.
- String region = s3.getBucketLocation(b -> b.bucket(fs.getUri().getHost()))
- .locationConstraintAsString();
+ String region = getS3AInternals().getBucketLocation();
assertEquals("Endpoint config setting and bucket location differ: ",
endPointRegion, region);
}
@@ -159,7 +172,7 @@ protected void useFailFastConfiguration() {
}
/**
- * Expect a filesystem to not be created from a configuration
+ * Expect a filesystem to not be created from a configuration.
* @return the exception intercepted
* @throws Exception any other exception
*/
@@ -358,8 +371,7 @@ public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty()
try {
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- S3Client s3 = fs.getAmazonS3ClientForTesting("configuration");
- assertNotNull(s3);
+ S3Client s3 = getS3Client("configuration");
SdkClientConfiguration clientConfiguration = getField(s3, SdkClientConfiguration.class,
"clientConfiguration");
@@ -393,8 +405,7 @@ public void testDefaultUserAgent() throws Exception {
conf = new Configuration();
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- S3Client s3 = fs.getAmazonS3ClientForTesting("User Agent");
- assertNotNull(s3);
+ S3Client s3 = getS3Client("User Agent");
SdkClientConfiguration clientConfiguration = getField(s3, SdkClientConfiguration.class,
"clientConfiguration");
Assertions.assertThat(clientConfiguration.option(SdkClientOption.CLIENT_USER_AGENT))
@@ -408,8 +419,7 @@ public void testCustomUserAgent() throws Exception {
conf.set(Constants.USER_AGENT_PREFIX, "MyApp");
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- S3Client s3 = fs.getAmazonS3ClientForTesting("User agent");
- assertNotNull(s3);
+ S3Client s3 = getS3Client("User agent");
SdkClientConfiguration clientConfiguration = getField(s3, SdkClientConfiguration.class,
"clientConfiguration");
Assertions.assertThat(clientConfiguration.option(SdkClientOption.CLIENT_USER_AGENT))
@@ -422,7 +432,7 @@ public void testRequestTimeout() throws Exception {
conf = new Configuration();
conf.set(REQUEST_TIMEOUT, "120");
fs = S3ATestUtils.createTestFileSystem(conf);
- S3Client s3 = fs.getAmazonS3ClientForTesting("Request timeout (ms)");
+ S3Client s3 = getS3Client("Request timeout (ms)");
SdkClientConfiguration clientConfiguration = getField(s3, SdkClientConfiguration.class,
"clientConfiguration");
assertEquals("Configured " + REQUEST_TIMEOUT +
@@ -436,7 +446,7 @@ public void testCloseIdempotent() throws Throwable {
conf = new Configuration();
fs = S3ATestUtils.createTestFileSystem(conf);
AWSCredentialProviderList credentials =
- fs.shareCredentials("testCloseIdempotent");
+ getS3AInternals().shareCredentials("testCloseIdempotent");
credentials.close();
fs.close();
assertTrue("Closing FS didn't close credentials " + credentials,
@@ -529,36 +539,34 @@ public void testConfOptionPropagationToFS() throws Exception {
}
@Test(timeout = 10_000L)
- public void testS3SpecificSignerOverride() throws IOException {
+ public void testS3SpecificSignerOverride() throws Exception {
Configuration config = new Configuration();
+ removeBaseAndBucketOverrides(config,
+ CUSTOM_SIGNERS, SIGNING_ALGORITHM_S3, SIGNING_ALGORITHM_STS, AWS_REGION);
config.set(CUSTOM_SIGNERS,
- "CustomS3Signer:" + CustomS3Signer.class.getName() + ",CustomSTSSigner:"
- + CustomSTSSigner.class.getName());
+ "CustomS3Signer:" + CustomS3Signer.class.getName()
+ + ",CustomSTSSigner:" + CustomSTSSigner.class.getName());
config.set(SIGNING_ALGORITHM_S3, "CustomS3Signer");
config.set(SIGNING_ALGORITHM_STS, "CustomSTSSigner");
- config.set(AWS_REGION, "eu-west-1");
+ config.set(AWS_REGION, EU_WEST_1);
fs = S3ATestUtils.createTestFileSystem(config);
- S3Client s3Client = fs.getAmazonS3ClientForTesting("testS3SpecificSignerOverride");
+ S3Client s3Client = getS3Client("testS3SpecificSignerOverride");
+ final String bucket = fs.getBucket();
StsClient stsClient =
- STSClientFactory.builder(config, fs.getBucket(), new AnonymousAWSCredentialsProvider(), "",
+ STSClientFactory.builder(config, bucket, new AnonymousAWSCredentialsProvider(), "",
"").build();
- try {
- stsClient.getSessionToken();
- } catch (StsException exception) {
- // Expected 403, as credentials are not provided.
- }
+ intercept(StsException.class, "", () ->
+ stsClient.getSessionToken());
- try {
- s3Client.headBucket(HeadBucketRequest.builder().bucket(fs.getBucket()).build());
- } catch (S3Exception exception) {
- // Expected 403, as credentials are not provided.
- }
+ intercept(AccessDeniedException.class, "", () ->
+ Invoker.once("head", bucket, () ->
+ s3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build())));
Assertions.assertThat(CustomS3Signer.isS3SignerCalled())
.describedAs("Custom S3 signer not called").isTrue();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java
index 3a72206641452..7e399f347100f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java
@@ -51,7 +51,7 @@ protected S3AEncryptionMethods getSSEAlgorithm() {
@Override
protected void assertEncrypted(Path path) throws IOException {
- HeadObjectResponse md = getFileSystem().getObjectMetadata(path);
+ HeadObjectResponse md = getS3AInternals().getObjectMetadata(path);
assertEquals("SSE Algorithm", EncryptionTestUtils.AWS_KMS_SSE_ALGORITHM,
md.serverSideEncryptionAsString());
assertThat(md.ssekmsKeyId(), containsString("arn:aws:kms:"));
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java
index 41dd820b69bd4..e7e006e83ec8e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java
@@ -118,7 +118,8 @@ public void testEncryptionOverRename() throws Throwable {
S3AFileSystem fs = getFileSystem();
Path path = path(getMethodName() + "find-encryption-algo");
ContractTestUtils.touch(fs, path);
- String sseAlgorithm = fs.getObjectMetadata(path).serverSideEncryptionAsString();
+ String sseAlgorithm = getS3AInternals().getObjectMetadata(path)
+ .serverSideEncryptionAsString();
if(StringUtils.isBlank(sseAlgorithm) ||
!sseAlgorithm.equals(AWS_KMS_SSE_ALGORITHM)) {
skip("Test bucket is not configured with " + AWS_KMS_SSE_ALGORITHM);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
index c957ab7b6a438..e5e109ad91b50 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
@@ -25,7 +25,6 @@
import java.util.List;
import org.assertj.core.api.Assertions;
-import org.junit.Assert;
import org.junit.Test;
import software.amazon.awssdk.awscore.AwsExecutionAttribute;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
@@ -34,14 +33,12 @@
import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
-import software.amazon.awssdk.services.s3.model.S3Exception;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext;
import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
import static org.apache.hadoop.fs.s3a.Statistic.STORE_REGION_PROBE;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_301_MOVED_PERMANENTLY;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
@@ -68,13 +65,7 @@ public void testWithoutRegionConfig() throws IOException {
S3AFileSystem fs = new S3AFileSystem();
fs.initialize(getFileSystem().getUri(), conf);
- try {
- fs.getBucketMetadata();
- } catch (S3Exception exception) {
- if (exception.statusCode() == SC_301_MOVED_PERMANENTLY) {
- Assert.fail(exception.toString());
- }
- }
+ fs.getS3AInternals().getBucketMetadata();
Assertions.assertThat(fs.getInstrumentation().getCounterValue(STORE_REGION_PROBE))
.describedAs("Region is not configured, region probe should have been made").isEqualTo(1);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
index 6e85f6bc783dc..ee92bc1706eb8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
@@ -411,8 +411,8 @@ private static T verifyNoTrailingSlash(String role, T o) {
*/
private GetBucketEncryptionResponse getDefaultEncryption() throws IOException {
S3AFileSystem fs = getFileSystem();
- S3Client s3 = fs.getAmazonS3ClientForTesting("check default encryption");
- try {
+ S3Client s3 = getS3AInternals().getAmazonS3V2ClientForTesting("check default encryption");
+ try (AuditSpan s = span()){
return Invoker.once("getBucketEncryption()",
fs.getBucket(),
() -> s3.getBucketEncryption(GetBucketEncryptionRequest.builder()
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
index 35bb709f659f9..290a4d995c757 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
@@ -115,7 +115,7 @@ protected Configuration createConfiguration() {
public void testSTS() throws IOException {
Configuration conf = getContract().getConf();
S3AFileSystem testFS = getFileSystem();
- credentials = testFS.shareCredentials("testSTS");
+ credentials = getS3AInternals().shareCredentials("testSTS");
String bucket = testFS.getBucket();
StsClientBuilder builder = STSClientFactory.builder(
@@ -363,7 +363,7 @@ public E expectedSessionRequestFailure(
final String region,
final String exceptionText) throws Exception {
try(AWSCredentialProviderList parentCreds =
- getFileSystem().shareCredentials("test");
+ getS3AInternals().shareCredentials("test");
DurationInfo ignored = new DurationInfo(LOG, "requesting credentials")) {
Configuration conf = new Configuration(getContract().getConf());
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
index a6269c437665a..246d111d14b8d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
@@ -251,4 +251,9 @@ public interface S3ATestConstants {
* Value: {@value}.
*/
String PROJECT_BUILD_DIRECTORY_PROPERTY = "project.build.directory";
+
+ /**
+ * AWS ireland region.
+ */
+ String EU_WEST_1 = "eu-west-1";
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
index 44488c259a097..ce7cebd9ac4ab 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -30,42 +30,46 @@
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import javax.annotation.Nullable;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.ContainerCredentialsProvider;
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
-import org.apache.hadoop.fs.s3a.adapter.V1V2AwsCredentialProviderAdapter;
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.auth.AbstractSessionCredentialsProvider;
import org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider;
+import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider;
import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
import org.apache.hadoop.io.retry.RetryPolicy;
-import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3AUtils.*;
-import static org.apache.hadoop.fs.s3a.auth.AwsCredentialListProvider.ABSTRACT_PROVIDER;
-import static org.apache.hadoop.fs.s3a.auth.AwsCredentialListProvider.NOT_AWS_V2_PROVIDER;
-import static org.apache.hadoop.fs.s3a.auth.AwsCredentialListProvider.STANDARD_AWS_PROVIDERS;
-import static org.apache.hadoop.fs.s3a.auth.AwsCredentialListProvider.buildAWSProviderList;
-import static org.apache.hadoop.fs.s3a.auth.AwsCredentialListProvider.createAWSCredentialProviderSet;
+import static org.apache.hadoop.fs.s3a.Constants.ASSUMED_ROLE_CREDENTIALS_PROVIDER;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.DEFAULT_CSVTEST_FILE;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.authenticationContains;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.buildClassListString;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.getCSVTestPath;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.STANDARD_AWS_PROVIDERS;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.buildAWSProviderList;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSCredentialProviderList;
+import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.DOES_NOT_IMPLEMENT;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.apache.hadoop.test.LambdaTestUtils.interceptFuture;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
/**
* Unit tests for {@link Constants#AWS_CREDENTIALS_PROVIDER} logic.
@@ -78,19 +82,18 @@ public class TestS3AAWSCredentialsProvider {
private static final URI TESTFILE_URI = new Path(
DEFAULT_CSVTEST_FILE).toUri();
- @Rule
- public ExpectedException exception = ExpectedException.none();
+ private static final Logger LOG = LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class);
@Test
public void testProviderWrongClass() throws Exception {
expectProviderInstantiationFailure(this.getClass(),
- NOT_AWS_V2_PROVIDER);
+ DOES_NOT_IMPLEMENT + " software.amazon.awssdk.auth.credentials.AwsCredentialsProvider");
}
@Test
public void testProviderAbstractClass() throws Exception {
expectProviderInstantiationFailure(AbstractProvider.class,
- ABSTRACT_PROVIDER);
+ InstantiationIOException.ABSTRACT_PROVIDER);
}
@Test
@@ -103,14 +106,14 @@ public void testProviderNotAClass() throws Exception {
public void testProviderConstructorError() throws Exception {
expectProviderInstantiationFailure(
ConstructorSignatureErrorProvider.class,
- CONSTRUCTOR_EXCEPTION);
+ InstantiationIOException.CONSTRUCTOR_EXCEPTION);
}
@Test
public void testProviderFailureError() throws Exception {
expectProviderInstantiationFailure(
ConstructorFailureProvider.class,
- INSTANTIATION_EXCEPTION);
+ InstantiationIOException.INSTANTIATION_EXCEPTION);
}
@Test
@@ -122,7 +125,7 @@ public void testInstantiationChain() throws Throwable {
+ " ,\n " + AnonymousAWSCredentialsProvider.NAME);
Path testFile = getCSVTestPath(conf);
- AWSCredentialProviderList list = createAWSCredentialProviderSet(
+ AWSCredentialProviderList list = createAWSCredentialProviderList(
testFile.toUri(), conf);
List> expectedClasses =
Arrays.asList(
@@ -138,9 +141,9 @@ public void testDefaultChain() throws Exception {
Configuration conf = new Configuration(false);
// use the default credential provider chain
conf.unset(AWS_CREDENTIALS_PROVIDER);
- AWSCredentialProviderList list1 = createAWSCredentialProviderSet(
+ AWSCredentialProviderList list1 = createAWSCredentialProviderList(
uri1, conf);
- AWSCredentialProviderList list2 = createAWSCredentialProviderSet(
+ AWSCredentialProviderList list2 = createAWSCredentialProviderList(
uri2, conf);
List> expectedClasses = STANDARD_AWS_PROVIDERS;
assertCredentialProviders(expectedClasses, list1);
@@ -153,28 +156,7 @@ public void testDefaultChainNoURI() throws Exception {
// use the default credential provider chain
conf.unset(AWS_CREDENTIALS_PROVIDER);
assertCredentialProviders(STANDARD_AWS_PROVIDERS,
- createAWSCredentialProviderSet(null, conf));
- }
-
- @Test
- public void testConfiguredChainV1V2() throws Exception {
- URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2");
- List> credentialProviders =
- Arrays.asList(
- ContainerCredentialsProvider.class,
- AnonymousAWSCredentialsProvider.class);
- List> expectedClasses =
- Arrays.asList(
- V1V2AwsCredentialProviderAdapter.class,
- AnonymousAWSCredentialsProvider.class);
- Configuration conf =
- createProviderConfiguration(buildClassListString(credentialProviders));
- AWSCredentialProviderList list1 = createAWSCredentialProviderSet(
- uri1, conf);
- AWSCredentialProviderList list2 = createAWSCredentialProviderSet(
- uri2, conf);
- assertCredentialProviders(expectedClasses, list1);
- assertCredentialProviders(expectedClasses, list2);
+ createAWSCredentialProviderList(null, conf));
}
@Test
@@ -182,14 +164,15 @@ public void testConfiguredChain() throws Exception {
URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2");
List> expectedClasses =
Arrays.asList(
- EnvironmentVariableCredentialsProvider.class,
- InstanceProfileCredentialsProvider.class,
- AnonymousAWSCredentialsProvider.class);
+ IAMInstanceCredentialsProvider.class,
+ AnonymousAWSCredentialsProvider.class,
+ EnvironmentVariableCredentialsProvider.class
+ );
Configuration conf =
createProviderConfiguration(buildClassListString(expectedClasses));
- AWSCredentialProviderList list1 = createAWSCredentialProviderSet(
+ AWSCredentialProviderList list1 = createAWSCredentialProviderList(
uri1, conf);
- AWSCredentialProviderList list2 = createAWSCredentialProviderSet(
+ AWSCredentialProviderList list2 = createAWSCredentialProviderList(
uri2, conf);
assertCredentialProviders(expectedClasses, list1);
assertCredentialProviders(expectedClasses, list2);
@@ -203,9 +186,9 @@ public void testConfiguredChainUsesSharedInstanceProfile() throws Exception {
Arrays.asList(
InstanceProfileCredentialsProvider.class);
conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses));
- AWSCredentialProviderList list1 = createAWSCredentialProviderSet(
+ AWSCredentialProviderList list1 = createAWSCredentialProviderList(
uri1, conf);
- AWSCredentialProviderList list2 = createAWSCredentialProviderSet(
+ AWSCredentialProviderList list2 = createAWSCredentialProviderList(
uri2, conf);
assertCredentialProviders(expectedClasses, list1);
assertCredentialProviders(expectedClasses, list2);
@@ -222,51 +205,75 @@ public void testFallbackToDefaults() throws Throwable {
EnvironmentVariableCredentialsProvider.class),
Sets.newHashSet());
assertTrue("empty credentials", credentials.size() > 0);
+ }
+
+ @Test
+ public void testProviderConstructor() throws Throwable {
+ final AWSCredentialProviderList list = new AWSCredentialProviderList("name",
+ new AnonymousAWSCredentialsProvider(),
+ new ErrorProvider(TESTFILE_URI, new Configuration()));
+ Assertions.assertThat(list.getProviders())
+ .describedAs("provider list in %s", list)
+ .hasSize(2);
+ final AwsCredentials credentials = list.resolveCredentials();
+ Assertions.assertThat(credentials)
+ .isInstanceOf(AwsBasicCredentials.class);
+ assertCredentialResolution(credentials, null, null);
+ }
+
+ public static void assertCredentialResolution(AwsCredentials creds, String key, String secret) {
+ Assertions.assertThat(creds.accessKeyId())
+ .describedAs("access key of %s", creds)
+ .isEqualTo(key);
+ Assertions.assertThat(creds.secretAccessKey())
+ .describedAs("secret key of %s", creds)
+ .isEqualTo(secret);
+ }
+
+ private String buildClassList(Class... classes) {
+ return Arrays.stream(classes)
+ .map(Class::getCanonicalName)
+ .collect(Collectors.joining(","));
+ }
+ private String buildClassList(String... classes) {
+ return Arrays.stream(classes)
+ .collect(Collectors.joining(","));
}
/**
* A credential provider declared as abstract, so it cannot be instantiated.
*/
- static abstract class AbstractProvider implements AWSCredentialsProvider {
+ static abstract class AbstractProvider implements AwsCredentialsProvider {
+
+ @Override
+ public AwsCredentials resolveCredentials() {
+ return null;
+ }
}
/**
* A credential provider whose constructor signature doesn't match.
*/
protected static class ConstructorSignatureErrorProvider
- implements AWSCredentialsProvider {
+ extends AbstractProvider {
@SuppressWarnings("unused")
public ConstructorSignatureErrorProvider(String str) {
}
-
- @Override
- public AWSCredentials getCredentials() {
- return null;
- }
-
- @Override
- public void refresh() {
- }
}
/**
* A credential provider whose constructor raises an NPE.
*/
protected static class ConstructorFailureProvider
- implements AwsCredentialsProvider {
+ extends AbstractProvider {
@SuppressWarnings("unused")
public ConstructorFailureProvider() {
throw new NullPointerException("oops");
}
- @Override
- public AwsCredentials resolveCredentials() {
- return null;
- }
-
}
@Test
@@ -279,33 +286,23 @@ public void testAWSExceptionTranslation() throws Throwable {
}
}
- protected static class AWSExceptionRaisingFactory implements AWSCredentialsProvider {
+ protected static class AWSExceptionRaisingFactory extends AbstractProvider {
public static final String NO_AUTH = "No auth";
- public static AWSCredentialsProvider getInstance() {
+ public static AwsCredentialsProvider create() {
throw new NoAuthWithAWSException(NO_AUTH);
}
-
- @Override
- public AWSCredentials getCredentials() {
- return null;
- }
-
- @Override
- public void refresh() {
-
- }
}
@Test
public void testFactoryWrongType() throws Throwable {
expectProviderInstantiationFailure(
FactoryOfWrongType.class,
- CONSTRUCTOR_EXCEPTION);
+ InstantiationIOException.CONSTRUCTOR_EXCEPTION);
}
- static class FactoryOfWrongType implements AWSCredentialsProvider {
+ static class FactoryOfWrongType extends AbstractProvider {
public static final String NO_AUTH = "No auth";
@@ -314,14 +311,10 @@ public static String getInstance() {
}
@Override
- public AWSCredentials getCredentials() {
+ public AwsCredentials resolveCredentials() {
return null;
}
- @Override
- public void refresh() {
-
- }
}
/**
@@ -334,7 +327,7 @@ public void refresh() {
private IOException expectProviderInstantiationFailure(String option,
String expectedErrorText) throws Exception {
return intercept(IOException.class, expectedErrorText,
- () -> createAWSCredentialProviderSet(
+ () -> createAWSCredentialProviderList(
TESTFILE_URI,
createProviderConfiguration(option)));
}
@@ -385,7 +378,9 @@ private static void assertCredentialProviders(
AWSCredentialProviderList list) {
assertNotNull(list);
List providers = list.getProviders();
- assertEquals(expectedClasses.size(), providers.size());
+ Assertions.assertThat(providers)
+ .describedAs("providers")
+ .hasSize(expectedClasses.size());
for (int i = 0; i < expectedClasses.size(); ++i) {
Class expectedClass =
expectedClasses.get(i);
@@ -493,22 +488,13 @@ public void testIOEInConstructorPropagation() throws Throwable {
/**
* Credential provider which raises an IOE when constructed.
*/
- protected static class IOERaisingProvider implements AWSCredentialsProvider {
+ protected static class IOERaisingProvider extends AbstractProvider {
public IOERaisingProvider(URI uri, Configuration conf)
throws IOException {
throw new InterruptedIOException("expected");
}
- @Override
- public AWSCredentials getCredentials() {
- return null;
- }
-
- @Override
- public void refresh() {
-
- }
}
private static final AwsCredentials EXPECTED_CREDENTIALS =
@@ -538,13 +524,13 @@ public void testConcurrentAuthentication() throws Throwable {
Configuration conf = createProviderConfiguration(SlowProvider.class.getName());
Path testFile = getCSVTestPath(conf);
- AWSCredentialProviderList list = createAWSCredentialProviderSet(testFile.toUri(), conf);
+ AWSCredentialProviderList list = createAWSCredentialProviderList(testFile.toUri(), conf);
SlowProvider provider = (SlowProvider) list.getProviders().get(0);
ExecutorService pool = Executors.newFixedThreadPool(CONCURRENT_THREADS);
- List> results = new ArrayList<>();
+ List> results = new ArrayList<>();
try {
assertFalse(
@@ -560,15 +546,15 @@ public void testConcurrentAuthentication() throws Throwable {
}
for (int i = 0; i < CONCURRENT_THREADS; i++) {
- results.add(pool.submit(() -> list.getCredentials()));
+ results.add(pool.submit(() -> list.resolveCredentials()));
}
- for (Future result : results) {
- AWSCredentials credentials = result.get();
+ for (Future result : results) {
+ AwsCredentials credentials = result.get();
assertEquals("Access key from credential provider",
- "expectedAccessKey", credentials.getAWSAccessKeyId());
+ "expectedAccessKey", credentials.accessKeyId());
assertEquals("Secret key from credential provider",
- "expectedSecret", credentials.getAWSSecretKey());
+ "expectedSecret", credentials.secretAccessKey());
}
} finally {
pool.awaitTermination(10, TimeUnit.SECONDS);
@@ -577,7 +563,7 @@ public void testConcurrentAuthentication() throws Throwable {
assertTrue(
"Provider initialized without errors. isInitialized should be true",
- provider.isInitialized());
+ provider.isInitialized());
assertTrue(
"Provider initialized without errors. hasCredentials should be true",
provider.hasCredentials());
@@ -608,12 +594,12 @@ public void testConcurrentAuthenticationError() throws Throwable {
Configuration conf = createProviderConfiguration(ErrorProvider.class.getName());
Path testFile = getCSVTestPath(conf);
- AWSCredentialProviderList list = createAWSCredentialProviderSet(testFile.toUri(), conf);
+ AWSCredentialProviderList list = createAWSCredentialProviderList(testFile.toUri(), conf);
ErrorProvider provider = (ErrorProvider) list.getProviders().get(0);
ExecutorService pool = Executors.newFixedThreadPool(CONCURRENT_THREADS);
- List> results = new ArrayList<>();
+ List> results = new ArrayList<>();
try {
assertFalse("Provider not initialized. isInitialized should be false",
@@ -627,10 +613,10 @@ public void testConcurrentAuthenticationError() throws Throwable {
}
for (int i = 0; i < CONCURRENT_THREADS; i++) {
- results.add(pool.submit(() -> list.getCredentials()));
+ results.add(pool.submit(() -> list.resolveCredentials()));
}
- for (Future result : results) {
+ for (Future result : results) {
interceptFuture(CredentialInitializationException.class,
"expected error",
result
@@ -651,4 +637,39 @@ public void testConcurrentAuthenticationError() throws Throwable {
"Provider initialization failed. getInitializationException should contain the error",
provider.getInitializationException().getMessage().contains("expected error"));
}
+
+
+ /**
+ * V2 Credentials whose factory method raises ClassNotFoundException.
+ * This will fall back to an attempted v1 load which will fail because it
+ * is the wrong type.
+ * The exception raised will be from the v2 instantiation attempt,
+ * not the v1 attempt.
+ */
+ @Test
+ public void testV2ClassNotFound() throws Throwable {
+ InstantiationIOException expected = intercept(InstantiationIOException.class,
+ "simulated v2 CNFE",
+ () -> createAWSCredentialProviderList(
+ TESTFILE_URI,
+ createProviderConfiguration(V2CredentialProviderDoesNotInstantiate.class.getName())));
+ // print for the curious
+ LOG.info("{}", expected.toString());
+ }
+
+ /**
+ * V2 credentials which raises an instantiation exception in
+ * the factory method.
+ */
+ public static final class V2CredentialProviderDoesNotInstantiate
+ extends AbstractProvider {
+
+ private V2CredentialProviderDoesNotInstantiate() {
+ }
+
+ public static AwsCredentialsProvider create() throws ClassNotFoundException {
+ throw new ClassNotFoundException("simulated v2 CNFE");
+ }
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java
index 1f85c8fdef304..0ec96624ef171 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java
@@ -75,7 +75,7 @@ public void testDeleteOnExit() throws Exception {
// unset S3CSE property from config to avoid pathIOE.
conf.unset(Constants.S3_ENCRYPTION_ALGORITHM);
testFs.initialize(uri, conf);
- S3Client testS3 = testFs.getAmazonS3ClientForTesting("mocking");
+ S3Client testS3 = testFs.getS3AInternals().getAmazonS3V2ClientForTesting("mocking");
Path path = new Path("/file");
String key = path.toUri().getPath().substring(1);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java
new file mode 100644
index 0000000000000..b0e1b57d75471
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.adapter;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
+import org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider;
+import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
+
+import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.DEFAULT_CSVTEST_FILE;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.ANONYMOUS_CREDENTIALS_V1;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.EC2_CONTAINER_CREDENTIALS_V1;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.ENVIRONMENT_CREDENTIALS_V1;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSCredentialProviderList;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Unit tests for v1 to v2 credential provider logic.
+ */
+public class TestV1CredentialsProvider {
+
+ /**
+ * URI of the landsat images.
+ */
+ private static final URI TESTFILE_URI = new Path(
+ DEFAULT_CSVTEST_FILE).toUri();
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestV1CredentialsProvider.class);
+
+
+ @Test
+ public void testV1V2Mapping() throws Exception {
+ URI uri1 = new URI("s3a://bucket1");
+
+ List> expectedClasses =
+ Arrays.asList(
+ IAMInstanceCredentialsProvider.class,
+ AnonymousAWSCredentialsProvider.class,
+ EnvironmentVariableCredentialsProvider.class);
+ Configuration conf =
+ createProviderConfiguration(buildClassList(
+ EC2_CONTAINER_CREDENTIALS_V1,
+ ANONYMOUS_CREDENTIALS_V1,
+ ENVIRONMENT_CREDENTIALS_V1));
+ AWSCredentialProviderList list1 = createAWSCredentialProviderList(
+ uri1, conf);
+ assertCredentialProviders(expectedClasses, list1);
+ }
+
+ @Test
+ public void testV1Wrapping() throws Exception {
+ URI uri1 = new URI("s3a://bucket1");
+
+ List> expectedClasses =
+ Arrays.asList(
+ V1ToV2AwsCredentialProviderAdapter.class,
+ V1ToV2AwsCredentialProviderAdapter.class);
+ Configuration conf =
+ createProviderConfiguration(buildClassList(
+ LegacyV1CredentialProvider.class.getName(),
+ LegacyV1CredentialProviderWithConf.class.getName()));
+ AWSCredentialProviderList list1 = createAWSCredentialProviderList(
+ uri1, conf);
+ assertCredentialProviders(expectedClasses, list1);
+ }
+
+ private String buildClassList(String... classes) {
+ return Arrays.stream(classes)
+ .collect(Collectors.joining(","));
+ }
+
+
+ /**
+ * Expect a provider to raise an exception on failure.
+ * @param option aws provider option string.
+ * @param expectedErrorText error text to expect
+ * @return the exception raised
+ * @throws Exception any unexpected exception thrown.
+ */
+ private IOException expectProviderInstantiationFailure(String option,
+ String expectedErrorText) throws Exception {
+ return intercept(IOException.class, expectedErrorText,
+ () -> createAWSCredentialProviderList(
+ TESTFILE_URI,
+ createProviderConfiguration(option)));
+ }
+
+ /**
+ * Create a configuration with a specific provider.
+ * @param providerOption option for the aws credential provider option.
+ * @return a configuration to use in test cases
+ */
+ private Configuration createProviderConfiguration(
+ final String providerOption) {
+ Configuration conf = new Configuration(false);
+ conf.set(AWS_CREDENTIALS_PROVIDER, providerOption);
+ return conf;
+ }
+
+ /**
+ * Asserts expected provider classes in list.
+ * @param expectedClasses expected provider classes
+ * @param list providers to check
+ */
+ private static void assertCredentialProviders(
+ List> expectedClasses,
+ AWSCredentialProviderList list) {
+ assertNotNull(list);
+ List providers = list.getProviders();
+ Assertions.assertThat(providers)
+ .describedAs("providers")
+ .hasSize(expectedClasses.size());
+ for (int i = 0; i < expectedClasses.size(); ++i) {
+ Class expectedClass =
+ expectedClasses.get(i);
+ AwsCredentialsProvider provider = providers.get(i);
+ assertNotNull(
+ String.format("At position %d, expected class is %s, but found null.",
+ i, expectedClass), provider);
+ assertTrue(
+ String.format("At position %d, expected class is %s, but found %s.",
+ i, expectedClass, provider.getClass()),
+ expectedClass.isAssignableFrom(provider.getClass()));
+ }
+ }
+
+
+ public static class LegacyV1CredentialProvider implements AWSCredentialsProvider {
+
+ public LegacyV1CredentialProvider() {
+ }
+
+ @Override
+ public AWSCredentials getCredentials() {
+ return null;
+ }
+
+ @Override
+ public void refresh() {
+
+ }
+ }
+
+ /**
+ * V1 credentials with a configuration constructor.
+ */
+ public static final class LegacyV1CredentialProviderWithConf
+ extends LegacyV1CredentialProvider {
+
+ public LegacyV1CredentialProviderWithConf(Configuration conf) {
+ }
+ }
+
+ /**
+ * V1 Credentials whose factory method raises ClassNotFoundException.
+ * Expect this to fail rather than trigger recursive recovery;
+ * exception will be wrapped with something intended to be informative.
+ */
+ @Test
+ public void testV1InstantiationFailurePropagation() throws Throwable {
+ InstantiationIOException expected = intercept(InstantiationIOException.class,
+ "simulated CNFE",
+ () -> createAWSCredentialProviderList(
+ TESTFILE_URI,
+ createProviderConfiguration(V1CredentialProviderDoesNotInstantiate.class.getName())));
+ // print for the curious
+ LOG.info("{}", expected.toString());
+ }
+
+
+ /**
+ * V1 credentials which raises an instantiation exception.
+ */
+ public static final class V1CredentialProviderDoesNotInstantiate
+ extends LegacyV1CredentialProvider {
+
+ private V1CredentialProviderDoesNotInstantiate() {
+ }
+
+ public static AWSCredentialsProvider getInstance() throws ClassNotFoundException {
+ throw new ClassNotFoundException("simulated CNFE");
+ }
+ }
+
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java
index 5c33f19270ebb..e2297e37e50c4 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java
@@ -25,7 +25,6 @@
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
-import java.util.function.Consumer;
import java.util.stream.Collectors;
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditManager.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditManager.java
index bd60165ebe42e..ea7a1a34da735 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditManager.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditManager.java
@@ -34,6 +34,7 @@
import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.enableLoggingAuditor;
import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.resetAuditOptions;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_EXECUTION_INTERCEPTORS;
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_REQUEST_HANDLERS;
import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.UNAUDITED_OPERATION;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticCounter;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupCounterStatistic;
@@ -59,6 +60,7 @@ public Configuration createConfiguration() {
enableLoggingAuditor(conf);
conf.set(AUDIT_EXECUTION_INTERCEPTORS,
SimpleAWSExecutionInterceptor.CLASS);
+ conf.set(AUDIT_REQUEST_HANDLERS, "not-valid-class");
return conf;
}
@@ -114,8 +116,8 @@ public void testInvokeOutOfSpanRejected() throws Throwable {
}
@Test
- public void testRequestHandlerBinding() throws Throwable {
- describe("Verify that extra request handlers can be added and that they"
+ public void testExecutionInterceptorBinding() throws Throwable {
+ describe("Verify that extra ExecutionInterceptor can be added and that they"
+ " will be invoked during request execution");
final long baseCount = SimpleAWSExecutionInterceptor.getInvocationCount();
final S3AFileSystem fs = getFileSystem();
@@ -131,5 +133,9 @@ public void testRequestHandlerBinding() throws Throwable {
.isGreaterThan(exec0);
assertThatStatisticCounter(iostats(), AUDIT_FAILURE.getSymbol())
.isZero();
+ Assertions.assertThat(SimpleAWSExecutionInterceptor.getStaticConf())
+ .describedAs("configuratin of SimpleAWSExecutionInterceptor")
+ .isNotNull()
+ .isSameAs(fs.getConf());
}
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/SimpleAWSExecutionInterceptor.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/SimpleAWSExecutionInterceptor.java
index 8014b05187387..bf9b90bcdf31e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/SimpleAWSExecutionInterceptor.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/SimpleAWSExecutionInterceptor.java
@@ -24,17 +24,23 @@
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+
/**
* Simple AWS interceptor to verify dynamic loading of extra
* execution interceptors during auditing setup.
* The invocation counter tracks the count of calls to
* {@link #beforeExecution}.
*/
-public final class SimpleAWSExecutionInterceptor implements ExecutionInterceptor {
+public final class SimpleAWSExecutionInterceptor extends Configured
+ implements ExecutionInterceptor {
public static final String CLASS
= "org.apache.hadoop.fs.s3a.audit.SimpleAWSExecutionInterceptor";
+ private static Configuration staticConf;
+
/** Count of invocations. */
private static final AtomicLong INVOCATIONS = new AtomicLong(0);
@@ -42,6 +48,7 @@ public final class SimpleAWSExecutionInterceptor implements ExecutionInterceptor
public void beforeExecution(Context.BeforeExecution context,
ExecutionAttributes executionAttributes) {
INVOCATIONS.incrementAndGet();
+ staticConf = getConf();
}
/**
@@ -51,4 +58,14 @@ public void beforeExecution(Context.BeforeExecution context,
public static long getInvocationCount() {
return INVOCATIONS.get();
}
+
+ /**
+ * get the static conf, which is set the config of the
+ * last executor invoked.
+ * @return the static configuration.
+ */
+
+ public static Configuration getStaticConf() {
+ return staticConf;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
index 1c6e00655acb2..7d604f2ef5149 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
@@ -52,13 +52,14 @@
import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
import org.apache.hadoop.fs.s3a.commit.impl.CommitContext;
import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations;
+import org.apache.hadoop.fs.s3a.impl.InstantiationIOException;
import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool;
import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics;
import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-import static org.apache.hadoop.fs.s3a.auth.AwsCredentialListProvider.E_FORBIDDEN_AWS_PROVIDER;
+import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.E_FORBIDDEN_AWS_PROVIDER;
import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.*;
import static org.apache.hadoop.fs.s3a.auth.RoleModel.*;
import static org.apache.hadoop.fs.s3a.auth.RolePolicies.*;
@@ -189,7 +190,12 @@ public void testAssumedInvalidRole() throws Throwable {
conf.set(ASSUMED_ROLE_ARN, ROLE_ARN_EXAMPLE);
interceptClosing(StsException.class,
"",
- () -> new AssumedRoleCredentialProvider(uri, conf));
+ () -> {
+ AssumedRoleCredentialProvider p =
+ new AssumedRoleCredentialProvider(uri, conf);
+ p.resolveCredentials();
+ return p;
+ });
}
@Test
@@ -241,7 +247,7 @@ public void testAssumeRoleCannotAuthAssumedRole() throws Exception {
conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER,
AssumedRoleCredentialProvider.NAME);
expectFileSystemCreateFailure(conf,
- IOException.class,
+ InstantiationIOException.class,
E_FORBIDDEN_AWS_PROVIDER);
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
index cdf89211fd7fc..ad7d59a7319cf 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
@@ -152,7 +152,7 @@ private Configuration createTestConfig(String identifier) {
}
private String determineRegion(String bucketName) throws IOException {
- return getFileSystem().getBucketLocation(bucketName);
+ return getS3AInternals().getBucketLocation(bucketName);
}
@Private
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java
index 186887d745bfc..852f03ea618fd 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java
@@ -151,13 +151,17 @@ public static Configuration newAssumedRoleConfig(
final String roleARN) {
Configuration conf = new Configuration(srcConf);
removeBaseAndBucketOverrides(conf,
+ S3A_BUCKET_PROBE,
DELEGATION_TOKEN_BINDING,
ASSUMED_ROLE_ARN,
- AWS_CREDENTIALS_PROVIDER);
+ AWS_CREDENTIALS_PROVIDER,
+ ASSUMED_ROLE_SESSION_DURATION);
conf.set(AWS_CREDENTIALS_PROVIDER, AssumedRoleCredentialProvider.NAME);
conf.set(ASSUMED_ROLE_ARN, roleARN);
conf.set(ASSUMED_ROLE_SESSION_NAME, "test");
conf.set(ASSUMED_ROLE_SESSION_DURATION, "15m");
+ // force in bucket resolution during startup
+ conf.setInt(S3A_BUCKET_PROBE, 1);
disableFilesystemCaching(conf);
return conf;
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/CountInvocationsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/CountInvocationsProvider.java
index 4ee79e7220afc..4c7cd5c667999 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/CountInvocationsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/CountInvocationsProvider.java
@@ -20,6 +20,8 @@
import java.util.concurrent.atomic.AtomicLong;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
@@ -31,14 +33,35 @@
public class CountInvocationsProvider
implements AwsCredentialsProvider {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ CountInvocationsProvider.class);
+
public static final String NAME = CountInvocationsProvider.class.getName();
public static final AtomicLong COUNTER = new AtomicLong(0);
+ private final AtomicLong instanceCounter = new AtomicLong(0);
+
@Override
public AwsCredentials resolveCredentials() {
- COUNTER.incrementAndGet();
- throw new CredentialInitializationException("no credentials");
+ final long global = COUNTER.incrementAndGet();
+ final long local = instanceCounter.incrementAndGet();
+ final String msg =
+ String.format("counter with global count %d and local count %d", global, local);
+ LOG.debug("resolving credentials from {}", msg);
+ throw new CredentialInitializationException("no credentials from " + msg);
+ }
+
+ public long getInstanceCounter() {
+ return instanceCounter.get();
+ }
+
+ @Override
+ public String toString() {
+ return "CountInvocationsProvider{" +
+ "instanceCounter=" + instanceCounter.get() +
+ "; global counter=" + COUNTER.get() +
+ '}';
}
public static long getInvocationCount() {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java
index 28784b17c9ce8..ebad90336f7d0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java
@@ -582,7 +582,7 @@ public void testDelegationBindingMismatch2() throws Throwable {
protected HeadBucketResponse readLandsatMetadata(final S3AFileSystem delegatedFS)
throws Exception {
AWSCredentialProviderList testingCreds
- = delegatedFS.shareCredentials("testing");
+ = delegatedFS.getS3AInternals().shareCredentials("testing");
URI landsat = new URI(DEFAULT_CSVTEST_FILE);
DefaultS3ClientFactory factory
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
index 7f13cb3a4d161..efc775966859d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
@@ -186,11 +186,15 @@ public void testCreateAndUseDT() throws Throwable {
final MarshalledCredentials creds;
try(S3ADelegationTokens dt2 = instantiateDTSupport(getConfiguration())) {
dt2.start();
+ // first creds are good
+ dt2.getCredentialProviders().resolveCredentials();
+
+ // reset to the original dt
dt2.resetTokenBindingToDT(originalDT);
final AwsSessionCredentials awsSessionCreds
= verifySessionCredentials(
- dt2.getCredentialProviders().resolveCredentials());
+ dt2.getCredentialProviders().resolveCredentials());
final MarshalledCredentials origCreds = fromAWSCredentials(
awsSessionCreds);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java
index 44854e84d894c..258c34b5cb84f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java
@@ -741,8 +741,8 @@ private void validateContent(Path dir,
*/
private void validateStorageClass(Path dir, String expectedStorageClass) throws Exception {
Path expectedFile = getPart0000(dir);
- S3AFileSystem fs = getFileSystem();
- String actualStorageClass = fs.getObjectMetadata(expectedFile).storageClassAsString();
+ String actualStorageClass = getS3AInternals().getObjectMetadata(expectedFile)
+ .storageClassAsString();
Assertions.assertThat(actualStorageClass)
.describedAs("Storage class of object %s", expectedFile)
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
index de0048c25581c..4e02d01458fb2 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
@@ -214,7 +214,7 @@ protected Configuration createConfiguration() {
public void setup() throws Exception {
super.setup();
S3AFileSystem fs = getFileSystem();
- s3client = fs.getAmazonS3ClientForTesting("markers");
+ s3client = getS3AInternals().getAmazonS3V2ClientForTesting("markers");
bucket = fs.getBucket();
Path base = new Path(methodPath(), "base");
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java
index 006c989604fd7..ccc71c58644f2 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java
@@ -125,8 +125,8 @@ private void skipQuietly(String text) {
}
protected void assertStorageClass(Path hugeFile) throws IOException {
- S3AFileSystem fs = getFileSystem();
- String actual = fs.getObjectMetadata(hugeFile).storageClassAsString();
+
+ String actual = getS3AInternals().getObjectMetadata(hugeFile).storageClassAsString();
assertTrue(
"Storage class of object is " + actual + ", expected " + STORAGE_CLASS_REDUCED_REDUNDANCY,
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/StreamPublisher.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/StreamPublisher.java
index c770b8897338f..461aef726876c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/StreamPublisher.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/StreamPublisher.java
@@ -35,12 +35,12 @@ final class StreamPublisher implements SdkPublisher {
private final Iterator iterator;
private Boolean done = false;
- public StreamPublisher(Stream data, Executor executor) {
+ StreamPublisher(Stream data, Executor executor) {
this.iterator = data.iterator();
this.executor = executor;
}
- public StreamPublisher(Stream data) {
+ StreamPublisher(Stream data) {
this(data, Runnable::run);
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestSelectEventStreamPublisher.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestSelectEventStreamPublisher.java
index faf32fe4fd94d..fdf3b5b725376 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestSelectEventStreamPublisher.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestSelectEventStreamPublisher.java
@@ -156,7 +156,9 @@ public void handlesErrors() throws IOException {
SelectObjectContentEventStream.recordsBuilder()
.payload(SdkBytes.fromUtf8String("bar"))
.build())
- .map(e -> { throw SdkException.create("error!", null); }));
+ .map(e -> {
+ throw SdkException.create("error!", null);
+ }));
try (AbortableInputStream inputStream =
selectEventStreamPublisher.toRecordsInputStream(e -> {})) {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/ITestAWSStatisticCollection.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/ITestAWSStatisticCollection.java
index e7696996dbd1a..8c97d896edbde 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/ITestAWSStatisticCollection.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/ITestAWSStatisticCollection.java
@@ -53,7 +53,7 @@ public void testLandsatStatistics() throws Throwable {
conf.unset("fs.s3a.bucket.landsat-pds.endpoint");
try (S3AFileSystem fs = (S3AFileSystem) path.getFileSystem(conf)) {
- fs.getObjectMetadata(path);
+ fs.getS3AInternals().getObjectMetadata(path);
IOStatistics iostats = fs.getIOStatistics();
assertThatStatisticCounter(iostats,
STORE_IO_REQUEST.getSymbol())
@@ -71,7 +71,7 @@ public void testCommonCrawlStatistics() throws Throwable {
conf.set(ENDPOINT, DEFAULT_ENDPOINT);
try (S3AFileSystem fs = (S3AFileSystem) path.getFileSystem(conf)) {
- fs.getObjectMetadata(path);
+ fs.getS3AInternals().getObjectMetadata(path);
IOStatistics iostats = fs.getIOStatistics();
assertThatStatisticCounter(iostats,
STORE_IO_REQUEST.getSymbol())