diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 9e15b3f6c56cd..956b074e862cb 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -67,6 +67,10 @@ private Constants() { public static final String SECURE_CONNECTIONS = "fs.s3a.connection.ssl.enabled"; public static final boolean DEFAULT_SECURE_CONNECTIONS = true; + + //allow access to requester pay buckets + public static final String ALLOW_REQUESTER_PAYS = "fs.s3a.requester-pays.enabled"; + public static final boolean DEFAULT_ALLOW_REQUESTER_PAYS = false; //use a custom endpoint? public static final String ENDPOINT = "fs.s3a.endpoint"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index d4e09e300d734..b1b88e0cf6389 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -90,6 +90,8 @@ private static void initConnectionSettings(Configuration conf, DEFAULT_MAXIMUM_CONNECTIONS, 1)); boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS); + boolean requesterPays = conf.getBoolean(ALLOW_REQUESTER_PAYS, + DEFAULT_ALLOW_REQUESTER_PAYS); awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES, 0)); @@ -107,6 +109,12 @@ private static void initConnectionSettings(Configuration conf, LOG.debug("Signer override = {}", signerOverride); awsConf.setSignerOverride(signerOverride); } + System.out.printf("\n\nRequester Pays is enabled? %b\n\n", requesterPays); + if (requesterPays == true) { + LOG.info("fs.s3a.requester-pays.enabled has been set to true. You will be charged for any requests made to Requester Pays buckets."); + LOG.info("For more information on Requester Pays buckets visit: http://docs.aws.amazon.com/AmazonS3/latest/dev/RequesterPaysBuckets.html"); + awsConf.addHeader("x-amz-request-payer", "requester"); + } } /**