From 1fabfae602c5dcba298fb4d4ebcd78ccc2505bc9 Mon Sep 17 00:00:00 2001 From: Antonio Matarrese Date: Tue, 27 Jun 2017 16:24:09 +0100 Subject: [PATCH 01/11] Update AWS Java SDK to support ECS Task Roles --- plugins/repository-s3/build.gradle | 2 +- .../elasticsearch/repositories/s3/InternalAwsS3Service.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 9db78e5cbb2ca..04a0d5d0edc50 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -23,7 +23,7 @@ esplugin { } versions << [ - 'aws': '1.10.69' + 'aws': '1.11.155' ] dependencies { diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java index 58a130bcd95c2..a84820dffbc35 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java @@ -27,7 +27,7 @@ import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.InstanceProfileCredentialsProvider; +import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; @@ -166,10 +166,10 @@ protected void doClose() throws ElasticsearchException { } static class PrivilegedInstanceProfileCredentialsProvider implements AWSCredentialsProvider { - private final InstanceProfileCredentialsProvider credentials; + private final EC2ContainerCredentialsProviderWrapper credentials; private PrivilegedInstanceProfileCredentialsProvider() { - this.credentials = new InstanceProfileCredentialsProvider(); + this.credentials = new EC2ContainerCredentialsProviderWrapper(); } @Override From 2f275f30315c8dddb0acb6d504fa363a438f12b5 Mon Sep 17 00:00:00 2001 From: Antonio Matarrese Date: Thu, 29 Jun 2017 13:58:53 +0100 Subject: [PATCH 02/11] Revert "Update AWS Java SDK to support ECS Task Roles" This reverts commit 1fabfae602c5dcba298fb4d4ebcd78ccc2505bc9. --- plugins/repository-s3/build.gradle | 2 +- .../elasticsearch/repositories/s3/InternalAwsS3Service.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 04a0d5d0edc50..9db78e5cbb2ca 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -23,7 +23,7 @@ esplugin { } versions << [ - 'aws': '1.11.155' + 'aws': '1.10.69' ] dependencies { diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java index a84820dffbc35..58a130bcd95c2 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java @@ -27,7 +27,7 @@ import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; +import com.amazonaws.auth.InstanceProfileCredentialsProvider; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; @@ -166,10 +166,10 @@ protected void doClose() throws ElasticsearchException { } static class PrivilegedInstanceProfileCredentialsProvider implements AWSCredentialsProvider { - private final EC2ContainerCredentialsProviderWrapper credentials; + private final InstanceProfileCredentialsProvider credentials; private PrivilegedInstanceProfileCredentialsProvider() { - this.credentials = new EC2ContainerCredentialsProviderWrapper(); + this.credentials = new InstanceProfileCredentialsProvider(); } @Override From 6e5482a68ec991f948f47fec6d8511da50b58866 Mon Sep 17 00:00:00 2001 From: Antonio Matarrese Date: Wed, 5 Jul 2017 14:33:47 +0100 Subject: [PATCH 03/11] Update AWS Java SDK to support ECS Task Roles --- plugins/repository-s3/build.gradle | 2 +- .../repositories/s3/InternalAwsS3Service.java | 6 +- .../repositories/s3/AmazonS3Wrapper.java | 60 +++++++++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 9db78e5cbb2ca..e871696e5bcbf 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -23,7 +23,7 @@ esplugin { } versions << [ - 'aws': '1.10.69' + 'aws': '1.11.16' ] dependencies { diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java index 58a130bcd95c2..8a2ac0722bb43 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java @@ -27,7 +27,7 @@ import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.InstanceProfileCredentialsProvider; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; @@ -166,10 +166,10 @@ protected void doClose() throws ElasticsearchException { } static class PrivilegedInstanceProfileCredentialsProvider implements AWSCredentialsProvider { - private final InstanceProfileCredentialsProvider credentials; + private final DefaultAWSCredentialsProviderChain credentials; private PrivilegedInstanceProfileCredentialsProvider() { - this.credentials = new InstanceProfileCredentialsProvider(); + this.credentials = new DefaultAWSCredentialsProviderChain(); } @Override diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java index 2ce64d5c6fc83..128fe32755a2c 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java @@ -30,6 +30,7 @@ import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; import com.amazonaws.services.s3.model.AccessControlList; import com.amazonaws.services.s3.model.Bucket; +import com.amazonaws.services.s3.model.BucketAccelerateConfiguration; import com.amazonaws.services.s3.model.BucketCrossOriginConfiguration; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration; import com.amazonaws.services.s3.model.BucketLoggingConfiguration; @@ -59,6 +60,7 @@ import com.amazonaws.services.s3.model.DeleteObjectsResult; import com.amazonaws.services.s3.model.DeleteVersionRequest; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; +import com.amazonaws.services.s3.model.GetBucketAccelerateConfigurationRequest; import com.amazonaws.services.s3.model.GetBucketAclRequest; import com.amazonaws.services.s3.model.GetBucketCrossOriginConfigurationRequest; import com.amazonaws.services.s3.model.GetBucketLifecycleConfigurationRequest; @@ -83,6 +85,8 @@ import com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest; import com.amazonaws.services.s3.model.ListNextBatchOfVersionsRequest; import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; import com.amazonaws.services.s3.model.ListPartsRequest; import com.amazonaws.services.s3.model.ListVersionsRequest; import com.amazonaws.services.s3.model.MultipartUploadListing; @@ -94,6 +98,7 @@ import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.RestoreObjectRequest; import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.SetBucketAccelerateConfigurationRequest; import com.amazonaws.services.s3.model.SetBucketAclRequest; import com.amazonaws.services.s3.model.SetBucketCrossOriginConfigurationRequest; import com.amazonaws.services.s3.model.SetBucketLifecycleConfigurationRequest; @@ -167,6 +172,21 @@ public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws A return delegate.listObjects(listObjectsRequest); } + @Override + public ListObjectsV2Result listObjectsV2(final String bucketName) throws AmazonClientException, AmazonServiceException { + return delegate.listObjectsV2(bucketName); + } + + @Override + public ListObjectsV2Result listObjectsV2(final String bucketName, final String prefix) throws AmazonClientException, AmazonServiceException { + return delegate.listObjectsV2(bucketName, prefix); + } + + @Override + public ListObjectsV2Result listObjectsV2(final ListObjectsV2Request listObjectsV2Request) throws AmazonClientException, AmazonServiceException { + return delegate.listObjectsV2(listObjectsV2Request); + } + @Override public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) throws AmazonClientException, AmazonServiceException { return delegate.listNextBatchOfObjects(previousObjectListing); @@ -332,6 +352,11 @@ public ObjectMetadata getObject(GetObjectRequest getObjectRequest, File destinat return delegate.getObject(getObjectRequest, destinationFile); } + @Override + public String getObjectAsString(final String bucketName, final String key) throws AmazonServiceException, AmazonClientException { + return null; + } + @Override public void deleteBucket(DeleteBucketRequest deleteBucketRequest) throws AmazonClientException, AmazonServiceException { delegate.deleteBucket(deleteBucketRequest); @@ -373,6 +398,36 @@ public boolean doesObjectExist(String bucketName, String objectName) throws Amaz return delegate.doesObjectExist(bucketName, objectName); } + @Override + public BucketAccelerateConfiguration getBucketAccelerateConfiguration(final String bucket) throws AmazonServiceException, AmazonClientException { + return delegate.getBucketAccelerateConfiguration(bucket); + } + + @Override + public BucketAccelerateConfiguration getBucketAccelerateConfiguration(final GetBucketAccelerateConfigurationRequest getBucketAccelerateConfigurationRequest) throws AmazonServiceException, AmazonClientException { + return delegate.getBucketAccelerateConfiguration(getBucketAccelerateConfigurationRequest); + } + + @Override + public void setBucketAccelerateConfiguration(final String bucketName, final BucketAccelerateConfiguration accelerateConfiguration) throws AmazonServiceException, AmazonClientException { + delegate.setBucketAccelerateConfiguration(bucketName, accelerateConfiguration); + } + + @Override + public void setBucketAccelerateConfiguration(final SetBucketAccelerateConfigurationRequest setBucketAccelerateConfigurationRequest) throws AmazonServiceException, AmazonClientException { + delegate.setBucketAccelerateConfiguration(setBucketAccelerateConfigurationRequest); + } + + @Override + public com.amazonaws.services.s3.model.Region getRegion() { + return delegate.getRegion(); + } + + @Override + public URL getUrl(final String bucketName, final String key) { + return delegate.getUrl(bucketName,key); + } + @Override public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException { return delegate.putObject(putObjectRequest); @@ -388,6 +443,11 @@ public PutObjectResult putObject(String bucketName, String key, InputStream inpu return delegate.putObject(bucketName, key, input, metadata); } + @Override + public PutObjectResult putObject(final String bucketName, final String key, final String content) throws AmazonServiceException, AmazonClientException { + return delegate.putObject(bucketName, key, content); + } + @Override public CopyObjectResult copyObject(String sourceBucketName, String sourceKey, String destinationBucketName, String destinationKey) throws AmazonClientException, AmazonServiceException { return delegate.copyObject(sourceBucketName, sourceKey, destinationBucketName, destinationKey); From 85ce750a1aeb0ca114cf416132c84966299318b9 Mon Sep 17 00:00:00 2001 From: Antonio Matarrese Date: Fri, 29 Sep 2017 14:35:00 +0100 Subject: [PATCH 04/11] update aws version --- plugins/repository-s3/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index ba858bae7c713..e9412c59bdb6c 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -23,7 +23,7 @@ esplugin { } versions << [ - 'aws': '1.11.16' + 'aws': '1.11.187' ] dependencies { From fb79f62c869782772578a55b7adf514b8a7a905c Mon Sep 17 00:00:00 2001 From: Antonio Matarrese Date: Mon, 2 Oct 2017 16:55:23 +0100 Subject: [PATCH 05/11] s3 update library --- .../repositories/s3/AmazonS3Wrapper.java | 172 +++++++++++++++++- 1 file changed, 171 insertions(+), 1 deletion(-) diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java index 128fe32755a2c..6b8f68d0e901d 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java @@ -23,6 +23,7 @@ import com.amazonaws.AmazonServiceException; import com.amazonaws.AmazonWebServiceRequest; import com.amazonaws.HttpMethod; +import com.amazonaws.SdkClientException; import com.amazonaws.regions.Region; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.S3ClientOptions; @@ -48,24 +49,38 @@ import com.amazonaws.services.s3.model.CopyPartRequest; import com.amazonaws.services.s3.model.CopyPartResult; import com.amazonaws.services.s3.model.CreateBucketRequest; +import com.amazonaws.services.s3.model.DeleteBucketAnalyticsConfigurationRequest; +import com.amazonaws.services.s3.model.DeleteBucketAnalyticsConfigurationResult; import com.amazonaws.services.s3.model.DeleteBucketCrossOriginConfigurationRequest; +import com.amazonaws.services.s3.model.DeleteBucketInventoryConfigurationRequest; +import com.amazonaws.services.s3.model.DeleteBucketInventoryConfigurationResult; import com.amazonaws.services.s3.model.DeleteBucketLifecycleConfigurationRequest; +import com.amazonaws.services.s3.model.DeleteBucketMetricsConfigurationRequest; +import com.amazonaws.services.s3.model.DeleteBucketMetricsConfigurationResult; import com.amazonaws.services.s3.model.DeleteBucketPolicyRequest; import com.amazonaws.services.s3.model.DeleteBucketReplicationConfigurationRequest; import com.amazonaws.services.s3.model.DeleteBucketRequest; import com.amazonaws.services.s3.model.DeleteBucketTaggingConfigurationRequest; import com.amazonaws.services.s3.model.DeleteBucketWebsiteConfigurationRequest; import com.amazonaws.services.s3.model.DeleteObjectRequest; +import com.amazonaws.services.s3.model.DeleteObjectTaggingRequest; +import com.amazonaws.services.s3.model.DeleteObjectTaggingResult; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsResult; import com.amazonaws.services.s3.model.DeleteVersionRequest; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; import com.amazonaws.services.s3.model.GetBucketAccelerateConfigurationRequest; import com.amazonaws.services.s3.model.GetBucketAclRequest; +import com.amazonaws.services.s3.model.GetBucketAnalyticsConfigurationRequest; +import com.amazonaws.services.s3.model.GetBucketAnalyticsConfigurationResult; import com.amazonaws.services.s3.model.GetBucketCrossOriginConfigurationRequest; +import com.amazonaws.services.s3.model.GetBucketInventoryConfigurationRequest; +import com.amazonaws.services.s3.model.GetBucketInventoryConfigurationResult; import com.amazonaws.services.s3.model.GetBucketLifecycleConfigurationRequest; import com.amazonaws.services.s3.model.GetBucketLocationRequest; import com.amazonaws.services.s3.model.GetBucketLoggingConfigurationRequest; +import com.amazonaws.services.s3.model.GetBucketMetricsConfigurationRequest; +import com.amazonaws.services.s3.model.GetBucketMetricsConfigurationResult; import com.amazonaws.services.s3.model.GetBucketNotificationConfigurationRequest; import com.amazonaws.services.s3.model.GetBucketPolicyRequest; import com.amazonaws.services.s3.model.GetBucketReplicationConfigurationRequest; @@ -75,11 +90,19 @@ import com.amazonaws.services.s3.model.GetObjectAclRequest; import com.amazonaws.services.s3.model.GetObjectMetadataRequest; import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.GetObjectTaggingRequest; +import com.amazonaws.services.s3.model.GetObjectTaggingResult; import com.amazonaws.services.s3.model.GetS3AccountOwnerRequest; import com.amazonaws.services.s3.model.HeadBucketRequest; import com.amazonaws.services.s3.model.HeadBucketResult; import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; +import com.amazonaws.services.s3.model.ListBucketAnalyticsConfigurationsRequest; +import com.amazonaws.services.s3.model.ListBucketAnalyticsConfigurationsResult; +import com.amazonaws.services.s3.model.ListBucketInventoryConfigurationsRequest; +import com.amazonaws.services.s3.model.ListBucketInventoryConfigurationsResult; +import com.amazonaws.services.s3.model.ListBucketMetricsConfigurationsRequest; +import com.amazonaws.services.s3.model.ListBucketMetricsConfigurationsResult; import com.amazonaws.services.s3.model.ListBucketsRequest; import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; import com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest; @@ -100,9 +123,15 @@ import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.SetBucketAccelerateConfigurationRequest; import com.amazonaws.services.s3.model.SetBucketAclRequest; +import com.amazonaws.services.s3.model.SetBucketAnalyticsConfigurationRequest; +import com.amazonaws.services.s3.model.SetBucketAnalyticsConfigurationResult; import com.amazonaws.services.s3.model.SetBucketCrossOriginConfigurationRequest; +import com.amazonaws.services.s3.model.SetBucketInventoryConfigurationRequest; +import com.amazonaws.services.s3.model.SetBucketInventoryConfigurationResult; import com.amazonaws.services.s3.model.SetBucketLifecycleConfigurationRequest; import com.amazonaws.services.s3.model.SetBucketLoggingConfigurationRequest; +import com.amazonaws.services.s3.model.SetBucketMetricsConfigurationRequest; +import com.amazonaws.services.s3.model.SetBucketMetricsConfigurationResult; import com.amazonaws.services.s3.model.SetBucketNotificationConfigurationRequest; import com.amazonaws.services.s3.model.SetBucketPolicyRequest; import com.amazonaws.services.s3.model.SetBucketReplicationConfigurationRequest; @@ -110,10 +139,16 @@ import com.amazonaws.services.s3.model.SetBucketVersioningConfigurationRequest; import com.amazonaws.services.s3.model.SetBucketWebsiteConfigurationRequest; import com.amazonaws.services.s3.model.SetObjectAclRequest; +import com.amazonaws.services.s3.model.SetObjectTaggingRequest; +import com.amazonaws.services.s3.model.SetObjectTaggingResult; import com.amazonaws.services.s3.model.StorageClass; import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; import com.amazonaws.services.s3.model.VersionListing; +import com.amazonaws.services.s3.model.analytics.AnalyticsConfiguration; +import com.amazonaws.services.s3.model.inventory.InventoryConfiguration; +import com.amazonaws.services.s3.model.metrics.MetricsConfiguration; +import com.amazonaws.services.s3.waiters.AmazonS3Waiters; import org.elasticsearch.common.SuppressForbidden; import java.io.File; @@ -354,7 +389,22 @@ public ObjectMetadata getObject(GetObjectRequest getObjectRequest, File destinat @Override public String getObjectAsString(final String bucketName, final String key) throws AmazonServiceException, AmazonClientException { - return null; + return delegate.getObjectAsString(bucketName,key); + } + + @Override + public GetObjectTaggingResult getObjectTagging(final GetObjectTaggingRequest getObjectTaggingRequest) { + return delegate.getObjectTagging(getObjectTaggingRequest); + } + + @Override + public SetObjectTaggingResult setObjectTagging(final SetObjectTaggingRequest setObjectTaggingRequest) { + return delegate.setObjectTagging(setObjectTaggingRequest); + } + + @Override + public DeleteObjectTaggingResult deleteObjectTagging(final DeleteObjectTaggingRequest deleteObjectTaggingRequest) { + return delegate.deleteObjectTagging(deleteObjectTaggingRequest); } @Override @@ -418,16 +468,136 @@ public void setBucketAccelerateConfiguration(final SetBucketAccelerateConfigurat delegate.setBucketAccelerateConfiguration(setBucketAccelerateConfigurationRequest); } + @Override + public DeleteBucketMetricsConfigurationResult deleteBucketMetricsConfiguration(final String bucketName, final String id) throws AmazonServiceException, SdkClientException { + return delegate.deleteBucketMetricsConfiguration(bucketName, id); + } + + @Override + public DeleteBucketMetricsConfigurationResult deleteBucketMetricsConfiguration(final DeleteBucketMetricsConfigurationRequest deleteBucketMetricsConfigurationRequest) throws AmazonServiceException, SdkClientException { + return delegate.deleteBucketMetricsConfiguration(deleteBucketMetricsConfigurationRequest); + } + + @Override + public GetBucketMetricsConfigurationResult getBucketMetricsConfiguration(final String bucketName, final String id) throws AmazonServiceException, SdkClientException { + return delegate.getBucketMetricsConfiguration(bucketName, id); + } + + @Override + public GetBucketMetricsConfigurationResult getBucketMetricsConfiguration(final GetBucketMetricsConfigurationRequest getBucketMetricsConfigurationRequest) throws AmazonServiceException, SdkClientException { + return delegate.getBucketMetricsConfiguration(getBucketMetricsConfigurationRequest); + } + + @Override + public SetBucketMetricsConfigurationResult setBucketMetricsConfiguration(final String bucketName, final MetricsConfiguration metricsConfiguration) throws AmazonServiceException, SdkClientException { + return delegate.setBucketMetricsConfiguration(bucketName, metricsConfiguration); + } + + @Override + public SetBucketMetricsConfigurationResult setBucketMetricsConfiguration(final SetBucketMetricsConfigurationRequest setBucketMetricsConfigurationRequest) throws AmazonServiceException, SdkClientException { + return delegate.setBucketMetricsConfiguration(setBucketMetricsConfigurationRequest); + } + + @Override + public ListBucketMetricsConfigurationsResult listBucketMetricsConfigurations(final ListBucketMetricsConfigurationsRequest listBucketMetricsConfigurationsRequest) throws AmazonServiceException, SdkClientException { + return delegate.listBucketMetricsConfigurations(listBucketMetricsConfigurationsRequest); + } + + @Override + public DeleteBucketAnalyticsConfigurationResult deleteBucketAnalyticsConfiguration(final String bucketName, final String id) throws AmazonServiceException, SdkClientException { + return delegate.deleteBucketAnalyticsConfiguration(bucketName, id); + } + + @Override + public DeleteBucketAnalyticsConfigurationResult deleteBucketAnalyticsConfiguration(final DeleteBucketAnalyticsConfigurationRequest deleteBucketAnalyticsConfigurationRequest) throws AmazonServiceException, SdkClientException { + return delegate.deleteBucketAnalyticsConfiguration(deleteBucketAnalyticsConfigurationRequest); + } + + @Override + public GetBucketAnalyticsConfigurationResult getBucketAnalyticsConfiguration(final String bucketName, final String id) throws AmazonServiceException, SdkClientException { + return delegate.getBucketAnalyticsConfiguration(bucketName, id); + } + + @Override + public GetBucketAnalyticsConfigurationResult getBucketAnalyticsConfiguration(final GetBucketAnalyticsConfigurationRequest getBucketAnalyticsConfigurationRequest) throws AmazonServiceException, SdkClientException { + return delegate.getBucketAnalyticsConfiguration(getBucketAnalyticsConfigurationRequest); + } + + @Override + public SetBucketAnalyticsConfigurationResult setBucketAnalyticsConfiguration(final String bucketName, final AnalyticsConfiguration analyticsConfiguration) throws AmazonServiceException, SdkClientException { + return delegate.setBucketAnalyticsConfiguration(bucketName, analyticsConfiguration); + } + + @Override + public SetBucketAnalyticsConfigurationResult setBucketAnalyticsConfiguration(final SetBucketAnalyticsConfigurationRequest setBucketAnalyticsConfigurationRequest) throws AmazonServiceException, SdkClientException { + return delegate.setBucketAnalyticsConfiguration(setBucketAnalyticsConfigurationRequest); + } + + @Override + public ListBucketAnalyticsConfigurationsResult listBucketAnalyticsConfigurations(final ListBucketAnalyticsConfigurationsRequest listBucketAnalyticsConfigurationsRequest) throws AmazonServiceException, SdkClientException { + return delegate.listBucketAnalyticsConfigurations(listBucketAnalyticsConfigurationsRequest); + } + + @Override + public DeleteBucketInventoryConfigurationResult deleteBucketInventoryConfiguration(final String bucketName, final String id) throws AmazonServiceException, SdkClientException { + return delegate.deleteBucketInventoryConfiguration(bucketName, id); + } + + @Override + public DeleteBucketInventoryConfigurationResult deleteBucketInventoryConfiguration(final DeleteBucketInventoryConfigurationRequest deleteBucketInventoryConfigurationRequest) throws AmazonServiceException, SdkClientException { + return delegate.deleteBucketInventoryConfiguration(deleteBucketInventoryConfigurationRequest); + } + + @Override + public GetBucketInventoryConfigurationResult getBucketInventoryConfiguration(final String bucketName, final String id) throws AmazonServiceException, SdkClientException { + return delegate.getBucketInventoryConfiguration(bucketName, id); + } + + @Override + public GetBucketInventoryConfigurationResult getBucketInventoryConfiguration(final GetBucketInventoryConfigurationRequest getBucketInventoryConfigurationRequest) throws AmazonServiceException, SdkClientException { + return delegate.getBucketInventoryConfiguration(getBucketInventoryConfigurationRequest); + } + + @Override + public SetBucketInventoryConfigurationResult setBucketInventoryConfiguration(final String bucketName, final InventoryConfiguration inventoryConfiguration) throws AmazonServiceException, SdkClientException { + return delegate.setBucketInventoryConfiguration(bucketName, inventoryConfiguration); + } + + @Override + public SetBucketInventoryConfigurationResult setBucketInventoryConfiguration(final SetBucketInventoryConfigurationRequest setBucketInventoryConfigurationRequest) throws AmazonServiceException, SdkClientException { + return delegate.setBucketInventoryConfiguration(setBucketInventoryConfigurationRequest); + } + + @Override + public ListBucketInventoryConfigurationsResult listBucketInventoryConfigurations(final ListBucketInventoryConfigurationsRequest listBucketInventoryConfigurationsRequest) throws AmazonServiceException, SdkClientException { + return delegate.listBucketInventoryConfigurations(listBucketInventoryConfigurationsRequest); + } + + @Override + public void shutdown() { + delegate.shutdown(); + } + @Override public com.amazonaws.services.s3.model.Region getRegion() { return delegate.getRegion(); } + @Override + public String getRegionName() { + return delegate.getRegionName(); + } + @Override public URL getUrl(final String bucketName, final String key) { return delegate.getUrl(bucketName,key); } + @Override + public AmazonS3Waiters waiters() { + return delegate.waiters(); + } + @Override public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException { return delegate.putObject(putObjectRequest); From 907f0ac612b46552c6b21316be43ccb308ca3c3d Mon Sep 17 00:00:00 2001 From: Antonio Matarrese Date: Mon, 2 Oct 2017 17:03:34 +0100 Subject: [PATCH 06/11] missing space --- .../java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java index 6b8f68d0e901d..0133ac3d9097f 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java @@ -389,7 +389,7 @@ public ObjectMetadata getObject(GetObjectRequest getObjectRequest, File destinat @Override public String getObjectAsString(final String bucketName, final String key) throws AmazonServiceException, AmazonClientException { - return delegate.getObjectAsString(bucketName,key); + return delegate.getObjectAsString(bucketName, key); } @Override From f2eb80ef92d0c2b52dee9b34d2a99569a0ca10b8 Mon Sep 17 00:00:00 2001 From: Antonio Matarrese Date: Tue, 3 Oct 2017 10:48:30 +0100 Subject: [PATCH 07/11] new generate sha1 --- .../repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-core-1.11.187.jar.sha1 | 1 + plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-kms-1.11.187.jar.sha1 | 1 + plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1 | 1 - plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.187.jar.sha1 | 1 + 6 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.187.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.187.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.187.jar.sha1 diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1 deleted file mode 100644 index 2971a33d7d91b..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a1f02d5f26ba1d8c37e2bf9c847db3c6729dda00 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.187.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.187.jar.sha1 new file mode 100644 index 0000000000000..a5293a9bf6580 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.187.jar.sha1 @@ -0,0 +1 @@ +6f47fcd3c2917bef69dc36aba203c5ea4af9bf24 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1 deleted file mode 100644 index 22e1f924664fa..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ed74ff3872193b4704a751f0b72ab2cf0db0651b \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.187.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.187.jar.sha1 new file mode 100644 index 0000000000000..ff7c6f616449c --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.187.jar.sha1 @@ -0,0 +1 @@ +f2f4e9afe15bd6a718ad64716042e52f406e668b \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1 deleted file mode 100644 index 64e7336c2bc91..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6fa48bf0bff43f26436956b88d8d3764b6cf109e \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.187.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.187.jar.sha1 new file mode 100644 index 0000000000000..b7e3a511c8c9d --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.187.jar.sha1 @@ -0,0 +1 @@ +2647df2408c435289293260dc27f0762ce259c30 \ No newline at end of file From fd0d6643e31488ed9095652fbb136c4e76620eda Mon Sep 17 00:00:00 2001 From: Antonio Matarrese Date: Tue, 3 Oct 2017 17:17:43 +0100 Subject: [PATCH 08/11] thirdPartyAudit.excludes --- plugins/repository-s3/build.gradle | 9 +++++++++ .../repositories/s3/AwsS3ServiceImplTests.java | 6 +++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index e9412c59bdb6c..3204a7a4d16e5 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -76,6 +76,15 @@ thirdPartyAudit.excludes = [ 'org.apache.avalon.framework.logger.Logger', 'org.apache.log.Hierarchy', 'org.apache.log.Logger', + 'software/amazon/ion/IonReader', + 'software/amazon/ion/IonSystem', + 'software/amazon/ion/IonType', + 'software/amazon/ion/IonWriter', + 'software/amazon/ion/Timestamp', + 'software/amazon/ion/system/IonBinaryWriterBuilder', + 'software/amazon/ion/system/IonSystemBuilder', + 'software/amazon/ion/system/IonTextWriterBuilder', + 'software/amazon/ion/system/IonWriterBuilder' ] // jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9) diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java index 9863402ec37c1..f98c588455f0e 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java @@ -95,7 +95,7 @@ private void assertCredentials(Settings singleRepositorySettings, Settings setti } public void testAWSDefaultConfiguration() { - launchAWSConfigurationTest(Settings.EMPTY, Settings.EMPTY, Protocol.HTTPS, null, -1, null, null, 3, false, + launchAWSConfigurationTest(Settings.EMPTY, Settings.EMPTY, Protocol.HTTPS, null, -1, null, null, 3, true, ClientConfiguration.DEFAULT_SOCKET_TIMEOUT); } @@ -111,7 +111,7 @@ public void testAWSConfigurationWithAwsSettings() { .put("s3.client.default.read_timeout", "10s") .build(); launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username", - "aws_proxy_password", 3, false, 10000); + "aws_proxy_password", 3, true, 10000); } public void testRepositoryMaxRetries() { @@ -119,7 +119,7 @@ public void testRepositoryMaxRetries() { .put("s3.client.default.max_retries", 5) .build(); launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, null, -1, null, - null, 5, false, 50000); + null, 5, true, 50000); } public void testRepositoryThrottleRetries() { From d5ad7d4c265a954c73769a422d72a5dd02b778b6 Mon Sep 17 00:00:00 2001 From: Antonio Matarrese Date: Sat, 6 Jan 2018 16:53:48 +0000 Subject: [PATCH 09/11] ContainerCredentialProvider --- .../elasticsearch/repositories/s3/InternalAwsS3Service.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java index e2f5dcaea981a..48efcaff15ad7 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java @@ -23,7 +23,7 @@ import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; @@ -158,10 +158,10 @@ protected void doClose() throws ElasticsearchException { } static class PrivilegedInstanceProfileCredentialsProvider implements AWSCredentialsProvider { - private final DefaultAWSCredentialsProviderChain credentials; + private final EC2ContainerCredentialsProviderWrapper credentials; private PrivilegedInstanceProfileCredentialsProvider() { - this.credentials = new DefaultAWSCredentialsProviderChain(); + this.credentials = new EC2ContainerCredentialsProviderWrapper(); } @Override From 008edf69ef7b601d03465becac091f1d0ca304fc Mon Sep 17 00:00:00 2001 From: Antonio Matarrese Date: Sun, 14 Jan 2018 02:12:56 +0000 Subject: [PATCH 10/11] rename inner class in AwsS3Service --- .../elasticsearch/repositories/s3/InternalAwsS3Service.java | 6 +++--- .../repositories/s3/AwsS3ServiceImplTests.java | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java index 48efcaff15ad7..e950cc8e92c56 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java @@ -132,7 +132,7 @@ static AWSCredentialsProvider buildCredentials(Logger logger, DeprecationLogger } if (credentials == null) { logger.debug("Using instance profile credentials"); - return new PrivilegedInstanceProfileCredentialsProvider(); + return new PrivilegedEC2ContainerCredentialsProviderWrapper(); } else { logger.debug("Using basic key/secret credentials"); return new StaticCredentialsProvider(credentials); @@ -157,10 +157,10 @@ protected void doClose() throws ElasticsearchException { IdleConnectionReaper.shutdown(); } - static class PrivilegedInstanceProfileCredentialsProvider implements AWSCredentialsProvider { + static class PrivilegedEC2ContainerCredentialsProviderWrapper implements AWSCredentialsProvider { private final EC2ContainerCredentialsProviderWrapper credentials; - private PrivilegedInstanceProfileCredentialsProvider() { + private PrivilegedEC2ContainerCredentialsProviderWrapper() { this.credentials = new EC2ContainerCredentialsProviderWrapper(); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java index f85f2eb6f322f..7893499d249de 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java @@ -24,8 +24,6 @@ import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import org.elasticsearch.common.settings.MockSecureSettings; -import org.elasticsearch.common.settings.SecureSetting; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; @@ -39,7 +37,7 @@ public void testAWSCredentialsWithSystemProviders() { S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(Settings.EMPTY, "default"); AWSCredentialsProvider credentialsProvider = InternalAwsS3Service.buildCredentials(logger, deprecationLogger, clientSettings, Settings.EMPTY); - assertThat(credentialsProvider, instanceOf(InternalAwsS3Service.PrivilegedInstanceProfileCredentialsProvider.class)); + assertThat(credentialsProvider, instanceOf(InternalAwsS3Service.PrivilegedEC2ContainerCredentialsProviderWrapper.class)); } public void testAwsCredsDefaultSettings() { From 5c32ae86ba2ebcf0cbe466956f64f204e14bf6ae Mon Sep 17 00:00:00 2001 From: Antonio Matarrese Date: Sun, 28 Jan 2018 10:27:31 +0000 Subject: [PATCH 11/11] delete old generated sha1 --- .../repository-s3/licenses/aws-java-sdk-core-1.11.187.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-kms-1.11.187.jar.sha1 | 1 - plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.187.jar.sha1 | 1 - 3 files changed, 3 deletions(-) delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.187.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.187.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.187.jar.sha1 diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.187.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.187.jar.sha1 deleted file mode 100644 index a5293a9bf6580..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.187.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6f47fcd3c2917bef69dc36aba203c5ea4af9bf24 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.187.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.187.jar.sha1 deleted file mode 100644 index ff7c6f616449c..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.187.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f2f4e9afe15bd6a718ad64716042e52f406e668b \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.187.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.187.jar.sha1 deleted file mode 100644 index b7e3a511c8c9d..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.187.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2647df2408c435289293260dc27f0762ce259c30 \ No newline at end of file