Skip to content

Commit a3b50c3

Browse files
jed326Jay Deng
jed326
authored and
Jay Deng
committed
add sse-kms
Signed-off-by: Jay Deng <[email protected]>
1 parent 557ea3a commit a3b50c3

File tree

6 files changed

+210
-9
lines changed

6 files changed

+210
-9
lines changed

‎plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,9 @@
9595
import java.io.ByteArrayInputStream;
9696
import java.io.IOException;
9797
import java.io.InputStream;
98+
import java.nio.charset.StandardCharsets;
9899
import java.util.ArrayList;
100+
import java.util.Base64;
99101
import java.util.List;
100102
import java.util.Map;
101103
import java.util.concurrent.CompletableFuture;
@@ -540,7 +542,24 @@ void executeSingleUpload(
540542
putObjectRequestBuilder = putObjectRequestBuilder.metadata(metadata);
541543
}
542544
if (blobStore.serverSideEncryption()) {
543-
putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AES256);
545+
if (blobStore.serverSideEncryptionType().equals(ServerSideEncryption.AES256.toString())) {
546+
putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AES256);
547+
} else if (blobStore.serverSideEncryptionType().equals(ServerSideEncryption.AWS_KMS.toString())) {
548+
putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS);
549+
putObjectRequestBuilder.ssekmsKeyId(blobStore.serverSideEncryptionKmsKey());
550+
putObjectRequestBuilder.bucketKeyEnabled(blobStore.serverSideEncryptionBucketKey());
551+
if (blobStore.serverSideEncryptionEncryptionContext() != null
552+
&& !blobStore.serverSideEncryptionEncryptionContext().isEmpty()) {
553+
putObjectRequestBuilder.ssekmsEncryptionContext(
554+
Base64.getEncoder()
555+
.encodeToString(blobStore.serverSideEncryptionEncryptionContext().getBytes(StandardCharsets.UTF_8))
556+
);
557+
}
558+
}
559+
}
560+
561+
if (blobStore.expectedBucketOwner() != null && !blobStore.expectedBucketOwner().isEmpty()) {
562+
putObjectRequestBuilder.expectedBucketOwner(blobStore.expectedBucketOwner());
544563
}
545564

546565
PutObjectRequest putObjectRequest = putObjectRequestBuilder.build();
@@ -598,7 +617,24 @@ void executeMultipartUpload(
598617
}
599618

600619
if (blobStore.serverSideEncryption()) {
601-
createMultipartUploadRequestBuilder.serverSideEncryption(ServerSideEncryption.AES256);
620+
if (blobStore.serverSideEncryptionType().equals(ServerSideEncryption.AES256.toString())) {
621+
createMultipartUploadRequestBuilder.serverSideEncryption(ServerSideEncryption.AES256);
622+
} else if (blobStore.serverSideEncryptionType().equals(ServerSideEncryption.AWS_KMS.toString())) {
623+
createMultipartUploadRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS);
624+
createMultipartUploadRequestBuilder.ssekmsKeyId(blobStore.serverSideEncryptionKmsKey());
625+
createMultipartUploadRequestBuilder.bucketKeyEnabled(blobStore.serverSideEncryptionBucketKey());
626+
if (blobStore.serverSideEncryptionEncryptionContext() != null
627+
&& !blobStore.serverSideEncryptionEncryptionContext().isEmpty()) {
628+
createMultipartUploadRequestBuilder.ssekmsEncryptionContext(
629+
Base64.getEncoder()
630+
.encodeToString(blobStore.serverSideEncryptionEncryptionContext().getBytes(StandardCharsets.UTF_8))
631+
);
632+
}
633+
}
634+
}
635+
636+
if (blobStore.expectedBucketOwner() != null && !blobStore.expectedBucketOwner().isEmpty()) {
637+
createMultipartUploadRequestBuilder.expectedBucketOwner(blobStore.expectedBucketOwner());
602638
}
603639

604640
final InputStream requestInputStream;

‎plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,14 @@
5757
import static org.opensearch.repositories.s3.S3Repository.BUFFER_SIZE_SETTING;
5858
import static org.opensearch.repositories.s3.S3Repository.BULK_DELETE_SIZE;
5959
import static org.opensearch.repositories.s3.S3Repository.CANNED_ACL_SETTING;
60+
import static org.opensearch.repositories.s3.S3Repository.EXPECTED_BUCKET_OWNER_SETTING;
6061
import static org.opensearch.repositories.s3.S3Repository.PERMIT_BACKED_TRANSFER_ENABLED;
6162
import static org.opensearch.repositories.s3.S3Repository.REDIRECT_LARGE_S3_UPLOAD;
63+
import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_BUCKET_KEY_SETTING;
64+
import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_ENCRYPTION_CONTEXT_SETTING;
65+
import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_KMS_KEY_SETTING;
6266
import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_SETTING;
67+
import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_TYPE_SETTING;
6368
import static org.opensearch.repositories.s3.S3Repository.STORAGE_CLASS_SETTING;
6469
import static org.opensearch.repositories.s3.S3Repository.UPLOAD_RETRY_ENABLED;
6570

@@ -82,6 +87,11 @@ public class S3BlobStore implements BlobStore {
8287
private volatile boolean permitBackedTransferEnabled;
8388

8489
private volatile boolean serverSideEncryption;
90+
private volatile String serverSideEncryptionType;
91+
private volatile String serverSideEncryptionKmsKey;
92+
private volatile boolean serverSideEncryptionBucketKey;
93+
private volatile String serverSideEncryptionEncryptionContext;
94+
private volatile String expectedBucketOwner;
8595

8696
private volatile ObjectCannedACL cannedACL;
8797

@@ -119,7 +129,12 @@ public class S3BlobStore implements BlobStore {
119129
AsyncExecutorContainer normalExecutorBuilder,
120130
SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ,
121131
SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ,
122-
GenericStatsMetricPublisher genericStatsMetricPublisher
132+
GenericStatsMetricPublisher genericStatsMetricPublisher,
133+
String serverSideEncryptionType,
134+
String serverSideEncryptionKmsKey,
135+
boolean serverSideEncryptionBucketKey,
136+
String serverSideEncryptionEncryptionContext,
137+
String expectedBucketOwner
123138
) {
124139
this.service = service;
125140
this.s3AsyncService = s3AsyncService;
@@ -142,6 +157,11 @@ public class S3BlobStore implements BlobStore {
142157
this.lowPrioritySizeBasedBlockingQ = lowPrioritySizeBasedBlockingQ;
143158
this.genericStatsMetricPublisher = genericStatsMetricPublisher;
144159
this.permitBackedTransferEnabled = PERMIT_BACKED_TRANSFER_ENABLED.get(repositoryMetadata.settings());
160+
this.serverSideEncryptionType = serverSideEncryptionType;
161+
this.serverSideEncryptionKmsKey = serverSideEncryptionKmsKey;
162+
this.serverSideEncryptionBucketKey = serverSideEncryptionBucketKey;
163+
this.serverSideEncryptionEncryptionContext = serverSideEncryptionEncryptionContext;
164+
this.expectedBucketOwner = expectedBucketOwner;
145165
}
146166

147167
@Override
@@ -156,6 +176,11 @@ public void reload(RepositoryMetadata repositoryMetadata) {
156176
this.redirectLargeUploads = REDIRECT_LARGE_S3_UPLOAD.get(repositoryMetadata.settings());
157177
this.uploadRetryEnabled = UPLOAD_RETRY_ENABLED.get(repositoryMetadata.settings());
158178
this.permitBackedTransferEnabled = PERMIT_BACKED_TRANSFER_ENABLED.get(repositoryMetadata.settings());
179+
this.serverSideEncryptionType = SERVER_SIDE_ENCRYPTION_TYPE_SETTING.get(repositoryMetadata.settings());
180+
this.serverSideEncryptionKmsKey = SERVER_SIDE_ENCRYPTION_KMS_KEY_SETTING.get(repositoryMetadata.settings());
181+
this.serverSideEncryptionBucketKey = SERVER_SIDE_ENCRYPTION_BUCKET_KEY_SETTING.get(repositoryMetadata.settings());
182+
this.serverSideEncryptionEncryptionContext = SERVER_SIDE_ENCRYPTION_ENCRYPTION_CONTEXT_SETTING.get(repositoryMetadata.settings());
183+
this.expectedBucketOwner = EXPECTED_BUCKET_OWNER_SETTING.get(repositoryMetadata.settings());
159184
}
160185

161186
@Override
@@ -195,6 +220,26 @@ public boolean serverSideEncryption() {
195220
return serverSideEncryption;
196221
}
197222

223+
public String serverSideEncryptionType() {
224+
return serverSideEncryptionType;
225+
}
226+
227+
public String serverSideEncryptionKmsKey() {
228+
return serverSideEncryptionKmsKey;
229+
}
230+
231+
public boolean serverSideEncryptionBucketKey() {
232+
return serverSideEncryptionBucketKey;
233+
}
234+
235+
public String serverSideEncryptionEncryptionContext() {
236+
return serverSideEncryptionEncryptionContext;
237+
}
238+
239+
public String expectedBucketOwner() {
240+
return expectedBucketOwner;
241+
}
242+
198243
public long bufferSizeInBytes() {
199244
return bufferSize.getBytes();
200245
}

‎plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java‎

Lines changed: 71 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
package org.opensearch.repositories.s3;
3434

3535
import software.amazon.awssdk.services.s3.model.ObjectCannedACL;
36+
import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
3637
import software.amazon.awssdk.services.s3.model.StorageClass;
3738

3839
import org.apache.logging.log4j.LogManager;
@@ -123,11 +124,56 @@ class S3Repository extends MeteredBlobStoreRepository {
123124
static final Setting<String> BUCKET_SETTING = Setting.simpleString("bucket");
124125

125126
/**
126-
* When set to true files are encrypted on server side using AES256 algorithm.
127+
* When set to true files are encrypted on server side using AES256 algorithm by default.
127128
* Defaults to false.
128129
*/
129130
static final Setting<Boolean> SERVER_SIDE_ENCRYPTION_SETTING = Setting.boolSetting("server_side_encryption", false);
130131

132+
/**
133+
* The type of S3 Server Side Encryption to use.
134+
* Defaults to AES256.
135+
* Supports: AES256, aws:kms
136+
*/
137+
static final Setting<String> SERVER_SIDE_ENCRYPTION_TYPE_SETTING = Setting.simpleString(
138+
"server_side_encryption_type",
139+
ServerSideEncryption.AES256.toString(),
140+
value -> {
141+
if (!(value.equals(ServerSideEncryption.AES256.toString()) || value.equals(ServerSideEncryption.AWS_KMS.toString()))) {
142+
throw new IllegalArgumentException("server_side_encryption_type must be one of [AES256, aws:kms]");
143+
}
144+
}
145+
);
146+
147+
/**
148+
* The KMS key id to be used for SSE-KMS. Must be used when server_side_encryption_type setting is set to aws:kms.
149+
*/
150+
static final Setting<String> SERVER_SIDE_ENCRYPTION_KMS_KEY_SETTING = Setting.simpleString("server_side_encryption_kms_key_id");
151+
152+
/**
153+
* Whether to use S3 Bucket Keys along with SSE-KMS.
154+
* Defaults to true.
155+
*/
156+
static final Setting<Boolean> SERVER_SIDE_ENCRYPTION_BUCKET_KEY_SETTING = Setting.boolSetting(
157+
"server_side_encryption_bucket_key_enabled",
158+
true
159+
);
160+
161+
/**
162+
* Optional additional encryption context passed to S3 for use in KMS crypto operations. The setting value must be formatted as a key-value pair JSON object.
163+
*/
164+
static final Setting<String> SERVER_SIDE_ENCRYPTION_ENCRYPTION_CONTEXT_SETTING = Setting.simpleString(
165+
"server_side_encryption_encryption_context"
166+
);
167+
168+
/**
169+
* Optional setting to specify the expected S3 bucket owner. This is used to verify S3 bucket ownership before reading/writing data from/to a bucket.
170+
*/
171+
static final Setting<String> EXPECTED_BUCKET_OWNER_SETTING = Setting.simpleString("expected_bucket_owner", value -> {
172+
if (!(value.matches("\\d{12}") || value.isEmpty())) {
173+
throw new IllegalArgumentException("expected_bucket_owner must be a 12 digit AWS account id");
174+
}
175+
});
176+
131177
/**
132178
* Maximum size of files that can be uploaded using a single upload request.
133179
*/
@@ -284,6 +330,11 @@ class S3Repository extends MeteredBlobStoreRepository {
284330
private volatile BlobPath basePath;
285331

286332
private volatile boolean serverSideEncryption;
333+
private volatile String serverSideEncryptionType;
334+
private volatile String serverSideEncryptionKmsKey;
335+
private volatile boolean serverSideEncryptionBucketKey;
336+
private volatile String serverSideEncryptionEncryptionContext;
337+
private volatile String expectedBucketOwner;
287338

288339
private volatile String storageClass;
289340

@@ -436,7 +487,12 @@ protected S3BlobStore createBlobStore() {
436487
normalExecutorBuilder,
437488
normalPrioritySizeBasedBlockingQ,
438489
lowPrioritySizeBasedBlockingQ,
439-
genericStatsMetricPublisher
490+
genericStatsMetricPublisher,
491+
serverSideEncryptionType,
492+
serverSideEncryptionKmsKey,
493+
serverSideEncryptionBucketKey,
494+
serverSideEncryptionEncryptionContext,
495+
expectedBucketOwner
440496
);
441497
}
442498

@@ -492,6 +548,11 @@ private void readRepositoryMetadata() {
492548
}
493549

494550
this.serverSideEncryption = SERVER_SIDE_ENCRYPTION_SETTING.get(metadata.settings());
551+
this.serverSideEncryptionType = SERVER_SIDE_ENCRYPTION_TYPE_SETTING.get(metadata.settings());
552+
this.serverSideEncryptionKmsKey = SERVER_SIDE_ENCRYPTION_KMS_KEY_SETTING.get(metadata.settings());
553+
this.serverSideEncryptionBucketKey = SERVER_SIDE_ENCRYPTION_BUCKET_KEY_SETTING.get(metadata.settings());
554+
this.serverSideEncryptionEncryptionContext = SERVER_SIDE_ENCRYPTION_ENCRYPTION_CONTEXT_SETTING.get(metadata.settings());
555+
this.expectedBucketOwner = EXPECTED_BUCKET_OWNER_SETTING.get(metadata.settings());
495556
this.storageClass = STORAGE_CLASS_SETTING.get(metadata.settings());
496557
this.cannedACL = CANNED_ACL_SETTING.get(metadata.settings());
497558
this.bulkDeletesSize = BULK_DELETE_SIZE.get(metadata.settings());
@@ -505,13 +566,19 @@ private void readRepositoryMetadata() {
505566
}
506567

507568
logger.debug(
508-
"using bucket [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], cannedACL [{}], storageClass [{}]",
569+
"using bucket [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], cannedACL [{}], storageClass [{}], "
570+
+ "serverSideEncryptionType [{}], serverSideEncryptionKmsKey [{}], serverSideEncryptionBucketKey [{}], serverSideEncryptionEncryptionContext [{}], expectedBucketOwner [{}], ",
509571
bucket,
510572
chunkSize,
511573
serverSideEncryption,
512574
bufferSize,
513575
cannedACL,
514-
storageClass
576+
storageClass,
577+
serverSideEncryptionType,
578+
serverSideEncryptionKmsKey,
579+
serverSideEncryptionBucketKey,
580+
serverSideEncryptionEncryptionContext,
581+
expectedBucketOwner
515582
);
516583
}
517584

‎plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import software.amazon.awssdk.services.s3.model.ObjectCannedACL;
2525
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
2626
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
27+
import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
2728
import software.amazon.awssdk.services.s3.model.StorageClass;
2829
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
2930
import software.amazon.awssdk.services.s3.model.UploadPartResponse;
@@ -79,6 +80,11 @@
7980
import org.mockito.invocation.InvocationOnMock;
8081

8182
import static org.opensearch.repositories.s3.S3Repository.BULK_DELETE_SIZE;
83+
import static org.opensearch.repositories.s3.S3Repository.EXPECTED_BUCKET_OWNER_SETTING;
84+
import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_BUCKET_KEY_SETTING;
85+
import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_ENCRYPTION_CONTEXT_SETTING;
86+
import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_KMS_KEY_SETTING;
87+
import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_TYPE_SETTING;
8288
import static org.mockito.ArgumentMatchers.anyLong;
8389
import static org.mockito.ArgumentMatchers.anyMap;
8490
import static org.mockito.ArgumentMatchers.anyString;
@@ -461,7 +467,12 @@ private S3BlobStore createBlobStore() {
461467
asyncExecutorContainer,
462468
normalPrioritySizeBasedBlockingQ,
463469
lowPrioritySizeBasedBlockingQ,
464-
genericStatsMetricPublisher
470+
genericStatsMetricPublisher,
471+
SERVER_SIDE_ENCRYPTION_TYPE_SETTING.getDefault(Settings.EMPTY),
472+
SERVER_SIDE_ENCRYPTION_KMS_KEY_SETTING.getDefault(Settings.EMPTY),
473+
SERVER_SIDE_ENCRYPTION_BUCKET_KEY_SETTING.getDefault(Settings.EMPTY),
474+
SERVER_SIDE_ENCRYPTION_ENCRYPTION_CONTEXT_SETTING.getDefault(Settings.EMPTY),
475+
EXPECTED_BUCKET_OWNER_SETTING.getDefault(Settings.EMPTY)
465476
);
466477
}
467478

@@ -674,6 +685,18 @@ private void testLargeFilesRedirectedToSlowSyncClient(boolean expectException, W
674685

675686
final boolean serverSideEncryption = randomBoolean();
676687
when(blobStore.serverSideEncryption()).thenReturn(serverSideEncryption);
688+
if (serverSideEncryption) {
689+
if (randomBoolean()) {
690+
when(blobStore.serverSideEncryptionType()).thenReturn(ServerSideEncryption.AES256.toString());
691+
} else {
692+
when(blobStore.serverSideEncryptionType()).thenReturn(ServerSideEncryption.AWS_KMS.toString());
693+
when(blobStore.serverSideEncryptionKmsKey()).thenReturn(randomAlphaOfLength(10));
694+
when(blobStore.serverSideEncryptionBucketKey()).thenReturn(randomBoolean());
695+
when(blobStore.serverSideEncryptionEncryptionContext()).thenReturn(randomAlphaOfLength(10));
696+
}
697+
}
698+
699+
when(blobStore.expectedBucketOwner()).thenReturn(randomAlphaOfLength(12));
677700

678701
final StorageClass storageClass = randomFrom(StorageClass.values());
679702
when(blobStore.getStorageClass()).thenReturn(storageClass);

‎plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,11 @@
103103
import static org.opensearch.repositories.s3.S3ClientSettings.READ_TIMEOUT_SETTING;
104104
import static org.opensearch.repositories.s3.S3ClientSettings.REGION;
105105
import static org.opensearch.repositories.s3.S3Repository.BULK_DELETE_SIZE;
106+
import static org.opensearch.repositories.s3.S3Repository.EXPECTED_BUCKET_OWNER_SETTING;
107+
import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_BUCKET_KEY_SETTING;
108+
import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_ENCRYPTION_CONTEXT_SETTING;
109+
import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_KMS_KEY_SETTING;
110+
import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_TYPE_SETTING;
106111
import static org.hamcrest.Matchers.anyOf;
107112
import static org.hamcrest.Matchers.containsString;
108113
import static org.hamcrest.Matchers.equalTo;
@@ -268,7 +273,12 @@ protected AsyncMultiStreamBlobContainer createBlobContainer(
268273
asyncExecutorContainer,
269274
normalPrioritySizeBasedBlockingQ,
270275
lowPrioritySizeBasedBlockingQ,
271-
genericStatsMetricPublisher
276+
genericStatsMetricPublisher,
277+
SERVER_SIDE_ENCRYPTION_TYPE_SETTING.getDefault(Settings.EMPTY),
278+
SERVER_SIDE_ENCRYPTION_KMS_KEY_SETTING.getDefault(Settings.EMPTY),
279+
SERVER_SIDE_ENCRYPTION_BUCKET_KEY_SETTING.getDefault(Settings.EMPTY),
280+
SERVER_SIDE_ENCRYPTION_ENCRYPTION_CONTEXT_SETTING.getDefault(Settings.EMPTY),
281+
EXPECTED_BUCKET_OWNER_SETTING.getDefault(Settings.EMPTY)
272282
)
273283
) {
274284
@Override

0 commit comments

Comments
 (0)