Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow list-objects-v2 to run against an S3 Express bucket #3388

Merged
merged 35 commits into from
Feb 17, 2024
Merged
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
dbcfe73
Add placeholder types for S3 Express
ysaito1001 Jan 25, 2024
714aaad
Enable control flow to be altered for S3 Express
ysaito1001 Jan 25, 2024
9086db2
Fix lint-related failures in CI
ysaito1001 Jan 25, 2024
1f07f92
Temporarily disable S3 Express endpoint tests
ysaito1001 Jan 25, 2024
13c3c5a
Update aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk…
ysaito1001 Jan 29, 2024
38ce80f
Make s3_express inlineable modules `pub(crate)`
ysaito1001 Jan 29, 2024
ef1e1be
Remove unnecessary `set_shared_identity_resolver`
ysaito1001 Jan 29, 2024
9e4542b
Allow list-objects-v2 to run against S3 Express bucket
ysaito1001 Jan 29, 2024
67223e3
Add recording test for S3 Express list-objects-v2
ysaito1001 Jan 29, 2024
b30abb2
Exclude `s3_express` from aws-inlineable's module tree
ysaito1001 Jan 30, 2024
2fcd2f7
Enable S3 Express endpoint tests
ysaito1001 Jan 30, 2024
21ad106
Update aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk…
ysaito1001 Feb 7, 2024
9acaf29
Support presigning for S3 Express
ysaito1001 Feb 8, 2024
ec08ae9
Add integration test for S3 Express `get_object` presigning
ysaito1001 Feb 8, 2024
c2a2708
Make `cache_location` a method on `ResolveIdentity`
ysaito1001 Feb 8, 2024
15e89c6
Merge branch 'ysaito/s3express' into s3express-add-placeholders
ysaito1001 Feb 8, 2024
c4df9c7
Error instead of panic on missing bucket name
ysaito1001 Feb 8, 2024
0e405e1
Error instead of panic in `add_token_to_request`
ysaito1001 Feb 9, 2024
e47157f
Update comment for `invalidXmlRootAllowList`
ysaito1001 Feb 9, 2024
5a4e3b2
Implement `from_runtime_components` instead of `From` trait
ysaito1001 Feb 9, 2024
51518f9
Allow `config::Builder` to be created from `ConfigBag`
ysaito1001 Feb 9, 2024
573ef36
Add `TODO` to make `sign_http_request` user-friendly
ysaito1001 Feb 9, 2024
457f6b8
Add a reference to docs for `IdentityCacheLocation`
ysaito1001 Feb 9, 2024
f569231
Use rustTemplate instead of rustBlockTemplate for readability
ysaito1001 Feb 10, 2024
c3d754f
Exclude `s3_express` from `aws-inlineable`'s module tree
ysaito1001 Feb 10, 2024
5b27e4d
Merge branch 'main' into s3express-add-placeholders
ysaito1001 Feb 10, 2024
99a9112
Merge branch 's3express-add-placeholders' into s3express-allow-list-o…
ysaito1001 Feb 10, 2024
3233dbe
Add placeholder types for S3 Express and enable control flow to be re…
ysaito1001 Feb 10, 2024
63dcb93
Merge branch 'ysaito/s3express' into s3express-allow-list-objects-v2-…
ysaito1001 Feb 10, 2024
daea66b
Merge branch 'ysaito/s3express' into s3express-allow-list-objects-v2-…
ysaito1001 Feb 10, 2024
6c795b9
Merge branch 'ysaito/s3express' into s3express-allow-list-objects-v2-…
ysaito1001 Feb 10, 2024
d2354bd
Anonymize tokens in S3 Express recording test
ysaito1001 Feb 12, 2024
027c2cd
Remove `From<RuntimeComponents> for RuntimeComponentsBuilder`
ysaito1001 Feb 14, 2024
830ef02
Add a setting to replace session token name
ysaito1001 Feb 15, 2024
97cecd3
Add a tracking issue for `BuilderFromConfigBag`
ysaito1001 Feb 15, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add recording test for S3 Express list-objects-v2
ysaito1001 committed Jan 29, 2024
commit 67223e311970bfb7cb46b9333b702a8616bd3c81
193 changes: 193 additions & 0 deletions aws/sdk/integration-tests/s3/tests/data/express/list-objects-v2.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
{
"events": [
{
"connection_id": 0,
"action": {
"Request": {
"request": {
"uri": "https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?session",
"headers": {
"amz-sdk-request": [
"attempt=1; max=1"
],
"x-amz-user-agent": [
"aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0"
],
"x-amz-create-session-mode": [
"ReadWrite"
],
"x-amz-date": [
"20090213T233130Z"
],
"authorization": [
"AWS4-HMAC-SHA256 Credential=ANOTREAL/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-create-session-mode;x-amz-date;x-amz-user-agent, Signature=4d44fb95628114b17a0676e2da758dee74f8a0337a3f0183e5b91e4fea6d9303"
],
"user-agent": [
"aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0"
]
},
"method": "GET"
}
}
}
},
{
"connection_id": 0,
"action": {
"Eof": {
"ok": true,
"direction": "Request"
}
}
},
{
"connection_id": 0,
"action": {
"Response": {
"response": {
"Ok": {
"status": 200,
"headers": {
"x-amz-id-2": [
"TQk2NSay"
],
"x-amz-request-id": [
"0033eada6b00018d568cbe9f0509499a7de17df8"
],
"date": [
"Mon, 29 Jan 2024 18:48:00 GMT"
],
"content-type": [
"application/xml"
],
"content-length": [
"1035"
],
"server": [
"AmazonS3"
]
}
}
}
}
}
},
{
"connection_id": 0,
"action": {
"Data": {
"data": {
"Utf8": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<CreateSessionResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Credentials><SessionToken>AgAAAGxnCC7+1qgZF9Zo1aMcI+JZNlNSXzzvpCif4IsEAAAAAAAAADAoC5fHAQAAAAAAAK5b/HX2sWg7Eb+jLXyaCWq0KEMdETvdCBl/bqHtnVkbjoT2p4K7FXiJy7J6VBuCvQhcIlnINa6ed7GF2xgzFNyENQkps2RpLj1lEqWmPAtnoeQqWqYSPwnKh+TVL4vGXfV9mFEeZMEW8eP7o/BWRtYg2ERp3+vUGwRhFRXXIjwTrKe76U/l7mSK79xRV6nMxKJi4/tNG98GKX7FGdEU/dVKEhpe1mRvltkVyA9KnkIwvysLsQLN+rUsbmPhvMManqGCSWqpjbbETp2RUVbg3Vxa9bvlzfW8+BR0CdrZ1QUi5NdwV6gAaJZ8TzNfdvwZJy4asC6ay3t6VBVFtcV+yfWdNx5pHOnxm7siiIbW8iv0jZzuJCzk36QoOAP8Tt3bVOJ592sgkt/ZF1LcmPC1BCXzwG7uuexPdOuoG3WbkEKQxv7Qa84JooPQjaWSHVgQONdLMSSCXjuPn9YAeTyeuOkpARBHlf+KT0D72ISLvZFHn/j/gxnZww2hJUoKpM5obWdbij4yNH+DW8FHmAnRf5DWTBSLWk6SFpE7UvUCRb3LqxAuu0SoaRmdJACtBcKAIgr2HiPwULHYbOlkcKcj5K5W1UA9ssvd</SessionToken><SecretAccessKey>HR6zWlAl1yjsuP8jo/Rw6aqaooVeDeOM2EMn35/b</SecretAccessKey><AccessKeyId>CT7ODKCW3E7XYBDHMKLEOIBWIA</AccessKeyId><Expiration>2024-01-29T18:53:01Z</Expiration></Credentials></CreateSessionResult>"
},
"direction": "Response"
}
}
},
{
"connection_id": 0,
"action": {
"Eof": {
"ok": true,
"direction": "Response"
}
}
},
{
"connection_id": 1,
"action": {
"Request": {
"request": {
"uri": "https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?list-type=2",
"headers": {
"authorization": [
"AWS4-HMAC-SHA256 Credential=CT7ODKCW3E7XYBDHMKLEOIBWIA/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-s3session-token;x-amz-user-agent, Signature=12751fd6558c96b00f3dd624602b0f89e969a8885bada0f5e8b251953ef4e203"
],
"x-amz-date": [
"20090213T233130Z"
],
"amz-sdk-request": [
"attempt=1; max=3"
],
"x-amz-user-agent": [
"aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0"
],
"x-amz-s3session-token": [
"AgAAAGxnCC7+1qgZF9Zo1aMcI+JZNlNSXzzvpCif4IsEAAAAAAAAADAoC5fHAQAAAAAAAK5b/HX2sWg7Eb+jLXyaCWq0KEMdETvdCBl/bqHtnVkbjoT2p4K7FXiJy7J6VBuCvQhcIlnINa6ed7GF2xgzFNyENQkps2RpLj1lEqWmPAtnoeQqWqYSPwnKh+TVL4vGXfV9mFEeZMEW8eP7o/BWRtYg2ERp3+vUGwRhFRXXIjwTrKe76U/l7mSK79xRV6nMxKJi4/tNG98GKX7FGdEU/dVKEhpe1mRvltkVyA9KnkIwvysLsQLN+rUsbmPhvMManqGCSWqpjbbETp2RUVbg3Vxa9bvlzfW8+BR0CdrZ1QUi5NdwV6gAaJZ8TzNfdvwZJy4asC6ay3t6VBVFtcV+yfWdNx5pHOnxm7siiIbW8iv0jZzuJCzk36QoOAP8Tt3bVOJ592sgkt/ZF1LcmPC1BCXzwG7uuexPdOuoG3WbkEKQxv7Qa84JooPQjaWSHVgQONdLMSSCXjuPn9YAeTyeuOkpARBHlf+KT0D72ISLvZFHn/j/gxnZww2hJUoKpM5obWdbij4yNH+DW8FHmAnRf5DWTBSLWk6SFpE7UvUCRb3LqxAuu0SoaRmdJACtBcKAIgr2HiPwULHYbOlkcKcj5K5W1UA9ssvd"
],
"user-agent": [
"aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0"
]
},
"method": "GET"
}
}
}
},
{
"connection_id": 1,
"action": {
"Eof": {
"ok": true,
"direction": "Request"
}
}
},
{
"connection_id": 1,
"action": {
"Response": {
"response": {
"Ok": {
"status": 200,
"headers": {
"content-type": [
"application/xml"
],
"date": [
"Mon, 29 Jan 2024 18:48:00 GMT"
],
"x-amz-request-id": [
"0033eada6b00018d568cbf350509f775295f94b5"
],
"x-amz-id-2": [
"IltTpRJVF1U"
],
"server": [
"AmazonS3"
],
"content-length": [
"520"
],
"x-amz-bucket-region": [
"us-west-2"
]
}
}
}
}
}
},
{
"connection_id": 1,
"action": {
"Data": {
"data": {
"Utf8": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>s3express-test-bucket--usw2-az1--x-s3</Name><Prefix/><KeyCount>1</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><ChecksumAlgorithm>CRC32</ChecksumAlgorithm><ETag>&quot;b357dc928b454965a8dd11716a37dab8&quot;</ETag><Key>hello-world.txt</Key><LastModified>2024-01-29T18:32:24.000Z</LastModified><Size>14</Size><StorageClass>EXPRESS_ONEZONE</StorageClass></Contents></ListBucketResult>"
},
"direction": "Response"
}
}
},
{
"connection_id": 1,
"action": {
"Eof": {
"ok": true,
"direction": "Response"
}
}
}
],
"docs": "traffic recording of executing list-objects-v2 against an S3 Express One Zone bucket",
"version": "V0"
}
39 changes: 39 additions & 0 deletions aws/sdk/integration-tests/s3/tests/express.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/

use aws_sdk_s3::Config;
use aws_smithy_runtime::client::http::test_util::dvr::ReplayingClient;
use aws_smithy_runtime::test_util::capture_test_logs::capture_test_logs;

#[tokio::test]
async fn list_objects_v2() {
let _logs = capture_test_logs();

let http_client =
ReplayingClient::from_file("tests/data/express/list-objects-v2.json").unwrap();
let config = aws_config::from_env()
.http_client(http_client.clone())
.no_credentials()
.region("us-west-2")
.load()
.await;
let config = Config::from(&config)
.to_builder()
.with_test_defaults()
.build();
let client = aws_sdk_s3::Client::from_conf(config);

let result = client
.list_objects_v2()
.bucket("s3express-test-bucket--usw2-az1--x-s3")
.send()
.await;
dbg!(result).expect("success");

http_client
.validate_body_and_headers(Some(&["x-amz-s3session-token"]), "application/xml")
.await
.unwrap();
}